diff options
Diffstat (limited to '')
39 files changed, 13126 insertions, 0 deletions
diff --git a/src/lib-dict-backend/Makefile.am b/src/lib-dict-backend/Makefile.am new file mode 100644 index 0000000..80c5050 --- /dev/null +++ b/src/lib-dict-backend/Makefile.am @@ -0,0 +1,116 @@ +noinst_LTLIBRARIES = libdict_backend.la + +module_dictdir = $(moduledir)/dict +dict_drivers = @dict_drivers@ + +AM_CPPFLAGS = \ + -I$(top_srcdir)/src/lib \ + -I$(top_srcdir)/src/lib-test \ + -I$(top_srcdir)/src/lib-dict \ + -I$(top_srcdir)/src/lib-ldap \ + -I$(top_srcdir)/src/lib-sql \ + -I$(top_srcdir)/src/lib-settings \ + $(SQL_CFLAGS) + +NOPLUGIN_LDFLAGS = + +ldap_sources = \ + dict-ldap.c \ + dict-ldap-settings.c + +libdict_backend_la_SOURCES = \ + dict-cdb.c \ + dict-sql.c \ + dict-sql-settings.c \ + $(ldap_sources) +libdict_backend_la_LIBADD = + +nodist_libdict_backend_la_SOURCES = \ + dict-drivers-register.c + +noinst_HEADERS = \ + dict-ldap-settings.h \ + dict-sql.h \ + dict-sql-private.h \ + dict-sql-settings.h + +if LDAP_PLUGIN +LIBDICT_LDAP = libdict_ldap.la +libdict_ldap_la_DEPENDENCIES = $(LIBDOVECOT_LDAP) $(LIBDOVECOT_DEPS) +libdict_ldap_la_LDFLAGS = -module -avoid-version +libdict_ldap_la_LIBADD = $(LIBDOVECOT_LDAP) $(LIBDOVECOT) +libdict_ldap_la_CPPFLAGS = $(AM_CPPFLAGS) -DPLUGIN_BUILD +libdict_ldap_la_SOURCES = $(ldap_sources) +else +if HAVE_LDAP +libdict_backend_la_LIBADD += $(LIBDOVECOT_LDAP) +dict_drivers += ldap +endif +endif + +module_dict_LTLIBRARIES = \ + $(LIBDICT_LDAP) + +EXTRA_DIST = dict.conf + +dict-drivers-register.c: Makefile $(top_builddir)/config.h + rm -f $@ + echo '/* this file automatically generated by Makefile */' >$@ + echo '#include "lib.h"' >>$@ + echo '#include "dict.h"' >>$@ + echo '#include "ldap-client.h"' >>$@ + echo '#include "dict-sql.h"' >>$@ + for i in $(dict_drivers) null; do \ + if [ "$${i}" != "null" ]; then \ + echo "extern struct dict dict_driver_$${i};" >>$@ ; \ + fi; \ + done + echo 'void dict_drivers_register_all(void) {' >>$@ + echo 'dict_drivers_register_builtin();' >>$@ + echo 'dict_sql_register();' >>$@ + for i in $(dict_drivers) null; do \ + if [ "$${i}" != "null" ]; then \ + echo "dict_driver_register(&dict_driver_$${i});" >>$@ ; \ + fi; \ + done + echo '}' >>$@ + echo 'void dict_drivers_unregister_all(void) {' >>$@ + echo '#ifdef BUILTIN_LDAP' >>$@ + echo 'ldap_clients_cleanup();' >>$@ + echo '#endif' >>$@ + echo 'dict_drivers_unregister_builtin();' >>$@ + echo 'dict_sql_unregister();' >>$@ + for i in $(dict_drivers) null; do \ + if [ "$${i}" != "null" ]; then \ + echo "dict_driver_unregister(&dict_driver_$${i});" >>$@ ; \ + fi; \ + done + echo '}' >>$@ + +distclean-generic: + rm -f Makefile dict-drivers-register.c + +test_programs = \ + test-dict-sql + +noinst_PROGRAMS = $(test_programs) + +test_dict_sql_CFLAGS = $(AM_CPPFLAGS) -DDICT_SRC_DIR=\"$(top_srcdir)/src/lib-dict-backend\" +test_dict_sql_SOURCES = \ + test-dict-sql.c +test_dict_sql_LDADD = \ + $(noinst_LTLIBRARIES) \ + $(DICT_LIBS) \ + ../lib-sql/libdriver_test.la \ + ../lib-sql/libsql.la \ + ../lib-dovecot/libdovecot.la +test_dict_sql_DEPENDENCIES = \ + $(noinst_LTLIBRARIES) \ + ../lib-sql/libdriver_test.la \ + ../lib-sql/libsql.la \ + ../lib-dovecot/libdovecot.la + +check-local: + for bin in $(test_programs) $(check_PROGRAMS); do \ + if ! $(RUN_TEST) ./$$bin; then exit 1; fi; \ + done diff --git a/src/lib-dict-backend/Makefile.in b/src/lib-dict-backend/Makefile.in new file mode 100644 index 0000000..e5162e9 --- /dev/null +++ b/src/lib-dict-backend/Makefile.in @@ -0,0 +1,1016 @@ +# Makefile.in generated by automake 1.16.1 from Makefile.am. +# @configure_input@ + +# Copyright (C) 1994-2018 Free Software Foundation, Inc. + +# This Makefile.in is free software; the Free Software Foundation +# gives unlimited permission to copy and/or distribute it, +# with or without modifications, as long as this notice is preserved. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY, to the extent permitted by law; without +# even the implied warranty of MERCHANTABILITY or FITNESS FOR A +# PARTICULAR PURPOSE. + +@SET_MAKE@ + + + +VPATH = @srcdir@ +am__is_gnu_make = { \ + if test -z '$(MAKELEVEL)'; then \ + false; \ + elif test -n '$(MAKE_HOST)'; then \ + true; \ + elif test -n '$(MAKE_VERSION)' && test -n '$(CURDIR)'; then \ + true; \ + else \ + false; \ + fi; \ +} +am__make_running_with_option = \ + case $${target_option-} in \ + ?) ;; \ + *) echo "am__make_running_with_option: internal error: invalid" \ + "target option '$${target_option-}' specified" >&2; \ + exit 1;; \ + esac; \ + has_opt=no; \ + sane_makeflags=$$MAKEFLAGS; \ + if $(am__is_gnu_make); then \ + sane_makeflags=$$MFLAGS; \ + else \ + case $$MAKEFLAGS in \ + *\\[\ \ ]*) \ + bs=\\; \ + sane_makeflags=`printf '%s\n' "$$MAKEFLAGS" \ + | sed "s/$$bs$$bs[$$bs $$bs ]*//g"`;; \ + esac; \ + fi; \ + skip_next=no; \ + strip_trailopt () \ + { \ + flg=`printf '%s\n' "$$flg" | sed "s/$$1.*$$//"`; \ + }; \ + for flg in $$sane_makeflags; do \ + test $$skip_next = yes && { skip_next=no; continue; }; \ + case $$flg in \ + *=*|--*) continue;; \ + -*I) strip_trailopt 'I'; skip_next=yes;; \ + -*I?*) strip_trailopt 'I';; \ + -*O) strip_trailopt 'O'; skip_next=yes;; \ + -*O?*) strip_trailopt 'O';; \ + -*l) strip_trailopt 'l'; skip_next=yes;; \ + -*l?*) strip_trailopt 'l';; \ + -[dEDm]) skip_next=yes;; \ + -[JT]) skip_next=yes;; \ + esac; \ + case $$flg in \ + *$$target_option*) has_opt=yes; break;; \ + esac; \ + done; \ + test $$has_opt = yes +am__make_dryrun = (target_option=n; $(am__make_running_with_option)) +am__make_keepgoing = (target_option=k; $(am__make_running_with_option)) +pkgdatadir = $(datadir)/@PACKAGE@ +pkgincludedir = $(includedir)/@PACKAGE@ +pkglibdir = $(libdir)/@PACKAGE@ +pkglibexecdir = $(libexecdir)/@PACKAGE@ +am__cd = CDPATH="$${ZSH_VERSION+.}$(PATH_SEPARATOR)" && cd +install_sh_DATA = $(install_sh) -c -m 644 +install_sh_PROGRAM = $(install_sh) -c +install_sh_SCRIPT = $(install_sh) -c +INSTALL_HEADER = $(INSTALL_DATA) +transform = $(program_transform_name) +NORMAL_INSTALL = : +PRE_INSTALL = : +POST_INSTALL = : +NORMAL_UNINSTALL = : +PRE_UNINSTALL = : +POST_UNINSTALL = : +build_triplet = @build@ +host_triplet = @host@ +@HAVE_LDAP_TRUE@@LDAP_PLUGIN_FALSE@am__append_1 = $(LIBDOVECOT_LDAP) +@HAVE_LDAP_TRUE@@LDAP_PLUGIN_FALSE@am__append_2 = ldap +noinst_PROGRAMS = $(am__EXEEXT_1) +subdir = src/lib-dict-backend +ACLOCAL_M4 = $(top_srcdir)/aclocal.m4 +am__aclocal_m4_deps = $(top_srcdir)/m4/ac_checktype2.m4 \ + $(top_srcdir)/m4/ac_typeof.m4 $(top_srcdir)/m4/arc4random.m4 \ + $(top_srcdir)/m4/blockdev.m4 $(top_srcdir)/m4/c99_vsnprintf.m4 \ + $(top_srcdir)/m4/clock_gettime.m4 $(top_srcdir)/m4/crypt.m4 \ + $(top_srcdir)/m4/crypt_xpg6.m4 $(top_srcdir)/m4/dbqlk.m4 \ + $(top_srcdir)/m4/dirent_dtype.m4 $(top_srcdir)/m4/dovecot.m4 \ + $(top_srcdir)/m4/fd_passing.m4 $(top_srcdir)/m4/fdatasync.m4 \ + $(top_srcdir)/m4/flexible_array_member.m4 \ + $(top_srcdir)/m4/glibc.m4 $(top_srcdir)/m4/gmtime_max.m4 \ + $(top_srcdir)/m4/gmtime_tm_gmtoff.m4 \ + $(top_srcdir)/m4/ioloop.m4 $(top_srcdir)/m4/iovec.m4 \ + $(top_srcdir)/m4/ipv6.m4 $(top_srcdir)/m4/libcap.m4 \ + $(top_srcdir)/m4/libtool.m4 $(top_srcdir)/m4/libwrap.m4 \ + $(top_srcdir)/m4/linux_mremap.m4 $(top_srcdir)/m4/ltoptions.m4 \ + $(top_srcdir)/m4/ltsugar.m4 $(top_srcdir)/m4/ltversion.m4 \ + $(top_srcdir)/m4/lt~obsolete.m4 $(top_srcdir)/m4/mmap_write.m4 \ + $(top_srcdir)/m4/mntctl.m4 $(top_srcdir)/m4/modules.m4 \ + $(top_srcdir)/m4/notify.m4 $(top_srcdir)/m4/nsl.m4 \ + $(top_srcdir)/m4/off_t_max.m4 $(top_srcdir)/m4/pkg.m4 \ + $(top_srcdir)/m4/pr_set_dumpable.m4 \ + $(top_srcdir)/m4/q_quotactl.m4 $(top_srcdir)/m4/quota.m4 \ + $(top_srcdir)/m4/random.m4 $(top_srcdir)/m4/rlimit.m4 \ + $(top_srcdir)/m4/sendfile.m4 $(top_srcdir)/m4/size_t_signed.m4 \ + $(top_srcdir)/m4/sockpeercred.m4 $(top_srcdir)/m4/sql.m4 \ + $(top_srcdir)/m4/ssl.m4 $(top_srcdir)/m4/st_tim.m4 \ + $(top_srcdir)/m4/static_array.m4 $(top_srcdir)/m4/test_with.m4 \ + $(top_srcdir)/m4/time_t.m4 $(top_srcdir)/m4/typeof.m4 \ + $(top_srcdir)/m4/typeof_dev_t.m4 \ + $(top_srcdir)/m4/uoff_t_max.m4 $(top_srcdir)/m4/vararg.m4 \ + $(top_srcdir)/m4/want_apparmor.m4 \ + $(top_srcdir)/m4/want_bsdauth.m4 \ + $(top_srcdir)/m4/want_bzlib.m4 \ + $(top_srcdir)/m4/want_cassandra.m4 \ + $(top_srcdir)/m4/want_cdb.m4 \ + $(top_srcdir)/m4/want_checkpassword.m4 \ + $(top_srcdir)/m4/want_clucene.m4 $(top_srcdir)/m4/want_db.m4 \ + $(top_srcdir)/m4/want_gssapi.m4 $(top_srcdir)/m4/want_icu.m4 \ + $(top_srcdir)/m4/want_ldap.m4 $(top_srcdir)/m4/want_lua.m4 \ + $(top_srcdir)/m4/want_lz4.m4 $(top_srcdir)/m4/want_lzma.m4 \ + $(top_srcdir)/m4/want_mysql.m4 $(top_srcdir)/m4/want_pam.m4 \ + $(top_srcdir)/m4/want_passwd.m4 $(top_srcdir)/m4/want_pgsql.m4 \ + $(top_srcdir)/m4/want_prefetch.m4 \ + $(top_srcdir)/m4/want_shadow.m4 \ + $(top_srcdir)/m4/want_sodium.m4 $(top_srcdir)/m4/want_solr.m4 \ + $(top_srcdir)/m4/want_sqlite.m4 \ + $(top_srcdir)/m4/want_stemmer.m4 \ + $(top_srcdir)/m4/want_systemd.m4 \ + $(top_srcdir)/m4/want_textcat.m4 \ + $(top_srcdir)/m4/want_unwind.m4 $(top_srcdir)/m4/want_zlib.m4 \ + $(top_srcdir)/m4/want_zstd.m4 $(top_srcdir)/configure.ac +am__configure_deps = $(am__aclocal_m4_deps) $(CONFIGURE_DEPENDENCIES) \ + $(ACLOCAL_M4) +DIST_COMMON = $(srcdir)/Makefile.am $(noinst_HEADERS) \ + $(am__DIST_COMMON) +mkinstalldirs = $(install_sh) -d +CONFIG_HEADER = $(top_builddir)/config.h +CONFIG_CLEAN_FILES = +CONFIG_CLEAN_VPATH_FILES = +am__EXEEXT_1 = test-dict-sql$(EXEEXT) +PROGRAMS = $(noinst_PROGRAMS) +am__vpath_adj_setup = srcdirstrip=`echo "$(srcdir)" | sed 's|.|.|g'`; +am__vpath_adj = case $$p in \ + $(srcdir)/*) f=`echo "$$p" | sed "s|^$$srcdirstrip/||"`;; \ + *) f=$$p;; \ + esac; +am__strip_dir = f=`echo $$p | sed -e 's|^.*/||'`; +am__install_max = 40 +am__nobase_strip_setup = \ + srcdirstrip=`echo "$(srcdir)" | sed 's/[].[^$$\\*|]/\\\\&/g'` +am__nobase_strip = \ + for p in $$list; do echo "$$p"; done | sed -e "s|$$srcdirstrip/||" +am__nobase_list = $(am__nobase_strip_setup); \ + for p in $$list; do echo "$$p $$p"; done | \ + sed "s| $$srcdirstrip/| |;"' / .*\//!s/ .*/ ./; s,\( .*\)/[^/]*$$,\1,' | \ + $(AWK) 'BEGIN { files["."] = "" } { files[$$2] = files[$$2] " " $$1; \ + if (++n[$$2] == $(am__install_max)) \ + { print $$2, files[$$2]; n[$$2] = 0; files[$$2] = "" } } \ + END { for (dir in files) print dir, files[dir] }' +am__base_list = \ + sed '$$!N;$$!N;$$!N;$$!N;$$!N;$$!N;$$!N;s/\n/ /g' | \ + sed '$$!N;$$!N;$$!N;$$!N;s/\n/ /g' +am__uninstall_files_from_dir = { \ + test -z "$$files" \ + || { test ! -d "$$dir" && test ! -f "$$dir" && test ! -r "$$dir"; } \ + || { echo " ( cd '$$dir' && rm -f" $$files ")"; \ + $(am__cd) "$$dir" && rm -f $$files; }; \ + } +am__installdirs = "$(DESTDIR)$(module_dictdir)" +LTLIBRARIES = $(module_dict_LTLIBRARIES) $(noinst_LTLIBRARIES) +am__DEPENDENCIES_1 = +@HAVE_LDAP_TRUE@@LDAP_PLUGIN_FALSE@am__DEPENDENCIES_2 = \ +@HAVE_LDAP_TRUE@@LDAP_PLUGIN_FALSE@ $(am__DEPENDENCIES_1) +libdict_backend_la_DEPENDENCIES = $(am__DEPENDENCIES_2) +am__objects_1 = dict-ldap.lo dict-ldap-settings.lo +am_libdict_backend_la_OBJECTS = dict-cdb.lo dict-sql.lo \ + dict-sql-settings.lo $(am__objects_1) +nodist_libdict_backend_la_OBJECTS = dict-drivers-register.lo +libdict_backend_la_OBJECTS = $(am_libdict_backend_la_OBJECTS) \ + $(nodist_libdict_backend_la_OBJECTS) +AM_V_lt = $(am__v_lt_@AM_V@) +am__v_lt_ = $(am__v_lt_@AM_DEFAULT_V@) +am__v_lt_0 = --silent +am__v_lt_1 = +am__libdict_ldap_la_SOURCES_DIST = dict-ldap.c dict-ldap-settings.c +am__objects_2 = libdict_ldap_la-dict-ldap.lo \ + libdict_ldap_la-dict-ldap-settings.lo +@LDAP_PLUGIN_TRUE@am_libdict_ldap_la_OBJECTS = $(am__objects_2) +libdict_ldap_la_OBJECTS = $(am_libdict_ldap_la_OBJECTS) +libdict_ldap_la_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CC \ + $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CCLD) \ + $(AM_CFLAGS) $(CFLAGS) $(libdict_ldap_la_LDFLAGS) $(LDFLAGS) \ + -o $@ +@LDAP_PLUGIN_TRUE@am_libdict_ldap_la_rpath = -rpath $(module_dictdir) +am_test_dict_sql_OBJECTS = test_dict_sql-test-dict-sql.$(OBJEXT) +test_dict_sql_OBJECTS = $(am_test_dict_sql_OBJECTS) +test_dict_sql_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) \ + $(LIBTOOLFLAGS) --mode=link $(CCLD) $(test_dict_sql_CFLAGS) \ + $(CFLAGS) $(AM_LDFLAGS) $(LDFLAGS) -o $@ +AM_V_P = $(am__v_P_@AM_V@) +am__v_P_ = $(am__v_P_@AM_DEFAULT_V@) +am__v_P_0 = false +am__v_P_1 = : +AM_V_GEN = $(am__v_GEN_@AM_V@) +am__v_GEN_ = $(am__v_GEN_@AM_DEFAULT_V@) +am__v_GEN_0 = @echo " GEN " $@; +am__v_GEN_1 = +AM_V_at = $(am__v_at_@AM_V@) +am__v_at_ = $(am__v_at_@AM_DEFAULT_V@) +am__v_at_0 = @ +am__v_at_1 = +DEFAULT_INCLUDES = -I.@am__isrc@ -I$(top_builddir) +depcomp = $(SHELL) $(top_srcdir)/depcomp +am__maybe_remake_depfiles = depfiles +am__depfiles_remade = ./$(DEPDIR)/dict-cdb.Plo \ + ./$(DEPDIR)/dict-drivers-register.Plo \ + ./$(DEPDIR)/dict-ldap-settings.Plo ./$(DEPDIR)/dict-ldap.Plo \ + ./$(DEPDIR)/dict-sql-settings.Plo ./$(DEPDIR)/dict-sql.Plo \ + ./$(DEPDIR)/libdict_ldap_la-dict-ldap-settings.Plo \ + ./$(DEPDIR)/libdict_ldap_la-dict-ldap.Plo \ + ./$(DEPDIR)/test_dict_sql-test-dict-sql.Po +am__mv = mv -f +COMPILE = $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) \ + $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) +LTCOMPILE = $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) \ + $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) \ + $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) \ + $(AM_CFLAGS) $(CFLAGS) +AM_V_CC = $(am__v_CC_@AM_V@) +am__v_CC_ = $(am__v_CC_@AM_DEFAULT_V@) +am__v_CC_0 = @echo " CC " $@; +am__v_CC_1 = +CCLD = $(CC) +LINK = $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) \ + $(LIBTOOLFLAGS) --mode=link $(CCLD) $(AM_CFLAGS) $(CFLAGS) \ + $(AM_LDFLAGS) $(LDFLAGS) -o $@ +AM_V_CCLD = $(am__v_CCLD_@AM_V@) +am__v_CCLD_ = $(am__v_CCLD_@AM_DEFAULT_V@) +am__v_CCLD_0 = @echo " CCLD " $@; +am__v_CCLD_1 = +SOURCES = $(libdict_backend_la_SOURCES) \ + $(nodist_libdict_backend_la_SOURCES) \ + $(libdict_ldap_la_SOURCES) $(test_dict_sql_SOURCES) +DIST_SOURCES = $(libdict_backend_la_SOURCES) \ + $(am__libdict_ldap_la_SOURCES_DIST) $(test_dict_sql_SOURCES) +am__can_run_installinfo = \ + case $$AM_UPDATE_INFO_DIR in \ + n|no|NO) false;; \ + *) (install-info --version) >/dev/null 2>&1;; \ + esac +HEADERS = $(noinst_HEADERS) +am__tagged_files = $(HEADERS) $(SOURCES) $(TAGS_FILES) $(LISP) +# Read a list of newline-separated strings from the standard input, +# and print each of them once, without duplicates. Input order is +# *not* preserved. +am__uniquify_input = $(AWK) '\ + BEGIN { nonempty = 0; } \ + { items[$$0] = 1; nonempty = 1; } \ + END { if (nonempty) { for (i in items) print i; }; } \ +' +# Make sure the list of sources is unique. This is necessary because, +# e.g., the same source file might be shared among _SOURCES variables +# for different programs/libraries. +am__define_uniq_tagged_files = \ + list='$(am__tagged_files)'; \ + unique=`for i in $$list; do \ + if test -f "$$i"; then echo $$i; else echo $(srcdir)/$$i; fi; \ + done | $(am__uniquify_input)` +ETAGS = etags +CTAGS = ctags +am__DIST_COMMON = $(srcdir)/Makefile.in $(top_srcdir)/depcomp +DISTFILES = $(DIST_COMMON) $(DIST_SOURCES) $(TEXINFOS) $(EXTRA_DIST) +ACLOCAL = @ACLOCAL@ +ACLOCAL_AMFLAGS = @ACLOCAL_AMFLAGS@ +AMTAR = @AMTAR@ +AM_DEFAULT_VERBOSITY = @AM_DEFAULT_VERBOSITY@ +APPARMOR_LIBS = @APPARMOR_LIBS@ +AR = @AR@ +AUTH_CFLAGS = @AUTH_CFLAGS@ +AUTH_LIBS = @AUTH_LIBS@ +AUTOCONF = @AUTOCONF@ +AUTOHEADER = @AUTOHEADER@ +AUTOMAKE = @AUTOMAKE@ +AWK = @AWK@ +BINARY_CFLAGS = @BINARY_CFLAGS@ +BINARY_LDFLAGS = @BINARY_LDFLAGS@ +BISON = @BISON@ +CASSANDRA_CFLAGS = @CASSANDRA_CFLAGS@ +CASSANDRA_LIBS = @CASSANDRA_LIBS@ +CC = @CC@ +CCDEPMODE = @CCDEPMODE@ +CDB_LIBS = @CDB_LIBS@ +CFLAGS = @CFLAGS@ +CLUCENE_CFLAGS = @CLUCENE_CFLAGS@ +CLUCENE_LIBS = @CLUCENE_LIBS@ +COMPRESS_LIBS = @COMPRESS_LIBS@ +CPP = @CPP@ +CPPFLAGS = @CPPFLAGS@ +CRYPT_LIBS = @CRYPT_LIBS@ +CXX = @CXX@ +CXXCPP = @CXXCPP@ +CXXDEPMODE = @CXXDEPMODE@ +CXXFLAGS = @CXXFLAGS@ +CYGPATH_W = @CYGPATH_W@ +DEFS = @DEFS@ +DEPDIR = @DEPDIR@ +DICT_LIBS = @DICT_LIBS@ +DLLIB = @DLLIB@ +DLLTOOL = @DLLTOOL@ +DSYMUTIL = @DSYMUTIL@ +DUMPBIN = @DUMPBIN@ +ECHO_C = @ECHO_C@ +ECHO_N = @ECHO_N@ +ECHO_T = @ECHO_T@ +EGREP = @EGREP@ +EXEEXT = @EXEEXT@ +FGREP = @FGREP@ +FLEX = @FLEX@ +FUZZER_CPPFLAGS = @FUZZER_CPPFLAGS@ +FUZZER_LDFLAGS = @FUZZER_LDFLAGS@ +GREP = @GREP@ +INSTALL = @INSTALL@ +INSTALL_DATA = @INSTALL_DATA@ +INSTALL_PROGRAM = @INSTALL_PROGRAM@ +INSTALL_SCRIPT = @INSTALL_SCRIPT@ +INSTALL_STRIP_PROGRAM = @INSTALL_STRIP_PROGRAM@ +KRB5CONFIG = @KRB5CONFIG@ +KRB5_CFLAGS = @KRB5_CFLAGS@ +KRB5_LIBS = @KRB5_LIBS@ +LD = @LD@ +LDAP_LIBS = @LDAP_LIBS@ +LDFLAGS = @LDFLAGS@ +LD_NO_WHOLE_ARCHIVE = @LD_NO_WHOLE_ARCHIVE@ +LD_WHOLE_ARCHIVE = @LD_WHOLE_ARCHIVE@ +LIBCAP = @LIBCAP@ +LIBDOVECOT = @LIBDOVECOT@ +LIBDOVECOT_COMPRESS = @LIBDOVECOT_COMPRESS@ +LIBDOVECOT_DEPS = @LIBDOVECOT_DEPS@ +LIBDOVECOT_DSYNC = @LIBDOVECOT_DSYNC@ +LIBDOVECOT_LA_LIBS = @LIBDOVECOT_LA_LIBS@ +LIBDOVECOT_LDA = @LIBDOVECOT_LDA@ +LIBDOVECOT_LDAP = @LIBDOVECOT_LDAP@ +LIBDOVECOT_LIBFTS = @LIBDOVECOT_LIBFTS@ +LIBDOVECOT_LIBFTS_DEPS = @LIBDOVECOT_LIBFTS_DEPS@ +LIBDOVECOT_LOGIN = @LIBDOVECOT_LOGIN@ +LIBDOVECOT_LUA = @LIBDOVECOT_LUA@ +LIBDOVECOT_LUA_DEPS = @LIBDOVECOT_LUA_DEPS@ +LIBDOVECOT_SQL = @LIBDOVECOT_SQL@ +LIBDOVECOT_STORAGE = @LIBDOVECOT_STORAGE@ +LIBDOVECOT_STORAGE_DEPS = @LIBDOVECOT_STORAGE_DEPS@ +LIBEXTTEXTCAT_CFLAGS = @LIBEXTTEXTCAT_CFLAGS@ +LIBEXTTEXTCAT_LIBS = @LIBEXTTEXTCAT_LIBS@ +LIBICONV = @LIBICONV@ +LIBICU_CFLAGS = @LIBICU_CFLAGS@ +LIBICU_LIBS = @LIBICU_LIBS@ +LIBOBJS = @LIBOBJS@ +LIBS = @LIBS@ +LIBSODIUM_CFLAGS = @LIBSODIUM_CFLAGS@ +LIBSODIUM_LIBS = @LIBSODIUM_LIBS@ +LIBTIRPC_CFLAGS = @LIBTIRPC_CFLAGS@ +LIBTIRPC_LIBS = @LIBTIRPC_LIBS@ +LIBTOOL = @LIBTOOL@ +LIBUNWIND_CFLAGS = @LIBUNWIND_CFLAGS@ +LIBUNWIND_LIBS = @LIBUNWIND_LIBS@ +LIBWRAP_LIBS = @LIBWRAP_LIBS@ +LINKED_STORAGE_LDADD = @LINKED_STORAGE_LDADD@ +LIPO = @LIPO@ +LN_S = @LN_S@ +LTLIBICONV = @LTLIBICONV@ +LTLIBOBJS = @LTLIBOBJS@ +LT_SYS_LIBRARY_PATH = @LT_SYS_LIBRARY_PATH@ +LUA_CFLAGS = @LUA_CFLAGS@ +LUA_LIBS = @LUA_LIBS@ +MAINT = @MAINT@ +MAKEINFO = @MAKEINFO@ +MANIFEST_TOOL = @MANIFEST_TOOL@ +MKDIR_P = @MKDIR_P@ +MODULE_LIBS = @MODULE_LIBS@ +MODULE_SUFFIX = @MODULE_SUFFIX@ +MYSQL_CFLAGS = @MYSQL_CFLAGS@ +MYSQL_CONFIG = @MYSQL_CONFIG@ +MYSQL_LIBS = @MYSQL_LIBS@ +NM = @NM@ +NMEDIT = @NMEDIT@ +NOPLUGIN_LDFLAGS = +OBJDUMP = @OBJDUMP@ +OBJEXT = @OBJEXT@ +OTOOL = @OTOOL@ +OTOOL64 = @OTOOL64@ +PACKAGE = @PACKAGE@ +PACKAGE_BUGREPORT = @PACKAGE_BUGREPORT@ +PACKAGE_NAME = @PACKAGE_NAME@ +PACKAGE_STRING = @PACKAGE_STRING@ +PACKAGE_TARNAME = @PACKAGE_TARNAME@ +PACKAGE_URL = @PACKAGE_URL@ +PACKAGE_VERSION = @PACKAGE_VERSION@ +PANDOC = @PANDOC@ +PATH_SEPARATOR = @PATH_SEPARATOR@ +PGSQL_CFLAGS = @PGSQL_CFLAGS@ +PGSQL_LIBS = @PGSQL_LIBS@ +PG_CONFIG = @PG_CONFIG@ +PIE_CFLAGS = @PIE_CFLAGS@ +PIE_LDFLAGS = @PIE_LDFLAGS@ +PKG_CONFIG = @PKG_CONFIG@ +PKG_CONFIG_LIBDIR = @PKG_CONFIG_LIBDIR@ +PKG_CONFIG_PATH = @PKG_CONFIG_PATH@ +QUOTA_LIBS = @QUOTA_LIBS@ +RANLIB = @RANLIB@ +RELRO_LDFLAGS = @RELRO_LDFLAGS@ +RPCGEN = @RPCGEN@ +RUN_TEST = @RUN_TEST@ +SED = @SED@ +SETTING_FILES = @SETTING_FILES@ +SET_MAKE = @SET_MAKE@ +SHELL = @SHELL@ +SQLITE_CFLAGS = @SQLITE_CFLAGS@ +SQLITE_LIBS = @SQLITE_LIBS@ +SQL_CFLAGS = @SQL_CFLAGS@ +SQL_LIBS = @SQL_LIBS@ +SSL_CFLAGS = @SSL_CFLAGS@ +SSL_LIBS = @SSL_LIBS@ +STRIP = @STRIP@ +SYSTEMD_CFLAGS = @SYSTEMD_CFLAGS@ +SYSTEMD_LIBS = @SYSTEMD_LIBS@ +VALGRIND = @VALGRIND@ +VERSION = @VERSION@ +ZSTD_CFLAGS = @ZSTD_CFLAGS@ +ZSTD_LIBS = @ZSTD_LIBS@ +abs_builddir = @abs_builddir@ +abs_srcdir = @abs_srcdir@ +abs_top_builddir = @abs_top_builddir@ +abs_top_srcdir = @abs_top_srcdir@ +ac_ct_AR = @ac_ct_AR@ +ac_ct_CC = @ac_ct_CC@ +ac_ct_CXX = @ac_ct_CXX@ +ac_ct_DUMPBIN = @ac_ct_DUMPBIN@ +am__include = @am__include@ +am__leading_dot = @am__leading_dot@ +am__quote = @am__quote@ +am__tar = @am__tar@ +am__untar = @am__untar@ +bindir = @bindir@ +build = @build@ +build_alias = @build_alias@ +build_cpu = @build_cpu@ +build_os = @build_os@ +build_vendor = @build_vendor@ +builddir = @builddir@ +datadir = @datadir@ +datarootdir = @datarootdir@ +dict_drivers = @dict_drivers@ $(am__append_2) +docdir = @docdir@ +dvidir = @dvidir@ +exec_prefix = @exec_prefix@ +host = @host@ +host_alias = @host_alias@ +host_cpu = @host_cpu@ +host_os = @host_os@ +host_vendor = @host_vendor@ +htmldir = @htmldir@ +includedir = @includedir@ +infodir = @infodir@ +install_sh = @install_sh@ +libdir = @libdir@ +libexecdir = @libexecdir@ +localedir = @localedir@ +localstatedir = @localstatedir@ +mandir = @mandir@ +mkdir_p = @mkdir_p@ +moduledir = @moduledir@ +oldincludedir = @oldincludedir@ +pdfdir = @pdfdir@ +prefix = @prefix@ +program_transform_name = @program_transform_name@ +psdir = @psdir@ +rundir = @rundir@ +runstatedir = @runstatedir@ +sbindir = @sbindir@ +sharedstatedir = @sharedstatedir@ +sql_drivers = @sql_drivers@ +srcdir = @srcdir@ +ssldir = @ssldir@ +statedir = @statedir@ +sysconfdir = @sysconfdir@ +systemdservicetype = @systemdservicetype@ +systemdsystemunitdir = @systemdsystemunitdir@ +target_alias = @target_alias@ +top_build_prefix = @top_build_prefix@ +top_builddir = @top_builddir@ +top_srcdir = @top_srcdir@ +noinst_LTLIBRARIES = libdict_backend.la +module_dictdir = $(moduledir)/dict +AM_CPPFLAGS = \ + -I$(top_srcdir)/src/lib \ + -I$(top_srcdir)/src/lib-test \ + -I$(top_srcdir)/src/lib-dict \ + -I$(top_srcdir)/src/lib-ldap \ + -I$(top_srcdir)/src/lib-sql \ + -I$(top_srcdir)/src/lib-settings \ + $(SQL_CFLAGS) + +ldap_sources = \ + dict-ldap.c \ + dict-ldap-settings.c + +libdict_backend_la_SOURCES = \ + dict-cdb.c \ + dict-sql.c \ + dict-sql-settings.c \ + $(ldap_sources) + +libdict_backend_la_LIBADD = $(am__append_1) +nodist_libdict_backend_la_SOURCES = \ + dict-drivers-register.c + +noinst_HEADERS = \ + dict-ldap-settings.h \ + dict-sql.h \ + dict-sql-private.h \ + dict-sql-settings.h + +@LDAP_PLUGIN_TRUE@LIBDICT_LDAP = libdict_ldap.la +@LDAP_PLUGIN_TRUE@libdict_ldap_la_DEPENDENCIES = $(LIBDOVECOT_LDAP) $(LIBDOVECOT_DEPS) +@LDAP_PLUGIN_TRUE@libdict_ldap_la_LDFLAGS = -module -avoid-version +@LDAP_PLUGIN_TRUE@libdict_ldap_la_LIBADD = $(LIBDOVECOT_LDAP) $(LIBDOVECOT) +@LDAP_PLUGIN_TRUE@libdict_ldap_la_CPPFLAGS = $(AM_CPPFLAGS) -DPLUGIN_BUILD +@LDAP_PLUGIN_TRUE@libdict_ldap_la_SOURCES = $(ldap_sources) +module_dict_LTLIBRARIES = \ + $(LIBDICT_LDAP) + +EXTRA_DIST = dict.conf +test_programs = \ + test-dict-sql + +test_dict_sql_CFLAGS = $(AM_CPPFLAGS) -DDICT_SRC_DIR=\"$(top_srcdir)/src/lib-dict-backend\" +test_dict_sql_SOURCES = \ + test-dict-sql.c + +test_dict_sql_LDADD = \ + $(noinst_LTLIBRARIES) \ + $(DICT_LIBS) \ + ../lib-sql/libdriver_test.la \ + ../lib-sql/libsql.la \ + ../lib-dovecot/libdovecot.la + +test_dict_sql_DEPENDENCIES = \ + $(noinst_LTLIBRARIES) \ + ../lib-sql/libdriver_test.la \ + ../lib-sql/libsql.la \ + ../lib-dovecot/libdovecot.la + +all: all-am + +.SUFFIXES: +.SUFFIXES: .c .lo .o .obj +$(srcdir)/Makefile.in: @MAINTAINER_MODE_TRUE@ $(srcdir)/Makefile.am $(am__configure_deps) + @for dep in $?; do \ + case '$(am__configure_deps)' in \ + *$$dep*) \ + ( cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh ) \ + && { if test -f $@; then exit 0; else break; fi; }; \ + exit 1;; \ + esac; \ + done; \ + echo ' cd $(top_srcdir) && $(AUTOMAKE) --foreign src/lib-dict-backend/Makefile'; \ + $(am__cd) $(top_srcdir) && \ + $(AUTOMAKE) --foreign src/lib-dict-backend/Makefile +Makefile: $(srcdir)/Makefile.in $(top_builddir)/config.status + @case '$?' in \ + *config.status*) \ + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh;; \ + *) \ + echo ' cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__maybe_remake_depfiles)'; \ + cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__maybe_remake_depfiles);; \ + esac; + +$(top_builddir)/config.status: $(top_srcdir)/configure $(CONFIG_STATUS_DEPENDENCIES) + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh + +$(top_srcdir)/configure: @MAINTAINER_MODE_TRUE@ $(am__configure_deps) + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh +$(ACLOCAL_M4): @MAINTAINER_MODE_TRUE@ $(am__aclocal_m4_deps) + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh +$(am__aclocal_m4_deps): + +clean-noinstPROGRAMS: + @list='$(noinst_PROGRAMS)'; test -n "$$list" || exit 0; \ + echo " rm -f" $$list; \ + rm -f $$list || exit $$?; \ + test -n "$(EXEEXT)" || exit 0; \ + list=`for p in $$list; do echo "$$p"; done | sed 's/$(EXEEXT)$$//'`; \ + echo " rm -f" $$list; \ + rm -f $$list + +install-module_dictLTLIBRARIES: $(module_dict_LTLIBRARIES) + @$(NORMAL_INSTALL) + @list='$(module_dict_LTLIBRARIES)'; test -n "$(module_dictdir)" || list=; \ + list2=; for p in $$list; do \ + if test -f $$p; then \ + list2="$$list2 $$p"; \ + else :; fi; \ + done; \ + test -z "$$list2" || { \ + echo " $(MKDIR_P) '$(DESTDIR)$(module_dictdir)'"; \ + $(MKDIR_P) "$(DESTDIR)$(module_dictdir)" || exit 1; \ + echo " $(LIBTOOL) $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=install $(INSTALL) $(INSTALL_STRIP_FLAG) $$list2 '$(DESTDIR)$(module_dictdir)'"; \ + $(LIBTOOL) $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=install $(INSTALL) $(INSTALL_STRIP_FLAG) $$list2 "$(DESTDIR)$(module_dictdir)"; \ + } + +uninstall-module_dictLTLIBRARIES: + @$(NORMAL_UNINSTALL) + @list='$(module_dict_LTLIBRARIES)'; test -n "$(module_dictdir)" || list=; \ + for p in $$list; do \ + $(am__strip_dir) \ + echo " $(LIBTOOL) $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=uninstall rm -f '$(DESTDIR)$(module_dictdir)/$$f'"; \ + $(LIBTOOL) $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=uninstall rm -f "$(DESTDIR)$(module_dictdir)/$$f"; \ + done + +clean-module_dictLTLIBRARIES: + -test -z "$(module_dict_LTLIBRARIES)" || rm -f $(module_dict_LTLIBRARIES) + @list='$(module_dict_LTLIBRARIES)'; \ + locs=`for p in $$list; do echo $$p; done | \ + sed 's|^[^/]*$$|.|; s|/[^/]*$$||; s|$$|/so_locations|' | \ + sort -u`; \ + test -z "$$locs" || { \ + echo rm -f $${locs}; \ + rm -f $${locs}; \ + } + +clean-noinstLTLIBRARIES: + -test -z "$(noinst_LTLIBRARIES)" || rm -f $(noinst_LTLIBRARIES) + @list='$(noinst_LTLIBRARIES)'; \ + locs=`for p in $$list; do echo $$p; done | \ + sed 's|^[^/]*$$|.|; s|/[^/]*$$||; s|$$|/so_locations|' | \ + sort -u`; \ + test -z "$$locs" || { \ + echo rm -f $${locs}; \ + rm -f $${locs}; \ + } + +libdict_backend.la: $(libdict_backend_la_OBJECTS) $(libdict_backend_la_DEPENDENCIES) $(EXTRA_libdict_backend_la_DEPENDENCIES) + $(AM_V_CCLD)$(LINK) $(libdict_backend_la_OBJECTS) $(libdict_backend_la_LIBADD) $(LIBS) + +libdict_ldap.la: $(libdict_ldap_la_OBJECTS) $(libdict_ldap_la_DEPENDENCIES) $(EXTRA_libdict_ldap_la_DEPENDENCIES) + $(AM_V_CCLD)$(libdict_ldap_la_LINK) $(am_libdict_ldap_la_rpath) $(libdict_ldap_la_OBJECTS) $(libdict_ldap_la_LIBADD) $(LIBS) + +test-dict-sql$(EXEEXT): $(test_dict_sql_OBJECTS) $(test_dict_sql_DEPENDENCIES) $(EXTRA_test_dict_sql_DEPENDENCIES) + @rm -f test-dict-sql$(EXEEXT) + $(AM_V_CCLD)$(test_dict_sql_LINK) $(test_dict_sql_OBJECTS) $(test_dict_sql_LDADD) $(LIBS) + +mostlyclean-compile: + -rm -f *.$(OBJEXT) + +distclean-compile: + -rm -f *.tab.c + +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/dict-cdb.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/dict-drivers-register.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/dict-ldap-settings.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/dict-ldap.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/dict-sql-settings.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/dict-sql.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libdict_ldap_la-dict-ldap-settings.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libdict_ldap_la-dict-ldap.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/test_dict_sql-test-dict-sql.Po@am__quote@ # am--include-marker + +$(am__depfiles_remade): + @$(MKDIR_P) $(@D) + @echo '# dummy' >$@-t && $(am__mv) $@-t $@ + +am--depfiles: $(am__depfiles_remade) + +.c.o: +@am__fastdepCC_TRUE@ $(AM_V_CC)$(COMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ $< +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='$<' object='$@' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(COMPILE) -c -o $@ $< + +.c.obj: +@am__fastdepCC_TRUE@ $(AM_V_CC)$(COMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ `$(CYGPATH_W) '$<'` +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='$<' object='$@' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(COMPILE) -c -o $@ `$(CYGPATH_W) '$<'` + +.c.lo: +@am__fastdepCC_TRUE@ $(AM_V_CC)$(LTCOMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ $< +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Plo +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='$<' object='$@' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(LTCOMPILE) -c -o $@ $< + +libdict_ldap_la-dict-ldap.lo: dict-ldap.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libdict_ldap_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT libdict_ldap_la-dict-ldap.lo -MD -MP -MF $(DEPDIR)/libdict_ldap_la-dict-ldap.Tpo -c -o libdict_ldap_la-dict-ldap.lo `test -f 'dict-ldap.c' || echo '$(srcdir)/'`dict-ldap.c +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/libdict_ldap_la-dict-ldap.Tpo $(DEPDIR)/libdict_ldap_la-dict-ldap.Plo +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='dict-ldap.c' object='libdict_ldap_la-dict-ldap.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libdict_ldap_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o libdict_ldap_la-dict-ldap.lo `test -f 'dict-ldap.c' || echo '$(srcdir)/'`dict-ldap.c + +libdict_ldap_la-dict-ldap-settings.lo: dict-ldap-settings.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libdict_ldap_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT libdict_ldap_la-dict-ldap-settings.lo -MD -MP -MF $(DEPDIR)/libdict_ldap_la-dict-ldap-settings.Tpo -c -o libdict_ldap_la-dict-ldap-settings.lo `test -f 'dict-ldap-settings.c' || echo '$(srcdir)/'`dict-ldap-settings.c +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/libdict_ldap_la-dict-ldap-settings.Tpo $(DEPDIR)/libdict_ldap_la-dict-ldap-settings.Plo +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='dict-ldap-settings.c' object='libdict_ldap_la-dict-ldap-settings.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libdict_ldap_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o libdict_ldap_la-dict-ldap-settings.lo `test -f 'dict-ldap-settings.c' || echo '$(srcdir)/'`dict-ldap-settings.c + +test_dict_sql-test-dict-sql.o: test-dict-sql.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_dict_sql_CFLAGS) $(CFLAGS) -MT test_dict_sql-test-dict-sql.o -MD -MP -MF $(DEPDIR)/test_dict_sql-test-dict-sql.Tpo -c -o test_dict_sql-test-dict-sql.o `test -f 'test-dict-sql.c' || echo '$(srcdir)/'`test-dict-sql.c +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/test_dict_sql-test-dict-sql.Tpo $(DEPDIR)/test_dict_sql-test-dict-sql.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='test-dict-sql.c' object='test_dict_sql-test-dict-sql.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_dict_sql_CFLAGS) $(CFLAGS) -c -o test_dict_sql-test-dict-sql.o `test -f 'test-dict-sql.c' || echo '$(srcdir)/'`test-dict-sql.c + +test_dict_sql-test-dict-sql.obj: test-dict-sql.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_dict_sql_CFLAGS) $(CFLAGS) -MT test_dict_sql-test-dict-sql.obj -MD -MP -MF $(DEPDIR)/test_dict_sql-test-dict-sql.Tpo -c -o test_dict_sql-test-dict-sql.obj `if test -f 'test-dict-sql.c'; then $(CYGPATH_W) 'test-dict-sql.c'; else $(CYGPATH_W) '$(srcdir)/test-dict-sql.c'; fi` +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/test_dict_sql-test-dict-sql.Tpo $(DEPDIR)/test_dict_sql-test-dict-sql.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='test-dict-sql.c' object='test_dict_sql-test-dict-sql.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_dict_sql_CFLAGS) $(CFLAGS) -c -o test_dict_sql-test-dict-sql.obj `if test -f 'test-dict-sql.c'; then $(CYGPATH_W) 'test-dict-sql.c'; else $(CYGPATH_W) '$(srcdir)/test-dict-sql.c'; fi` + +mostlyclean-libtool: + -rm -f *.lo + +clean-libtool: + -rm -rf .libs _libs + +ID: $(am__tagged_files) + $(am__define_uniq_tagged_files); mkid -fID $$unique +tags: tags-am +TAGS: tags + +tags-am: $(TAGS_DEPENDENCIES) $(am__tagged_files) + set x; \ + here=`pwd`; \ + $(am__define_uniq_tagged_files); \ + shift; \ + if test -z "$(ETAGS_ARGS)$$*$$unique"; then :; else \ + test -n "$$unique" || unique=$$empty_fix; \ + if test $$# -gt 0; then \ + $(ETAGS) $(ETAGSFLAGS) $(AM_ETAGSFLAGS) $(ETAGS_ARGS) \ + "$$@" $$unique; \ + else \ + $(ETAGS) $(ETAGSFLAGS) $(AM_ETAGSFLAGS) $(ETAGS_ARGS) \ + $$unique; \ + fi; \ + fi +ctags: ctags-am + +CTAGS: ctags +ctags-am: $(TAGS_DEPENDENCIES) $(am__tagged_files) + $(am__define_uniq_tagged_files); \ + test -z "$(CTAGS_ARGS)$$unique" \ + || $(CTAGS) $(CTAGSFLAGS) $(AM_CTAGSFLAGS) $(CTAGS_ARGS) \ + $$unique + +GTAGS: + here=`$(am__cd) $(top_builddir) && pwd` \ + && $(am__cd) $(top_srcdir) \ + && gtags -i $(GTAGS_ARGS) "$$here" +cscopelist: cscopelist-am + +cscopelist-am: $(am__tagged_files) + list='$(am__tagged_files)'; \ + case "$(srcdir)" in \ + [\\/]* | ?:[\\/]*) sdir="$(srcdir)" ;; \ + *) sdir=$(subdir)/$(srcdir) ;; \ + esac; \ + for i in $$list; do \ + if test -f "$$i"; then \ + echo "$(subdir)/$$i"; \ + else \ + echo "$$sdir/$$i"; \ + fi; \ + done >> $(top_builddir)/cscope.files + +distclean-tags: + -rm -f TAGS ID GTAGS GRTAGS GSYMS GPATH tags + +distdir: $(BUILT_SOURCES) + $(MAKE) $(AM_MAKEFLAGS) distdir-am + +distdir-am: $(DISTFILES) + @srcdirstrip=`echo "$(srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \ + topsrcdirstrip=`echo "$(top_srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \ + list='$(DISTFILES)'; \ + dist_files=`for file in $$list; do echo $$file; done | \ + sed -e "s|^$$srcdirstrip/||;t" \ + -e "s|^$$topsrcdirstrip/|$(top_builddir)/|;t"`; \ + case $$dist_files in \ + */*) $(MKDIR_P) `echo "$$dist_files" | \ + sed '/\//!d;s|^|$(distdir)/|;s,/[^/]*$$,,' | \ + sort -u` ;; \ + esac; \ + for file in $$dist_files; do \ + if test -f $$file || test -d $$file; then d=.; else d=$(srcdir); fi; \ + if test -d $$d/$$file; then \ + dir=`echo "/$$file" | sed -e 's,/[^/]*$$,,'`; \ + if test -d "$(distdir)/$$file"; then \ + find "$(distdir)/$$file" -type d ! -perm -700 -exec chmod u+rwx {} \;; \ + fi; \ + if test -d $(srcdir)/$$file && test $$d != $(srcdir); then \ + cp -fpR $(srcdir)/$$file "$(distdir)$$dir" || exit 1; \ + find "$(distdir)/$$file" -type d ! -perm -700 -exec chmod u+rwx {} \;; \ + fi; \ + cp -fpR $$d/$$file "$(distdir)$$dir" || exit 1; \ + else \ + test -f "$(distdir)/$$file" \ + || cp -p $$d/$$file "$(distdir)/$$file" \ + || exit 1; \ + fi; \ + done +check-am: all-am + $(MAKE) $(AM_MAKEFLAGS) check-local +check: check-am +all-am: Makefile $(PROGRAMS) $(LTLIBRARIES) $(HEADERS) +installdirs: + for dir in "$(DESTDIR)$(module_dictdir)"; do \ + test -z "$$dir" || $(MKDIR_P) "$$dir"; \ + done +install: install-am +install-exec: install-exec-am +install-data: install-data-am +uninstall: uninstall-am + +install-am: all-am + @$(MAKE) $(AM_MAKEFLAGS) install-exec-am install-data-am + +installcheck: installcheck-am +install-strip: + if test -z '$(STRIP)'; then \ + $(MAKE) $(AM_MAKEFLAGS) INSTALL_PROGRAM="$(INSTALL_STRIP_PROGRAM)" \ + install_sh_PROGRAM="$(INSTALL_STRIP_PROGRAM)" INSTALL_STRIP_FLAG=-s \ + install; \ + else \ + $(MAKE) $(AM_MAKEFLAGS) INSTALL_PROGRAM="$(INSTALL_STRIP_PROGRAM)" \ + install_sh_PROGRAM="$(INSTALL_STRIP_PROGRAM)" INSTALL_STRIP_FLAG=-s \ + "INSTALL_PROGRAM_ENV=STRIPPROG='$(STRIP)'" install; \ + fi +mostlyclean-generic: + +clean-generic: + +maintainer-clean-generic: + @echo "This command is intended for maintainers to use" + @echo "it deletes files that may require special tools to rebuild." +clean: clean-am + +clean-am: clean-generic clean-libtool clean-module_dictLTLIBRARIES \ + clean-noinstLTLIBRARIES clean-noinstPROGRAMS mostlyclean-am + +distclean: distclean-am + -rm -f ./$(DEPDIR)/dict-cdb.Plo + -rm -f ./$(DEPDIR)/dict-drivers-register.Plo + -rm -f ./$(DEPDIR)/dict-ldap-settings.Plo + -rm -f ./$(DEPDIR)/dict-ldap.Plo + -rm -f ./$(DEPDIR)/dict-sql-settings.Plo + -rm -f ./$(DEPDIR)/dict-sql.Plo + -rm -f ./$(DEPDIR)/libdict_ldap_la-dict-ldap-settings.Plo + -rm -f ./$(DEPDIR)/libdict_ldap_la-dict-ldap.Plo + -rm -f ./$(DEPDIR)/test_dict_sql-test-dict-sql.Po + -rm -f Makefile +distclean-am: clean-am distclean-compile distclean-generic \ + distclean-tags + +dvi: dvi-am + +dvi-am: + +html: html-am + +html-am: + +info: info-am + +info-am: + +install-data-am: install-module_dictLTLIBRARIES + +install-dvi: install-dvi-am + +install-dvi-am: + +install-exec-am: + +install-html: install-html-am + +install-html-am: + +install-info: install-info-am + +install-info-am: + +install-man: + +install-pdf: install-pdf-am + +install-pdf-am: + +install-ps: install-ps-am + +install-ps-am: + +installcheck-am: + +maintainer-clean: maintainer-clean-am + -rm -f ./$(DEPDIR)/dict-cdb.Plo + -rm -f ./$(DEPDIR)/dict-drivers-register.Plo + -rm -f ./$(DEPDIR)/dict-ldap-settings.Plo + -rm -f ./$(DEPDIR)/dict-ldap.Plo + -rm -f ./$(DEPDIR)/dict-sql-settings.Plo + -rm -f ./$(DEPDIR)/dict-sql.Plo + -rm -f ./$(DEPDIR)/libdict_ldap_la-dict-ldap-settings.Plo + -rm -f ./$(DEPDIR)/libdict_ldap_la-dict-ldap.Plo + -rm -f ./$(DEPDIR)/test_dict_sql-test-dict-sql.Po + -rm -f Makefile +maintainer-clean-am: distclean-am maintainer-clean-generic + +mostlyclean: mostlyclean-am + +mostlyclean-am: mostlyclean-compile mostlyclean-generic \ + mostlyclean-libtool + +pdf: pdf-am + +pdf-am: + +ps: ps-am + +ps-am: + +uninstall-am: uninstall-module_dictLTLIBRARIES + +.MAKE: check-am install-am install-strip + +.PHONY: CTAGS GTAGS TAGS all all-am am--depfiles check check-am \ + check-local clean clean-generic clean-libtool \ + clean-module_dictLTLIBRARIES clean-noinstLTLIBRARIES \ + clean-noinstPROGRAMS cscopelist-am ctags ctags-am distclean \ + distclean-compile distclean-generic distclean-libtool \ + distclean-tags distdir dvi dvi-am html html-am info info-am \ + install install-am install-data install-data-am install-dvi \ + install-dvi-am install-exec install-exec-am install-html \ + install-html-am install-info install-info-am install-man \ + install-module_dictLTLIBRARIES install-pdf install-pdf-am \ + install-ps install-ps-am install-strip installcheck \ + installcheck-am installdirs maintainer-clean \ + maintainer-clean-generic mostlyclean mostlyclean-compile \ + mostlyclean-generic mostlyclean-libtool pdf pdf-am ps ps-am \ + tags tags-am uninstall uninstall-am \ + uninstall-module_dictLTLIBRARIES + +.PRECIOUS: Makefile + + +dict-drivers-register.c: Makefile $(top_builddir)/config.h + rm -f $@ + echo '/* this file automatically generated by Makefile */' >$@ + echo '#include "lib.h"' >>$@ + echo '#include "dict.h"' >>$@ + echo '#include "ldap-client.h"' >>$@ + echo '#include "dict-sql.h"' >>$@ + for i in $(dict_drivers) null; do \ + if [ "$${i}" != "null" ]; then \ + echo "extern struct dict dict_driver_$${i};" >>$@ ; \ + fi; \ + done + echo 'void dict_drivers_register_all(void) {' >>$@ + echo 'dict_drivers_register_builtin();' >>$@ + echo 'dict_sql_register();' >>$@ + for i in $(dict_drivers) null; do \ + if [ "$${i}" != "null" ]; then \ + echo "dict_driver_register(&dict_driver_$${i});" >>$@ ; \ + fi; \ + done + echo '}' >>$@ + echo 'void dict_drivers_unregister_all(void) {' >>$@ + echo '#ifdef BUILTIN_LDAP' >>$@ + echo 'ldap_clients_cleanup();' >>$@ + echo '#endif' >>$@ + echo 'dict_drivers_unregister_builtin();' >>$@ + echo 'dict_sql_unregister();' >>$@ + for i in $(dict_drivers) null; do \ + if [ "$${i}" != "null" ]; then \ + echo "dict_driver_unregister(&dict_driver_$${i});" >>$@ ; \ + fi; \ + done + echo '}' >>$@ + +distclean-generic: + rm -f Makefile dict-drivers-register.c + +check-local: + for bin in $(test_programs) $(check_PROGRAMS); do \ + if ! $(RUN_TEST) ./$$bin; then exit 1; fi; \ + done + +# Tell versions [3.59,3.63) of GNU make to not export all variables. +# Otherwise a system limit (for SysV at least) may be exceeded. +.NOEXPORT: diff --git a/src/lib-dict-backend/dict-cdb.c b/src/lib-dict-backend/dict-cdb.c new file mode 100644 index 0000000..c6fad30 --- /dev/null +++ b/src/lib-dict-backend/dict-cdb.c @@ -0,0 +1,266 @@ +/* Copyright (c) 2013-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "buffer.h" + +#ifdef BUILD_CDB +#include "dict-private.h" + +#include <string.h> +#include <cdb.h> +#include <unistd.h> +#include <fcntl.h> + +#define CDB_WITH_NULL 1 +#define CDB_WITHOUT_NULL 2 + +struct cdb_dict { + struct dict dict; + struct cdb cdb; + char *path; + int fd, flag; +}; + +struct cdb_dict_iterate_context { + struct dict_iterate_context ctx; + + enum dict_iterate_flags flags; + buffer_t *buffer; + const char *values[2]; + char *path; + unsigned cptr; + char *error; +}; + +static void cdb_dict_deinit(struct dict *_dict); + +static int +cdb_dict_init(struct dict *driver, const char *uri, + const struct dict_settings *set ATTR_UNUSED, + struct dict **dict_r, const char **error_r) +{ + struct cdb_dict *dict; + + dict = i_new(struct cdb_dict, 1); + dict->dict = *driver; + dict->path = i_strdup(uri); + dict->flag = CDB_WITH_NULL | CDB_WITHOUT_NULL; + + /* initialize cdb to 0 (unallocated) */ + i_zero(&dict->cdb); + + dict->fd = open(dict->path, O_RDONLY); + if (dict->fd == -1) { + *error_r = t_strdup_printf("open(%s) failed: %m", dict->path); + cdb_dict_deinit(&dict->dict); + return -1; + } + +#ifdef TINYCDB_VERSION + if (cdb_init(&dict->cdb, dict->fd) < 0) { + *error_r = t_strdup_printf("cdb_init(%s) failed: %m", dict->path); + cdb_dict_deinit(&dict->dict); + return -1; + } +#else + cdb_init(&dict->cdb, dict->fd); +#endif + + *dict_r = &dict->dict; + return 0; +} + +static void cdb_dict_deinit(struct dict *_dict) +{ + struct cdb_dict *dict = (struct cdb_dict *)_dict; + + /* we can safely deinit unallocated cdb */ + cdb_free(&dict->cdb); + + i_close_fd_path(&dict->fd, dict->path); + + i_free(dict->path); + i_free(dict); +} + +static int +cdb_dict_lookup(struct dict *_dict, + const struct dict_op_settings *set ATTR_UNUSED, + pool_t pool, + const char *key, const char **value_r, + const char **error_r) +{ + struct cdb_dict *dict = (struct cdb_dict *)_dict; + unsigned datalen; + int ret = 0; + char *data; + + /* keys and values may be null terminated... */ + if ((dict->flag & CDB_WITH_NULL) != 0) { + ret = cdb_find(&dict->cdb, key, (unsigned)strlen(key)+1); + if (ret > 0) + dict->flag &= ENUM_NEGATE(CDB_WITHOUT_NULL); + } + + /* ...or not */ + if (ret == 0 && (dict->flag & CDB_WITHOUT_NULL) != 0) { + ret = cdb_find(&dict->cdb, key, (unsigned)strlen(key)); + if (ret > 0) + dict->flag &= ENUM_NEGATE(CDB_WITH_NULL); + } + + if (ret <= 0) { + *value_r = NULL; + /* something bad with db */ + if (ret < 0) { + *error_r = t_strdup_printf("cdb_find(%s) failed: %m", dict->path); + return -1; + } + /* found nothing */ + return 0; + } + + datalen = cdb_datalen(&dict->cdb); + data = p_malloc(pool, datalen + 1); + if (cdb_read(&dict->cdb, data, datalen, cdb_datapos(&dict->cdb)) < 0) { + *error_r = t_strdup_printf("cdb_read(%s) failed: %m", dict->path); + return -1; + } + *value_r = data; + return 1; +} + +static struct dict_iterate_context * +cdb_dict_iterate_init(struct dict *_dict, + const struct dict_op_settings *set ATTR_UNUSED, + const char *path, enum dict_iterate_flags flags) +{ + struct cdb_dict_iterate_context *ctx = + i_new(struct cdb_dict_iterate_context, 1); + struct cdb_dict *dict = (struct cdb_dict *)_dict; + + ctx->ctx.dict = &dict->dict; + ctx->path = i_strdup(path); + ctx->flags = flags; + ctx->buffer = buffer_create_dynamic(default_pool, 256); + + cdb_seqinit(&ctx->cptr, &dict->cdb); + + return &ctx->ctx; +} + +static bool +cdb_dict_next(struct cdb_dict_iterate_context *ctx, const char **key_r) +{ + struct cdb_dict *dict = (struct cdb_dict *)ctx->ctx.dict; + char *data; + unsigned datalen; + int ret; + + if ((ret = cdb_seqnext(&ctx->cptr, &dict->cdb)) < 1) { + if (ret < 0) + ctx->error = i_strdup_printf("cdb_seqnext(%s) failed: %m", + dict->path); + return FALSE; + } + + buffer_set_used_size(ctx->buffer, 0); + + datalen = cdb_keylen(&dict->cdb); + data = buffer_append_space_unsafe(ctx->buffer, datalen + 1); + + if (cdb_read(&dict->cdb, data, datalen, cdb_keypos(&dict->cdb)) < 0) { + ctx->error = i_strdup_printf("cdb_read(%s) failed: %m", + dict->path); + return FALSE; + } + + data[datalen] = '\0'; + *key_r = data; + + return TRUE; +} + +static bool cdb_dict_iterate(struct dict_iterate_context *_ctx, + const char **key_r, const char *const **values_r) +{ + struct cdb_dict_iterate_context *ctx = + (struct cdb_dict_iterate_context *)_ctx; + struct cdb_dict *dict = (struct cdb_dict *)_ctx->dict; + const char *key; + bool match = FALSE; + char *data; + unsigned datalen; + + if (ctx->error != NULL) + return FALSE; + + while(!match && cdb_dict_next(ctx, &key)) { + if (((ctx->flags & DICT_ITERATE_FLAG_EXACT_KEY) != 0 && + strcmp(key, ctx->path) == 0) || + ((ctx->flags & DICT_ITERATE_FLAG_RECURSE) != 0 && + str_begins(key, ctx->path)) || + ((ctx->flags & DICT_ITERATE_FLAG_RECURSE) == 0 && + str_begins(key, ctx->path) && + strchr(key + strlen(ctx->path), '/') == NULL)) { + match = TRUE; + break; + } + } + + if (!match) + return FALSE; + + *key_r = key; + + if ((ctx->flags & DICT_ITERATE_FLAG_NO_VALUE) != 0) + return TRUE; + + datalen = cdb_datalen(&dict->cdb); + data = buffer_append_space_unsafe(ctx->buffer, datalen + 1); + + if (cdb_read(&dict->cdb, data, datalen, cdb_datapos(&dict->cdb)) < 0) { + ctx->error = i_strdup_printf("cdb_read(%s) failed: %m", + dict->path); + return FALSE; + } + + data[datalen] = '\0'; + ctx->values[0] = data; + *values_r = ctx->values; + + return TRUE; +} + +static int cdb_dict_iterate_deinit(struct dict_iterate_context *_ctx, + const char **error_r) +{ + int ret = 0; + struct cdb_dict_iterate_context *ctx = + (struct cdb_dict_iterate_context *)_ctx; + if (ctx->error != NULL) { + *error_r = t_strdup(ctx->error); + ret = -1; + } + + buffer_free(&ctx->buffer); + i_free(ctx->error); + i_free(ctx->path); + i_free(ctx); + + return ret; +} + + +struct dict dict_driver_cdb = { + .name = "cdb", + { + .init = cdb_dict_init, + .deinit = cdb_dict_deinit, + .lookup = cdb_dict_lookup, + .iterate_init = cdb_dict_iterate_init, + .iterate = cdb_dict_iterate, + .iterate_deinit = cdb_dict_iterate_deinit, + } +}; +#endif diff --git a/src/lib-dict-backend/dict-ldap-settings.c b/src/lib-dict-backend/dict-ldap-settings.c new file mode 100644 index 0000000..01205eb --- /dev/null +++ b/src/lib-dict-backend/dict-ldap-settings.c @@ -0,0 +1,313 @@ +/* Copyright (c) 2008-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" + +#if defined(BUILTIN_LDAP) || defined(PLUGIN_BUILD) + +#include "array.h" +#include "str.h" +#include "settings.h" +#include "dict-ldap-settings.h" + +#include <ctype.h> + +static const char *dict_ldap_commonName = "cn"; +static const char *dict_ldap_empty_filter = ""; + +enum section_type { + SECTION_ROOT = 0, + SECTION_MAP, + SECTION_FIELDS +}; + +struct dict_ldap_map_attribute { + const char *name; + const char *variable; +}; + +struct setting_parser_ctx { + pool_t pool; + struct dict_ldap_settings *set; + enum section_type type; + + struct dict_ldap_map cur_map; + ARRAY(struct dict_ldap_map_attribute) cur_attributes; +}; + +#undef DEF_STR +#undef DEF_BOOL +#undef DEF_UINT + +#define DEF_STR(name) DEF_STRUCT_STR(name, dict_ldap_map) +#define DEF_BOOL(name) DEF_STRUCT_BOOL(name, dict_ldap_map) +#define DEF_UINT(name) DEF_STRUCT_UINT(name ,dict_ldap_map) + +static const struct setting_def dict_ldap_map_setting_defs[] = { + DEF_STR(pattern), + DEF_STR(filter), + DEF_STR(filter_iter), + DEF_STR(username_attribute), + DEF_STR(value_attribute), + DEF_STR(base_dn), + DEF_STR(scope), + { 0, NULL, 0 } +}; + +static const char *pattern_read_name(const char **pattern) +{ + const char *p = *pattern, *name; + + if (*p == '{') { + /* ${name} */ + name = ++p; + p = strchr(p, '}'); + if (p == NULL) { + /* error, but allow anyway */ + *pattern += strlen(*pattern); + return ""; + } + *pattern = p + 1; + } else { + /* $name - ends at the first non-alnum_ character */ + name = p; + for (; *p != '\0'; p++) { + if (!i_isalnum(*p) && *p != '_') + break; + } + *pattern = p; + } + name = t_strdup_until(name, p); + return name; +} + +static const char *dict_ldap_attributes_map(struct setting_parser_ctx *ctx) +{ + struct dict_ldap_map_attribute *attributes; + string_t *pattern; + const char *p, *name; + unsigned int i, count; + + /* go through the variables in the pattern, replace them with plain + '$' character and add its ldap attribute */ + pattern = t_str_new(strlen(ctx->cur_map.pattern) + 1); + attributes = array_get_modifiable(&ctx->cur_attributes, &count); + + p_array_init(&ctx->cur_map.ldap_attributes, ctx->pool, count); + for (p = ctx->cur_map.pattern; *p != '\0';) { + if (*p != '$') { + str_append_c(pattern, *p); + p++; + continue; + } + p++; + str_append_c(pattern, '$'); + + name = pattern_read_name(&p); + for (i = 0; i < count; i++) { + if (attributes[i].variable != NULL && + strcmp(attributes[i].variable, name) == 0) + break; + } + if (i == count) { + return t_strconcat("Missing LDAP attribute for variable: ", + name, NULL); + } + + /* mark this attribute as used */ + attributes[i].variable = NULL; + array_push_back(&ctx->cur_map.ldap_attributes, + &attributes[i].name); + } + + /* make sure there aren't any unused attributes */ + for (i = 0; i < count; i++) { + if (attributes[i].variable != NULL) { + return t_strconcat("Unused variable: ", + attributes[i].variable, NULL); + } + } + + if (ctx->set->max_attribute_count < count) + ctx->set->max_attribute_count = count; + ctx->cur_map.pattern = p_strdup(ctx->pool, str_c(pattern)); + return NULL; +} + +static const char *dict_ldap_map_finish(struct setting_parser_ctx *ctx) +{ + if (ctx->cur_map.pattern == NULL) + return "Missing setting: pattern"; + if (ctx->cur_map.filter == NULL) + ctx->cur_map.filter = dict_ldap_empty_filter; + if (*ctx->cur_map.filter != '\0') { + const char *ptr = ctx->cur_map.filter; + if (*ptr != '(') + return "Filter must start with ("; + while(*ptr != '\0') ptr++; + ptr--; + if (*ptr != ')') + return "Filter must end with )"; + } + if (ctx->cur_map.value_attribute == NULL) + return "Missing setting: value_attribute"; + + if (ctx->cur_map.username_attribute == NULL) { + /* default to commonName */ + ctx->cur_map.username_attribute = dict_ldap_commonName; + } + if (ctx->cur_map.scope == NULL) { + ctx->cur_map.scope_val = 2; /* subtree */ + } else { + if (strcasecmp(ctx->cur_map.scope, "one") == 0) ctx->cur_map.scope_val = 1; + else if (strcasecmp(ctx->cur_map.scope, "base") == 0) ctx->cur_map.scope_val = 0; + else if (strcasecmp(ctx->cur_map.scope, "subtree") == 0) ctx->cur_map.scope_val = 2; + else return "Scope must be one, base or subtree"; + } + if (!array_is_created(&ctx->cur_map.ldap_attributes)) { + /* no attributes besides value. allocate the array anyway. */ + p_array_init(&ctx->cur_map.ldap_attributes, ctx->pool, 1); + if (strchr(ctx->cur_map.pattern, '$') != NULL) + return "Missing attributes for pattern variables"; + } + array_push_back(&ctx->set->maps, &ctx->cur_map); + i_zero(&ctx->cur_map); + return NULL; +} + +static const char * +parse_setting(const char *key, const char *value, + struct setting_parser_ctx *ctx) +{ + struct dict_ldap_map_attribute *attribute; + + switch (ctx->type) { + case SECTION_ROOT: + if (strcmp(key, "uri") == 0) { + ctx->set->uri = p_strdup(ctx->pool, value); + return NULL; + } + if (strcmp(key, "bind_dn") == 0) { + ctx->set->bind_dn = p_strdup(ctx->pool, value); + return NULL; + } + if (strcmp(key, "password") == 0) { + ctx->set->password = p_strdup(ctx->pool, value); + return NULL; + } + if (strcmp(key, "timeout") == 0) { + if (str_to_uint(value, &ctx->set->timeout) != 0) { + return "Invalid timeout value"; + } + return NULL; + } + if (strcmp(key, "max_idle_time") == 0) { + if (str_to_uint(value, &ctx->set->max_idle_time) != 0) { + return "Invalid max_idle_time value"; + } + return NULL; + } + if (strcmp(key, "debug") == 0) { + if (str_to_uint(value, &ctx->set->debug) != 0) { + return "invalid debug value"; + } + return NULL; + } + if (strcmp(key, "tls") == 0) { + if (strcasecmp(value, "yes") == 0) { + ctx->set->require_ssl = TRUE; + ctx->set->start_tls = TRUE; + } else if (strcasecmp(value, "no") == 0) { + ctx->set->require_ssl = FALSE; + ctx->set->start_tls = FALSE; + } else if (strcasecmp(value, "try") == 0) { + ctx->set->require_ssl = FALSE; + ctx->set->start_tls = TRUE; + } else { + return "tls must be yes, try or no"; + } + return NULL; + } + break; + case SECTION_MAP: + return parse_setting_from_defs(ctx->pool, + dict_ldap_map_setting_defs, + &ctx->cur_map, key, value); + case SECTION_FIELDS: + if (*value != '$') { + return t_strconcat("Value is missing '$' for attribute: ", + key, NULL); + } + attribute = array_append_space(&ctx->cur_attributes); + attribute->name = p_strdup(ctx->pool, key); + attribute->variable = p_strdup(ctx->pool, value + 1); + return NULL; + } + return t_strconcat("Unknown setting: ", key, NULL); +} + +static bool +parse_section(const char *type, const char *name ATTR_UNUSED, + struct setting_parser_ctx *ctx, const char **error_r) +{ + switch (ctx->type) { + case SECTION_ROOT: + if (type == NULL) + return FALSE; + if (strcmp(type, "map") == 0) { + array_clear(&ctx->cur_attributes); + ctx->type = SECTION_MAP; + return TRUE; + } + break; + case SECTION_MAP: + if (type == NULL) { + ctx->type = SECTION_ROOT; + *error_r = dict_ldap_map_finish(ctx); + return FALSE; + } + if (strcmp(type, "fields") == 0) { + ctx->type = SECTION_FIELDS; + return TRUE; + } + break; + case SECTION_FIELDS: + if (type == NULL) { + ctx->type = SECTION_MAP; + *error_r = dict_ldap_attributes_map(ctx); + return FALSE; + } + break; + } + *error_r = t_strconcat("Unknown section: ", type, NULL); + return FALSE; +} + +struct dict_ldap_settings * +dict_ldap_settings_read(pool_t pool, const char *path, const char **error_r) +{ + struct setting_parser_ctx ctx; + + i_zero(&ctx); + ctx.pool = pool; + ctx.set = p_new(pool, struct dict_ldap_settings, 1); + t_array_init(&ctx.cur_attributes, 16); + p_array_init(&ctx.set->maps, pool, 8); + + ctx.set->timeout = 30; /* default timeout */ + ctx.set->require_ssl = FALSE; /* try to start SSL */ + ctx.set->start_tls = TRUE; + + if (!settings_read(path, NULL, parse_setting, parse_section, + &ctx, error_r)) + return NULL; + + if (ctx.set->uri == NULL) { + *error_r = t_strdup_printf("Error in configuration file %s: " + "Missing ldap uri", path); + return NULL; + } + + return ctx.set; +} + +#endif diff --git a/src/lib-dict-backend/dict-ldap-settings.h b/src/lib-dict-backend/dict-ldap-settings.h new file mode 100644 index 0000000..0919ca9 --- /dev/null +++ b/src/lib-dict-backend/dict-ldap-settings.h @@ -0,0 +1,36 @@ +#ifndef DICT_LDAP_SETTINGS_H +#define DICT_LDAP_SETTINGS_H + +struct dict_ldap_map { + /* pattern is in simplified form: all variables are stored as simple + '$' character. fields array is sorted by the variable index. */ + const char *pattern; + const char *filter; + const char *filter_iter; + const char *username_attribute; + const char *value_attribute; + const char *base_dn; + const char *scope; + int scope_val; + unsigned int timeout; + + ARRAY_TYPE(const_string) ldap_attributes; +}; + +struct dict_ldap_settings { + const char *uri; + const char *bind_dn; + const char *password; + unsigned int timeout; + unsigned int max_idle_time; + unsigned int debug; + unsigned int max_attribute_count; + bool require_ssl; + bool start_tls; + ARRAY(struct dict_ldap_map) maps; +}; + +struct dict_ldap_settings * +dict_ldap_settings_read(pool_t pool, const char *path, const char **error_r); + +#endif diff --git a/src/lib-dict-backend/dict-ldap.c b/src/lib-dict-backend/dict-ldap.c new file mode 100644 index 0000000..433871b --- /dev/null +++ b/src/lib-dict-backend/dict-ldap.c @@ -0,0 +1,500 @@ +/* Copyright (c) 2016-2018 Dovecot authors, see the included COPYING memcached */ + +#include "lib.h" + +#if defined(BUILTIN_LDAP) || defined(PLUGIN_BUILD) + +#include "array.h" +#include "module-dir.h" +#include "str.h" +#include "istream.h" +#include "ostream.h" +#include "var-expand.h" +#include "connection.h" +#include "llist.h" +#include "ldap-client.h" +#include "dict.h" +#include "dict-private.h" +#include "dict-ldap-settings.h" + +static const char *LDAP_ESCAPE_CHARS = "*,\\#+<>;\"()= "; + +struct ldap_dict; + +struct dict_ldap_op { + struct ldap_dict *dict; + const struct dict_ldap_map *map; + pool_t pool; + unsigned long txid; + struct dict_lookup_result res; + dict_lookup_callback_t *callback; + void *callback_ctx; +}; + +struct ldap_dict { + struct dict dict; + struct dict_ldap_settings *set; + + const char *uri; + const char *base_dn; + enum ldap_scope scope; + + pool_t pool; + + struct ldap_client *client; + + unsigned long last_txid; + unsigned int pending; + + struct ldap_dict *prev,*next; +}; + +static +void ldap_dict_lookup_async(struct dict *dict, + const struct dict_op_settings *set, const char *key, + dict_lookup_callback_t *callback, void *context); + + +static bool +dict_ldap_map_match(const struct dict_ldap_map *map, const char *path, + ARRAY_TYPE(const_string) *values, size_t *pat_len_r, + size_t *path_len_r, bool partial_ok, bool recurse) +{ + const char *path_start = path; + const char *pat, *attribute, *p; + size_t len; + + array_clear(values); + pat = map->pattern; + while (*pat != '\0' && *path != '\0') { + if (*pat == '$') { + /* variable */ + pat++; + if (*pat == '\0') { + /* pattern ended with this variable, + it'll match the rest of the path */ + len = strlen(path); + if (partial_ok) { + /* iterating - the last field never + matches fully. if there's a trailing + '/', drop it. */ + pat--; + if (path[len-1] == '/') { + attribute = t_strndup(path, len-1); + array_push_back(values, + &attribute); + } else { + array_push_back(values, &path); + } + } else { + array_push_back(values, &path); + path += len; + } + *path_len_r = path - path_start; + *pat_len_r = pat - map->pattern; + return TRUE; + } + /* pattern matches until the next '/' in path */ + p = strchr(path, '/'); + if (p != NULL) { + attribute = t_strdup_until(path, p); + array_push_back(values, &attribute); + path = p; + } else { + /* no '/' anymore, but it'll still match a + partial */ + array_push_back(values, &path); + path += strlen(path); + pat++; + } + } else if (*pat == *path) { + pat++; + path++; + } else { + return FALSE; + } + } + + *path_len_r = path - path_start; + *pat_len_r = pat - map->pattern; + + if (*pat == '\0') + return *path == '\0'; + else if (!partial_ok) + return FALSE; + else { + /* partial matches must end with '/'. */ + if (pat != map->pattern && pat[-1] != '/') + return FALSE; + /* if we're not recursing, there should be only one $variable + left. */ + if (recurse) + return TRUE; + return pat[0] == '$' && strchr(pat, '/') == NULL; + } +} + +static const struct dict_ldap_map * +ldap_dict_find_map(struct ldap_dict *dict, const char *path, + ARRAY_TYPE(const_string) *values) +{ + const struct dict_ldap_map *maps; + unsigned int i, count; + size_t len; + + t_array_init(values, dict->set->max_attribute_count); + maps = array_get(&dict->set->maps, &count); + for (i = 0; i < count; i++) { + if (dict_ldap_map_match(&maps[i], path, values, + &len, &len, FALSE, FALSE)) + return &maps[i]; + } + return NULL; +} + +static +int dict_ldap_connect(struct ldap_dict *dict, const char **error_r) +{ + struct ldap_client_settings set; + i_zero(&set); + set.uri = dict->set->uri; + set.bind_dn = dict->set->bind_dn; + set.password = dict->set->password; + set.timeout_secs = dict->set->timeout; + set.max_idle_time_secs = dict->set->max_idle_time; + set.debug = dict->set->debug; + set.require_ssl = dict->set->require_ssl; + set.start_tls = dict->set->start_tls; + return ldap_client_init(&set, &dict->client, error_r); +} + +#define IS_LDAP_ESCAPED_CHAR(c) \ + ((((unsigned char)(c)) & 0x80) != 0 || strchr(LDAP_ESCAPE_CHARS, (c)) != NULL) + +static const char *ldap_escape(const char *str) +{ + string_t *ret = NULL; + + for (const char *p = str; *p != '\0'; p++) { + if (IS_LDAP_ESCAPED_CHAR(*p)) { + if (ret == NULL) { + ret = t_str_new((size_t) (p - str) + 64); + str_append_data(ret, str, (size_t) (p - str)); + } + str_printfa(ret, "\\%02X", (unsigned char)*p); + } else if (ret != NULL) + str_append_c(ret, *p); + } + + return ret == NULL ? str : str_c(ret); +} + +static bool +ldap_dict_build_query(const struct dict_op_settings *set, + const struct dict_ldap_map *map, + ARRAY_TYPE(const_string) *values, bool priv, + string_t *query_r, const char **error_r) +{ + const char *template, *error; + ARRAY(struct var_expand_table) exp; + struct var_expand_table entry; + + t_array_init(&exp, 8); + entry.key = '\0'; + entry.value = ldap_escape(set->username); + entry.long_key = "username"; + array_push_back(&exp, &entry); + + if (priv) { + template = t_strdup_printf("(&(%s=%s)%s)", map->username_attribute, "%{username}", map->filter); + } else { + template = map->filter; + } + + for(size_t i = 0; i < array_count(values) && i < array_count(&map->ldap_attributes); i++) { + struct var_expand_table entry; + const char *value = array_idx_elem(values, i); + const char *long_key = array_idx_elem(&map->ldap_attributes, i); + + entry.value = ldap_escape(value); + entry.long_key = long_key; + array_push_back(&exp, &entry); + } + + array_append_zero(&exp); + + if (var_expand(query_r, template, array_front(&exp), &error) <= 0) { + *error_r = t_strdup_printf("Failed to expand %s: %s", template, error); + return FALSE; + } + return TRUE; +} + +static +int ldap_dict_init(struct dict *dict_driver, const char *uri, + const struct dict_settings *set ATTR_UNUSED, + struct dict **dict_r, const char **error_r) +{ + pool_t pool = pool_alloconly_create("ldap dict", 2048); + struct ldap_dict *dict = p_new(pool, struct ldap_dict, 1); + dict->pool = pool; + dict->dict = *dict_driver; + dict->uri = p_strdup(pool, uri); + dict->set = dict_ldap_settings_read(pool, uri, error_r); + + if (dict->set == NULL) { + pool_unref(&pool); + return -1; + } + + if (dict_ldap_connect(dict, error_r) < 0) { + pool_unref(&pool); + return -1; + } + + *dict_r = (struct dict*)dict; + *error_r = NULL; + return 0; +} + +static +void ldap_dict_deinit(struct dict *dict) +{ + struct ldap_dict *ctx = (struct ldap_dict *)dict; + + ldap_client_deinit(&ctx->client); + pool_unref(&ctx->pool); +} + +static void ldap_dict_wait(struct dict *dict) +{ + struct ldap_dict *ctx = (struct ldap_dict *)dict; + + i_assert(ctx->dict.ioloop == NULL); + + ctx->dict.prev_ioloop = current_ioloop; + ctx->dict.ioloop = io_loop_create(); + dict_switch_ioloop(dict); + + do { + io_loop_run(current_ioloop); + } while (ctx->pending > 0); + + io_loop_set_current(ctx->dict.prev_ioloop); + dict_switch_ioloop(dict); + io_loop_set_current(ctx->dict.ioloop); + io_loop_destroy(&ctx->dict.ioloop); + ctx->dict.prev_ioloop = NULL; +} + +static bool ldap_dict_switch_ioloop(struct dict *dict) +{ + struct ldap_dict *ctx = (struct ldap_dict *)dict; + + ldap_client_switch_ioloop(ctx->client); + return ctx->pending > 0; +} + +static +void ldap_dict_lookup_done(const struct dict_lookup_result *result, void *ctx) +{ + struct dict_lookup_result *res = ctx; + res->ret = result->ret; + res->value = t_strdup(result->value); + res->error = t_strdup(result->error); +} + +static void +ldap_dict_lookup_callback(struct ldap_result *result, struct dict_ldap_op *op) +{ + pool_t pool = op->pool; + struct ldap_search_iterator *iter; + const struct ldap_entry *entry; + + op->dict->pending--; + + if (ldap_result_has_failed(result)) { + op->res.ret = -1; + op->res.error = ldap_result_get_error(result); + } else { + iter = ldap_search_iterator_init(result); + entry = ldap_search_iterator_next(iter); + if (entry != NULL) { + if (op->dict->set->debug > 0) + i_debug("ldap_dict_lookup_callback got dn %s", ldap_entry_dn(entry)); + /* try extract value */ + const char *const *values = ldap_entry_get_attribute(entry, op->map->value_attribute); + if (values != NULL) { + const char **new_values; + + if (op->dict->set->debug > 0) + i_debug("ldap_dict_lookup_callback got attribute %s", op->map->value_attribute); + op->res.ret = 1; + new_values = p_new(op->pool, const char *, 2); + new_values[0] = p_strdup(op->pool, values[0]); + op->res.values = new_values; + op->res.value = op->res.values[0]; + } else { + if (op->dict->set->debug > 0) + i_debug("ldap_dict_lookup_callback dit not get attribute %s", op->map->value_attribute); + op->res.value = NULL; + } + } + ldap_search_iterator_deinit(&iter); + } + if (op->dict->dict.prev_ioloop != NULL) + io_loop_set_current(op->dict->dict.prev_ioloop); + op->callback(&op->res, op->callback_ctx); + if (op->dict->dict.prev_ioloop != NULL) { + io_loop_set_current(op->dict->dict.ioloop); + io_loop_stop(op->dict->dict.ioloop); + } + pool_unref(&pool); +} + +static int +ldap_dict_lookup(struct dict *dict, const struct dict_op_settings *set, + pool_t pool, const char *key, + const char **value_r, const char **error_r) +{ + struct dict_lookup_result res; + + ldap_dict_lookup_async(dict, set, key, ldap_dict_lookup_done, &res); + + ldap_dict_wait(dict); + if (res.ret < 0) { + *error_r = res.error; + return -1; + } + if (res.ret > 0) + *value_r = p_strdup(pool, res.value); + return res.ret; +} + +/* +static +struct dict_iterate_context *ldap_dict_iterate_init(struct dict *dict, + const char *const *paths, + enum dict_iterate_flags flags) +{ + return NULL; +} + +static +bool ldap_dict_iterate(struct dict_iterate_context *ctx, + const char **key_r, const char **value_r) +{ + return FALSE; +} + +static +int ldap_dict_iterate_deinit(struct dict_iterate_context *ctx) +{ + return -1; +} + +static +struct dict_transaction_context ldap_dict_transaction_init(struct dict *dict); + +static +int ldap_dict_transaction_commit(struct dict_transaction_context *ctx, + bool async, + dict_transaction_commit_callback_t *callback, + void *context); +static +void ldap_dict_transaction_rollback(struct dict_transaction_context *ctx); + +static +void ldap_dict_set(struct dict_transaction_context *ctx, + const char *key, const char *value); +static +void ldap_dict_unset(struct dict_transaction_context *ctx, + const char *key); +static +void ldap_dict_atomic_inc(struct dict_transaction_context *ctx, + const char *key, long long diff); +*/ + +static +void ldap_dict_lookup_async(struct dict *dict, + const struct dict_op_settings *set, + const char *key, + dict_lookup_callback_t *callback, void *context) +{ + struct ldap_search_input input; + struct ldap_dict *ctx = (struct ldap_dict*)dict; + struct dict_ldap_op *op; + const char *error; + + pool_t oppool = pool_alloconly_create("ldap dict lookup", 64); + string_t *query = str_new(oppool, 64); + op = p_new(oppool, struct dict_ldap_op, 1); + op->pool = oppool; + op->dict = ctx; + op->callback = callback; + op->callback_ctx = context; + op->txid = ctx->last_txid++; + + /* key needs to be transformed into something else */ + ARRAY_TYPE(const_string) values; + const char *attributes[2] = {0, 0}; + t_array_init(&values, 8); + const struct dict_ldap_map *map = ldap_dict_find_map(ctx, key, &values); + + if (map != NULL) { + op->map = map; + attributes[0] = map->value_attribute; + /* build lookup */ + i_zero(&input); + input.base_dn = map->base_dn; + input.scope = map->scope_val; + if (!ldap_dict_build_query(set, map, &values, strncmp(key, DICT_PATH_PRIVATE, strlen(DICT_PATH_PRIVATE))==0, query, &error)) { + op->res.error = error; + callback(&op->res, context); + pool_unref(&oppool); + return; + } + input.filter = str_c(query); + input.attributes = attributes; + input.timeout_secs = ctx->set->timeout; + ctx->pending++; + ldap_search_start(ctx->client, &input, ldap_dict_lookup_callback, op); + } else { + op->res.error = "no such key"; + callback(&op->res, context); + pool_unref(&oppool); + } +} + +struct dict dict_driver_ldap = { + .name = "ldap", + { + .init = ldap_dict_init, + .deinit = ldap_dict_deinit, + .wait = ldap_dict_wait, + .lookup = ldap_dict_lookup, + .lookup_async = ldap_dict_lookup_async, + .switch_ioloop = ldap_dict_switch_ioloop, + } +}; + +#ifndef BUILTIN_LDAP +/* Building a plugin */ +void dict_ldap_init(struct module *module ATTR_UNUSED); +void dict_ldap_deinit(void); + +void dict_ldap_init(struct module *module ATTR_UNUSED) +{ + dict_driver_register(&dict_driver_ldap); +} + +void dict_ldap_deinit(void) +{ + ldap_clients_cleanup(); + dict_driver_unregister(&dict_driver_ldap); +} + +const char *dict_ldap_plugin_dependencies[] = { NULL }; +#endif + +#endif diff --git a/src/lib-dict-backend/dict-sql-private.h b/src/lib-dict-backend/dict-sql-private.h new file mode 100644 index 0000000..11e9dfc --- /dev/null +++ b/src/lib-dict-backend/dict-sql-private.h @@ -0,0 +1,12 @@ +#ifndef DICT_SQL_PRIVATE_H +#define DICT_SQL_PRIVATE_H 1 + +struct sql_dict { + struct dict dict; + + pool_t pool; + struct sql_db *db; + const struct dict_sql_settings *set; +}; + +#endif diff --git a/src/lib-dict-backend/dict-sql-settings.c b/src/lib-dict-backend/dict-sql-settings.c new file mode 100644 index 0000000..0d44923 --- /dev/null +++ b/src/lib-dict-backend/dict-sql-settings.c @@ -0,0 +1,345 @@ +/* Copyright (c) 2008-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "str.h" +#include "hash.h" +#include "settings.h" +#include "dict-sql-settings.h" + +#include <ctype.h> + +enum section_type { + SECTION_ROOT = 0, + SECTION_MAP, + SECTION_FIELDS +}; + +struct dict_sql_map_field { + struct dict_sql_field sql_field; + const char *variable; +}; + +struct setting_parser_ctx { + pool_t pool; + struct dict_sql_settings *set; + enum section_type type; + + struct dict_sql_map cur_map; + ARRAY(struct dict_sql_map_field) cur_fields; +}; + +#define DEF_STR(name) DEF_STRUCT_STR(name, dict_sql_map) +#define DEF_BOOL(name) DEF_STRUCT_BOOL(name, dict_sql_map) + +static const struct setting_def dict_sql_map_setting_defs[] = { + DEF_STR(pattern), + DEF_STR(table), + DEF_STR(username_field), + DEF_STR(value_field), + DEF_STR(value_type), + DEF_BOOL(value_hexblob), + + { 0, NULL, 0 } +}; + +struct dict_sql_settings_cache { + pool_t pool; + const char *path; + struct dict_sql_settings *set; +}; + +static HASH_TABLE(const char *, struct dict_sql_settings_cache *) dict_sql_settings_cache; + +static const char *pattern_read_name(const char **pattern) +{ + const char *p = *pattern, *name; + + if (*p == '{') { + /* ${name} */ + name = ++p; + p = strchr(p, '}'); + if (p == NULL) { + /* error, but allow anyway */ + *pattern += strlen(*pattern); + return ""; + } + *pattern = p + 1; + } else { + /* $name - ends at the first non-alnum_ character */ + name = p; + for (; *p != '\0'; p++) { + if (!i_isalnum(*p) && *p != '_') + break; + } + *pattern = p; + } + name = t_strdup_until(name, p); + return name; +} + +static const char *dict_sql_fields_map(struct setting_parser_ctx *ctx) +{ + struct dict_sql_map_field *fields; + string_t *pattern; + const char *p, *name; + unsigned int i, count; + + /* go through the variables in the pattern, replace them with plain + '$' character and add its sql field */ + pattern = t_str_new(strlen(ctx->cur_map.pattern) + 1); + fields = array_get_modifiable(&ctx->cur_fields, &count); + + p_array_init(&ctx->cur_map.pattern_fields, ctx->pool, count); + for (p = ctx->cur_map.pattern; *p != '\0';) { + if (*p != '$') { + str_append_c(pattern, *p); + p++; + continue; + } + p++; + str_append_c(pattern, '$'); + + name = pattern_read_name(&p); + for (i = 0; i < count; i++) { + if (fields[i].variable != NULL && + strcmp(fields[i].variable, name) == 0) + break; + } + if (i == count) { + return t_strconcat("Missing SQL field for variable: ", + name, NULL); + } + + /* mark this field as used */ + fields[i].variable = NULL; + array_push_back(&ctx->cur_map.pattern_fields, + &fields[i].sql_field); + } + + /* make sure there aren't any unused fields */ + for (i = 0; i < count; i++) { + if (fields[i].variable != NULL) { + return t_strconcat("Unused variable: ", + fields[i].variable, NULL); + } + } + + if (ctx->set->max_pattern_fields_count < count) + ctx->set->max_pattern_fields_count = count; + ctx->cur_map.pattern = p_strdup(ctx->pool, str_c(pattern)); + return NULL; +} + +static bool +dict_sql_value_type_parse(const char *value_type, enum dict_sql_type *type_r) +{ + if (strcmp(value_type, "string") == 0) + *type_r = DICT_SQL_TYPE_STRING; + else if (strcmp(value_type, "hexblob") == 0) + *type_r = DICT_SQL_TYPE_HEXBLOB; + else if (strcmp(value_type, "int") == 0) + *type_r = DICT_SQL_TYPE_INT; + else if (strcmp(value_type, "uint") == 0) + *type_r = DICT_SQL_TYPE_UINT; + else + return FALSE; + return TRUE; +} + +static const char *dict_sql_map_finish(struct setting_parser_ctx *ctx) +{ + unsigned int i; + + if (ctx->cur_map.pattern == NULL) + return "Missing setting: pattern"; + if (ctx->cur_map.table == NULL) + return "Missing setting: table"; + if (ctx->cur_map.value_field == NULL) + return "Missing setting: value_field"; + + ctx->cur_map.value_fields = (const char *const *) + p_strsplit_spaces(ctx->pool, ctx->cur_map.value_field, ","); + ctx->cur_map.values_count = str_array_length(ctx->cur_map.value_fields); + + enum dict_sql_type *value_types = + p_new(ctx->pool, enum dict_sql_type, ctx->cur_map.values_count); + if (ctx->cur_map.value_type != NULL) { + const char *const *types = + t_strsplit_spaces(ctx->cur_map.value_type, ","); + if (str_array_length(types) != ctx->cur_map.values_count) + return "Number of fields in value_fields doesn't match value_type"; + for (i = 0; i < ctx->cur_map.values_count; i++) { + if (!dict_sql_value_type_parse(types[i], &value_types[i])) + return "Invalid value in value_type"; + } + } else { + for (i = 0; i < ctx->cur_map.values_count; i++) { + value_types[i] = ctx->cur_map.value_hexblob ? + DICT_SQL_TYPE_HEXBLOB : DICT_SQL_TYPE_STRING; + } + } + ctx->cur_map.value_types = value_types; + + if (ctx->cur_map.username_field == NULL) { + /* not all queries require this */ + ctx->cur_map.username_field = "'username_field not set'"; + } + + if (!array_is_created(&ctx->cur_map.pattern_fields)) { + /* no fields besides value. allocate the array anyway. */ + p_array_init(&ctx->cur_map.pattern_fields, ctx->pool, 1); + if (strchr(ctx->cur_map.pattern, '$') != NULL) + return "Missing fields for pattern variables"; + } + array_push_back(&ctx->set->maps, &ctx->cur_map); + i_zero(&ctx->cur_map); + return NULL; +} + +static const char * +parse_setting(const char *key, const char *value, + struct setting_parser_ctx *ctx) +{ + struct dict_sql_map_field *field; + size_t value_len; + + switch (ctx->type) { + case SECTION_ROOT: + if (strcmp(key, "connect") == 0) { + ctx->set->connect = p_strdup(ctx->pool, value); + return NULL; + } + break; + case SECTION_MAP: + return parse_setting_from_defs(ctx->pool, + dict_sql_map_setting_defs, + &ctx->cur_map, key, value); + case SECTION_FIELDS: + if (*value != '$') { + return t_strconcat("Value is missing '$' for field: ", + key, NULL); + } + field = array_append_space(&ctx->cur_fields); + field->sql_field.name = p_strdup(ctx->pool, key); + value_len = strlen(value); + if (str_begins(value, "${hexblob:") && + value[value_len-1] == '}') { + field->variable = p_strndup(ctx->pool, value + 10, + value_len-10-1); + field->sql_field.value_type = DICT_SQL_TYPE_HEXBLOB; + } else if (str_begins(value, "${int:") && + value[value_len-1] == '}') { + field->variable = p_strndup(ctx->pool, value + 6, + value_len-6-1); + field->sql_field.value_type = DICT_SQL_TYPE_INT; + } else if (str_begins(value, "${uint:") && + value[value_len-1] == '}') { + field->variable = p_strndup(ctx->pool, value + 7, + value_len-7-1); + field->sql_field.value_type = DICT_SQL_TYPE_UINT; + } else { + field->variable = p_strdup(ctx->pool, value + 1); + } + return NULL; + } + return t_strconcat("Unknown setting: ", key, NULL); +} + +static bool +parse_section(const char *type, const char *name ATTR_UNUSED, + struct setting_parser_ctx *ctx, const char **error_r) +{ + switch (ctx->type) { + case SECTION_ROOT: + if (type == NULL) + return FALSE; + if (strcmp(type, "map") == 0) { + array_clear(&ctx->cur_fields); + ctx->type = SECTION_MAP; + return TRUE; + } + break; + case SECTION_MAP: + if (type == NULL) { + ctx->type = SECTION_ROOT; + *error_r = dict_sql_map_finish(ctx); + return FALSE; + } + if (strcmp(type, "fields") == 0) { + ctx->type = SECTION_FIELDS; + return TRUE; + } + break; + case SECTION_FIELDS: + if (type == NULL) { + ctx->type = SECTION_MAP; + *error_r = dict_sql_fields_map(ctx); + return FALSE; + } + break; + } + *error_r = t_strconcat("Unknown section: ", type, NULL); + return FALSE; +} + +struct dict_sql_settings * +dict_sql_settings_read(const char *path, const char **error_r) +{ + struct setting_parser_ctx ctx; + struct dict_sql_settings_cache *cache; + pool_t pool; + + if (!hash_table_is_created(dict_sql_settings_cache)) { + hash_table_create(&dict_sql_settings_cache, default_pool, 0, + str_hash, strcmp); + } + + cache = hash_table_lookup(dict_sql_settings_cache, path); + if (cache != NULL) + return cache->set; + + i_zero(&ctx); + pool = pool_alloconly_create("dict sql settings", 1024); + ctx.pool = pool; + ctx.set = p_new(pool, struct dict_sql_settings, 1); + t_array_init(&ctx.cur_fields, 16); + p_array_init(&ctx.set->maps, pool, 8); + + if (!settings_read(path, NULL, parse_setting, parse_section, + &ctx, error_r)) { + pool_unref(&pool); + return NULL; + } + + if (ctx.set->connect == NULL) { + *error_r = t_strdup_printf("Error in configuration file %s: " + "Missing connect setting", path); + pool_unref(&pool); + return NULL; + } + + cache = p_new(pool, struct dict_sql_settings_cache, 1); + cache->pool = pool; + cache->path = p_strdup(pool, path); + cache->set = ctx.set; + + hash_table_insert(dict_sql_settings_cache, cache->path, cache); + return ctx.set; +} + +void dict_sql_settings_deinit(void) +{ + struct hash_iterate_context *iter; + struct dict_sql_settings_cache *cache; + const char *key; + + if (!hash_table_is_created(dict_sql_settings_cache)) + return; + + iter = hash_table_iterate_init(dict_sql_settings_cache); + while (hash_table_iterate(iter, dict_sql_settings_cache, &key, &cache)) + pool_unref(&cache->pool); + hash_table_iterate_deinit(&iter); + hash_table_destroy(&dict_sql_settings_cache); +} diff --git a/src/lib-dict-backend/dict-sql-settings.h b/src/lib-dict-backend/dict-sql-settings.h new file mode 100644 index 0000000..c9ee037 --- /dev/null +++ b/src/lib-dict-backend/dict-sql-settings.h @@ -0,0 +1,47 @@ +#ifndef DICT_SQL_SETTINGS_H +#define DICT_SQL_SETTINGS_H + +enum dict_sql_type { + DICT_SQL_TYPE_STRING = 0, + DICT_SQL_TYPE_INT, + DICT_SQL_TYPE_UINT, + DICT_SQL_TYPE_HEXBLOB +}; + +struct dict_sql_field { + const char *name; + enum dict_sql_type value_type; +}; + +struct dict_sql_map { + /* pattern is in simplified form: all variables are stored as simple + '$' character. fields array is sorted by the variable index. */ + const char *pattern; + const char *table; + const char *username_field; + const char *value_field; + const char *value_type; + bool value_hexblob; + + /* SQL field names, one for each $ variable in the pattern */ + ARRAY(struct dict_sql_field) pattern_fields; + + /* generated: */ + unsigned int values_count; + const char *const *value_fields; + const enum dict_sql_type *value_types; +}; + +struct dict_sql_settings { + const char *connect; + + unsigned int max_pattern_fields_count; + ARRAY(struct dict_sql_map) maps; +}; + +struct dict_sql_settings * +dict_sql_settings_read(const char *path, const char **error_r); + +void dict_sql_settings_deinit(void); + +#endif diff --git a/src/lib-dict-backend/dict-sql.c b/src/lib-dict-backend/dict-sql.c new file mode 100644 index 0000000..c44330b --- /dev/null +++ b/src/lib-dict-backend/dict-sql.c @@ -0,0 +1,1564 @@ +/* Copyright (c) 2005-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "istream.h" +#include "hex-binary.h" +#include "hash.h" +#include "str.h" +#include "sql-api-private.h" +#include "sql-db-cache.h" +#include "dict-private.h" +#include "dict-sql-settings.h" +#include "dict-sql.h" +#include "dict-sql-private.h" + +#include <unistd.h> +#include <fcntl.h> + +#define DICT_SQL_MAX_UNUSED_CONNECTIONS 10 + +enum sql_recurse_type { + SQL_DICT_RECURSE_NONE, + SQL_DICT_RECURSE_ONE, + SQL_DICT_RECURSE_FULL +}; + +struct sql_dict_param { + enum dict_sql_type value_type; + + const char *value_str; + int64_t value_int64; + const void *value_binary; + size_t value_binary_size; +}; +ARRAY_DEFINE_TYPE(sql_dict_param, struct sql_dict_param); + +struct sql_dict_iterate_context { + struct dict_iterate_context ctx; + pool_t pool; + + enum dict_iterate_flags flags; + const char *path; + + struct sql_result *result; + string_t *key; + const struct dict_sql_map *map; + size_t key_prefix_len, pattern_prefix_len; + unsigned int sql_fields_start_idx, next_map_idx; + bool destroyed; + bool synchronous_result; + bool iter_query_sent; + bool allow_null_map; /* allow next map to be NULL */ + const char *error; +}; + +struct sql_dict_inc_row { + struct sql_dict_inc_row *prev; + unsigned int rows; +}; + +struct sql_dict_prev { + const struct dict_sql_map *map; + char *key; + union { + char *str; + long long diff; + } value; +}; + +struct sql_dict_transaction_context { + struct dict_transaction_context ctx; + + struct sql_transaction_context *sql_ctx; + + pool_t inc_row_pool; + struct sql_dict_inc_row *inc_row; + + ARRAY(struct sql_dict_prev) prev_inc; + ARRAY(struct sql_dict_prev) prev_set; + + dict_transaction_commit_callback_t *async_callback; + void *async_context; + + char *error; +}; + +static struct sql_db_cache *dict_sql_db_cache; + +static void sql_dict_prev_inc_flush(struct sql_dict_transaction_context *ctx); +static void sql_dict_prev_set_flush(struct sql_dict_transaction_context *ctx); +static void sql_dict_prev_inc_free(struct sql_dict_transaction_context *ctx); +static void sql_dict_prev_set_free(struct sql_dict_transaction_context *ctx); + +static int +sql_dict_init(struct dict *driver, const char *uri, + const struct dict_settings *set, + struct dict **dict_r, const char **error_r) +{ + struct sql_settings sql_set; + struct sql_dict *dict; + pool_t pool; + + pool = pool_alloconly_create("sql dict", 2048); + dict = p_new(pool, struct sql_dict, 1); + dict->pool = pool; + dict->dict = *driver; + dict->set = dict_sql_settings_read(uri, error_r); + if (dict->set == NULL) { + pool_unref(&pool); + return -1; + } + i_zero(&sql_set); + sql_set.driver = driver->name; + sql_set.connect_string = dict->set->connect; + sql_set.event_parent = set->event_parent; + + if (sql_db_cache_new(dict_sql_db_cache, &sql_set, &dict->db, error_r) < 0) { + pool_unref(&pool); + return -1; + } + + *dict_r = &dict->dict; + return 0; +} + +static void sql_dict_deinit(struct dict *_dict) +{ + struct sql_dict *dict = (struct sql_dict *)_dict; + + sql_unref(&dict->db); + pool_unref(&dict->pool); +} + +static void sql_dict_wait(struct dict *_dict) +{ + struct sql_dict *dict = (struct sql_dict *)_dict; + sql_wait(dict->db); +} + +/* Try to match path to map->pattern. For example pattern="shared/x/$/$/y" + and path="shared/x/1/2/y", this is match and pattern_values=[1, 2]. */ +static bool +dict_sql_map_match(const struct dict_sql_map *map, const char *path, + ARRAY_TYPE(const_string) *pattern_values, size_t *pat_len_r, + size_t *path_len_r, bool partial_ok, bool recurse) +{ + const char *path_start = path; + const char *pat, *field, *p; + size_t len; + + array_clear(pattern_values); + pat = map->pattern; + while (*pat != '\0' && *path != '\0') { + if (*pat == '$') { + /* variable */ + pat++; + if (*pat == '\0') { + /* pattern ended with this variable, + it'll match the rest of the path */ + len = strlen(path); + if (partial_ok) { + /* iterating - the last field never + matches fully. if there's a trailing + '/', drop it. */ + pat--; + if (path[len-1] == '/') { + field = t_strndup(path, len-1); + array_push_back(pattern_values, + &field); + } else { + array_push_back(pattern_values, + &path); + } + } else { + array_push_back(pattern_values, &path); + path += len; + } + *path_len_r = path - path_start; + *pat_len_r = pat - map->pattern; + return TRUE; + } + /* pattern matches until the next '/' in path */ + p = strchr(path, '/'); + if (p != NULL) { + field = t_strdup_until(path, p); + array_push_back(pattern_values, &field); + path = p; + } else { + /* no '/' anymore, but it'll still match a + partial */ + array_push_back(pattern_values, &path); + path += strlen(path); + pat++; + } + } else if (*pat == *path) { + pat++; + path++; + } else { + return FALSE; + } + } + + *path_len_r = path - path_start; + *pat_len_r = pat - map->pattern; + + if (*pat == '\0') + return *path == '\0'; + else if (!partial_ok) + return FALSE; + else { + /* partial matches must end with '/'. */ + if (pat != map->pattern && pat[-1] != '/') + return FALSE; + /* if we're not recursing, there should be only one $variable + left. */ + if (recurse) + return TRUE; + return pat[0] == '$' && strchr(pat, '/') == NULL; + } +} + +static const struct dict_sql_map * +sql_dict_find_map(struct sql_dict *dict, const char *path, + ARRAY_TYPE(const_string) *pattern_values) +{ + const struct dict_sql_map *maps; + unsigned int i, count; + size_t len; + + t_array_init(pattern_values, dict->set->max_pattern_fields_count); + maps = array_get(&dict->set->maps, &count); + for (i = 0; i < count; i++) { + if (dict_sql_map_match(&maps[i], path, pattern_values, + &len, &len, FALSE, FALSE)) + return &maps[i]; + } + return NULL; +} + +static void +sql_dict_statement_bind(struct sql_statement *stmt, unsigned int column_idx, + const struct sql_dict_param *param) +{ + switch (param->value_type) { + case DICT_SQL_TYPE_STRING: + sql_statement_bind_str(stmt, column_idx, param->value_str); + break; + case DICT_SQL_TYPE_INT: + case DICT_SQL_TYPE_UINT: + sql_statement_bind_int64(stmt, column_idx, param->value_int64); + break; + case DICT_SQL_TYPE_HEXBLOB: + sql_statement_bind_binary(stmt, column_idx, param->value_binary, + param->value_binary_size); + break; + } +} + +static struct sql_statement * +sql_dict_statement_init(struct sql_dict *dict, const char *query, + const ARRAY_TYPE(sql_dict_param) *params) +{ + struct sql_statement *stmt; + struct sql_prepared_statement *prep_stmt; + const struct sql_dict_param *param; + + if ((sql_get_flags(dict->db) & SQL_DB_FLAG_PREP_STATEMENTS) != 0) { + prep_stmt = sql_prepared_statement_init(dict->db, query); + stmt = sql_statement_init_prepared(prep_stmt); + sql_prepared_statement_unref(&prep_stmt); + } else { + /* Prepared statements not supported by the backend. + Just use regular statements to avoid wasting memory. */ + stmt = sql_statement_init(dict->db, query); + } + + array_foreach(params, param) { + sql_dict_statement_bind(stmt, array_foreach_idx(params, param), + param); + } + return stmt; +} + +static int +sql_dict_value_get(const struct dict_sql_map *map, + enum dict_sql_type value_type, const char *field_name, + const char *value, const char *value_suffix, + ARRAY_TYPE(sql_dict_param) *params, const char **error_r) +{ + struct sql_dict_param *param; + buffer_t *buf; + + param = array_append_space(params); + param->value_type = value_type; + + switch (value_type) { + case DICT_SQL_TYPE_STRING: + if (value_suffix[0] != '\0') + value = t_strconcat(value, value_suffix, NULL); + param->value_str = value; + return 0; + case DICT_SQL_TYPE_INT: + if (value_suffix[0] != '\0' || + str_to_int64(value, ¶m->value_int64) < 0) { + *error_r = t_strdup_printf( + "%s field's value isn't 64bit signed integer: %s%s (in pattern: %s)", + field_name, value, value_suffix, map->pattern); + return -1; + } + return 0; + case DICT_SQL_TYPE_UINT: + if (value_suffix[0] != '\0' || value[0] == '-' || + str_to_int64(value, ¶m->value_int64) < 0) { + *error_r = t_strdup_printf( + "%s field's value isn't 64bit unsigned integer: %s%s (in pattern: %s)", + field_name, value, value_suffix, map->pattern); + return -1; + } + return 0; + case DICT_SQL_TYPE_HEXBLOB: + break; + } + + buf = t_buffer_create(strlen(value)/2); + if (hex_to_binary(value, buf) < 0) { + /* we shouldn't get untrusted input here. it's also a bit + annoying to handle this error. */ + *error_r = t_strdup_printf("%s field's value isn't hexblob: %s (in pattern: %s)", + field_name, value, map->pattern); + return -1; + } + str_append(buf, value_suffix); + param->value_binary = buf->data; + param->value_binary_size = buf->used; + return 0; +} + +static int +sql_dict_field_get_value(const struct dict_sql_map *map, + const struct dict_sql_field *field, + const char *value, const char *value_suffix, + ARRAY_TYPE(sql_dict_param) *params, + const char **error_r) +{ + return sql_dict_value_get(map, field->value_type, field->name, + value, value_suffix, params, error_r); +} + +static int +sql_dict_where_build(const char *username, const struct dict_sql_map *map, + const ARRAY_TYPE(const_string) *values_arr, + bool add_username, enum sql_recurse_type recurse_type, + string_t *query, ARRAY_TYPE(sql_dict_param) *params, + const char **error_r) +{ + const struct dict_sql_field *pattern_fields; + const char *const *pattern_values; + unsigned int i, count, count2, exact_count; + + pattern_fields = array_get(&map->pattern_fields, &count); + pattern_values = array_get(values_arr, &count2); + /* if we came here from iteration code there may be fewer + pattern_values */ + i_assert(count2 <= count); + + if (count2 == 0 && !add_username) { + /* we want everything */ + return 0; + } + + str_append(query, " WHERE"); + exact_count = count == count2 && recurse_type != SQL_DICT_RECURSE_NONE ? + count2-1 : count2; + if (exact_count != array_count(values_arr)) { + *error_r = t_strdup_printf("Key continues past the matched pattern %s", map->pattern); + return -1; + } + + for (i = 0; i < exact_count; i++) { + if (i > 0) + str_append(query, " AND"); + str_printfa(query, " %s = ?", pattern_fields[i].name); + if (sql_dict_field_get_value(map, &pattern_fields[i], + pattern_values[i], "", + params, error_r) < 0) + return -1; + } + switch (recurse_type) { + case SQL_DICT_RECURSE_NONE: + break; + case SQL_DICT_RECURSE_ONE: + if (i > 0) + str_append(query, " AND"); + if (i < count2) { + str_printfa(query, " %s LIKE ?", pattern_fields[i].name); + if (sql_dict_field_get_value(map, &pattern_fields[i], + pattern_values[i], "/%", + params, error_r) < 0) + return -1; + str_printfa(query, " AND %s NOT LIKE ?", pattern_fields[i].name); + if (sql_dict_field_get_value(map, &pattern_fields[i], + pattern_values[i], "/%/%", + params, error_r) < 0) + return -1; + } else { + str_printfa(query, " %s LIKE '%%' AND " + "%s NOT LIKE '%%/%%'", + pattern_fields[i].name, + pattern_fields[i].name); + } + break; + case SQL_DICT_RECURSE_FULL: + if (i < count2) { + if (i > 0) + str_append(query, " AND"); + str_printfa(query, " %s LIKE ", + pattern_fields[i].name); + if (sql_dict_field_get_value(map, &pattern_fields[i], + pattern_values[i], "/%", + params, error_r) < 0) + return -1; + } + break; + } + if (add_username) { + struct sql_dict_param *param = array_append_space(params); + if (count2 > 0) + str_append(query, " AND"); + str_printfa(query, " %s = ?", map->username_field); + param->value_type = DICT_SQL_TYPE_STRING; + param->value_str = t_strdup(username); + } + return 0; +} + +static int +sql_lookup_get_query(struct sql_dict *dict, + const struct dict_op_settings *set, + const char *key, + const struct dict_sql_map **map_r, + struct sql_statement **stmt_r, + const char **error_r) +{ + const struct dict_sql_map *map; + ARRAY_TYPE(const_string) pattern_values; + const char *error; + + map = *map_r = sql_dict_find_map(dict, key, &pattern_values); + if (map == NULL) { + *error_r = t_strdup_printf( + "sql dict lookup: Invalid/unmapped key: %s", key); + return -1; + } + + string_t *query = t_str_new(256); + ARRAY_TYPE(sql_dict_param) params; + t_array_init(¶ms, 4); + str_printfa(query, "SELECT %s FROM %s", + map->value_field, map->table); + if (sql_dict_where_build(set->username, map, &pattern_values, + key[0] == DICT_PATH_PRIVATE[0], + SQL_DICT_RECURSE_NONE, query, + ¶ms, &error) < 0) { + *error_r = t_strdup_printf( + "sql dict lookup: Failed to lookup key %s: %s", key, error); + return -1; + } + *stmt_r = sql_dict_statement_init(dict, str_c(query), ¶ms); + return 0; +} + +static const char * +sql_dict_result_unescape(enum dict_sql_type type, pool_t pool, + struct sql_result *result, unsigned int result_idx) +{ + const unsigned char *data; + size_t size; + const char *value; + string_t *str; + + switch (type) { + case DICT_SQL_TYPE_STRING: + case DICT_SQL_TYPE_INT: + case DICT_SQL_TYPE_UINT: + value = sql_result_get_field_value(result, result_idx); + return value == NULL ? "" : p_strdup(pool, value); + case DICT_SQL_TYPE_HEXBLOB: + break; + } + + data = sql_result_get_field_value_binary(result, result_idx, &size); + str = str_new(pool, size*2 + 1); + binary_to_hex_append(str, data, size); + return str_c(str); +} + +static const char * +sql_dict_result_unescape_value(const struct dict_sql_map *map, pool_t pool, + struct sql_result *result) +{ + return sql_dict_result_unescape(map->value_types[0], pool, result, 0); +} + +static const char *const * +sql_dict_result_unescape_values(const struct dict_sql_map *map, pool_t pool, + struct sql_result *result) +{ + const char **values; + unsigned int i; + + values = p_new(pool, const char *, map->values_count + 1); + for (i = 0; i < map->values_count; i++) { + values[i] = sql_dict_result_unescape(map->value_types[i], + pool, result, i); + } + return values; +} + +static const char * +sql_dict_result_unescape_field(const struct dict_sql_map *map, pool_t pool, + struct sql_result *result, unsigned int result_idx, + unsigned int sql_field_idx) +{ + const struct dict_sql_field *sql_field; + + sql_field = array_idx(&map->pattern_fields, sql_field_idx); + return sql_dict_result_unescape(sql_field->value_type, pool, + result, result_idx); +} + +static int sql_dict_lookup(struct dict *_dict, const struct dict_op_settings *set, + pool_t pool, const char *key, + const char **value_r, const char **error_r) +{ + struct sql_dict *dict = (struct sql_dict *)_dict; + const struct dict_sql_map *map; + struct sql_statement *stmt; + struct sql_result *result = NULL; + int ret; + + *value_r = NULL; + + if (sql_lookup_get_query(dict, set, key, &map, &stmt, error_r) < 0) + return -1; + + result = sql_statement_query_s(&stmt); + ret = sql_result_next_row(result); + if (ret < 0) { + *error_r = t_strdup_printf("dict sql lookup failed: %s", + sql_result_get_error(result)); + } else if (ret > 0) { + *value_r = sql_dict_result_unescape_value(map, pool, result); + } + + sql_result_unref(result); + return ret; +} + +struct sql_dict_lookup_context { + const struct dict_sql_map *map; + dict_lookup_callback_t *callback; + void *context; +}; + +static void +sql_dict_lookup_async_callback(struct sql_result *sql_result, + struct sql_dict_lookup_context *ctx) +{ + struct dict_lookup_result result; + + i_zero(&result); + result.ret = sql_result_next_row(sql_result); + if (result.ret < 0) + result.error = sql_result_get_error(sql_result); + else if (result.ret > 0) { + result.values = sql_dict_result_unescape_values(ctx->map, + pool_datastack_create(), sql_result); + result.value = result.values[0]; + if (result.value == NULL) { + /* NULL value returned. we'll treat this as + "not found", which is probably what is usually + wanted. */ + result.ret = 0; + } + } + ctx->callback(&result, ctx->context); + + i_free(ctx); +} + +static void +sql_dict_lookup_async(struct dict *_dict, + const struct dict_op_settings *set, + const char *key, + dict_lookup_callback_t *callback, void *context) +{ + struct sql_dict *dict = (struct sql_dict *)_dict; + const struct dict_sql_map *map; + struct sql_dict_lookup_context *ctx; + struct sql_statement *stmt; + const char *error; + + if (sql_lookup_get_query(dict, set, key, &map, &stmt, &error) < 0) { + struct dict_lookup_result result; + + i_zero(&result); + result.ret = -1; + result.error = error; + callback(&result, context); + } else { + ctx = i_new(struct sql_dict_lookup_context, 1); + ctx->callback = callback; + ctx->context = context; + ctx->map = map; + sql_statement_query(&stmt, sql_dict_lookup_async_callback, ctx); + } +} + +static const struct dict_sql_map * +sql_dict_iterate_find_next_map(struct sql_dict_iterate_context *ctx, + ARRAY_TYPE(const_string) *pattern_values) +{ + struct sql_dict *dict = (struct sql_dict *)ctx->ctx.dict; + const struct dict_sql_map *maps; + unsigned int i, count; + size_t pat_len, path_len; + bool recurse = (ctx->flags & DICT_ITERATE_FLAG_RECURSE) != 0; + + t_array_init(pattern_values, dict->set->max_pattern_fields_count); + maps = array_get(&dict->set->maps, &count); + for (i = ctx->next_map_idx; i < count; i++) { + if (dict_sql_map_match(&maps[i], ctx->path, + pattern_values, &pat_len, &path_len, + TRUE, recurse) && + (recurse || + array_count(pattern_values)+1 >= array_count(&maps[i].pattern_fields))) { + ctx->key_prefix_len = path_len; + ctx->pattern_prefix_len = pat_len; + ctx->next_map_idx = i + 1; + + str_truncate(ctx->key, 0); + str_append(ctx->key, ctx->path); + return &maps[i]; + } + } + return NULL; +} + +static int +sql_dict_iterate_build_next_query(struct sql_dict_iterate_context *ctx, + struct sql_statement **stmt_r, + const char **error_r) +{ + struct sql_dict *dict = (struct sql_dict *)ctx->ctx.dict; + const struct dict_op_settings_private *set = &ctx->ctx.set; + const struct dict_sql_map *map; + ARRAY_TYPE(const_string) pattern_values; + const struct dict_sql_field *pattern_fields; + enum sql_recurse_type recurse_type; + unsigned int i, count; + + map = sql_dict_iterate_find_next_map(ctx, &pattern_values); + /* NULL map is allowed if we have already done some lookups */ + if (map == NULL) { + if (!ctx->allow_null_map) { + *error_r = "Invalid/unmapped path"; + return -1; + } + return 0; + } + + if (ctx->result != NULL) { + sql_result_unref(ctx->result); + ctx->result = NULL; + } + + string_t *query = t_str_new(256); + str_append(query, "SELECT "); + if ((ctx->flags & DICT_ITERATE_FLAG_NO_VALUE) == 0) + str_printfa(query, "%s,", map->value_field); + + /* get all missing fields */ + pattern_fields = array_get(&map->pattern_fields, &count); + i = array_count(&pattern_values); + if (i == count) { + /* we always want to know the last field since we're + iterating its children */ + i_assert(i > 0); + i--; + } + ctx->sql_fields_start_idx = i; + for (; i < count; i++) + str_printfa(query, "%s,", pattern_fields[i].name); + str_truncate(query, str_len(query)-1); + + str_printfa(query, " FROM %s", map->table); + + if ((ctx->flags & DICT_ITERATE_FLAG_RECURSE) != 0) + recurse_type = SQL_DICT_RECURSE_FULL; + else if ((ctx->flags & DICT_ITERATE_FLAG_EXACT_KEY) != 0) + recurse_type = SQL_DICT_RECURSE_NONE; + else + recurse_type = SQL_DICT_RECURSE_ONE; + + ARRAY_TYPE(sql_dict_param) params; + t_array_init(¶ms, 4); + bool add_username = (ctx->path[0] == DICT_PATH_PRIVATE[0]); + if (sql_dict_where_build(set->username, map, &pattern_values, add_username, + recurse_type, query, ¶ms, error_r) < 0) + return -1; + + if ((ctx->flags & DICT_ITERATE_FLAG_SORT_BY_KEY) != 0) { + str_append(query, " ORDER BY "); + for (i = 0; i < count; i++) { + str_printfa(query, "%s", pattern_fields[i].name); + if (i < count-1) + str_append_c(query, ','); + } + } else if ((ctx->flags & DICT_ITERATE_FLAG_SORT_BY_VALUE) != 0) + str_printfa(query, " ORDER BY %s", map->value_field); + + if (ctx->ctx.max_rows > 0) { + i_assert(ctx->ctx.row_count < ctx->ctx.max_rows); + str_printfa(query, " LIMIT %"PRIu64, + ctx->ctx.max_rows - ctx->ctx.row_count); + } + + *stmt_r = sql_dict_statement_init(dict, str_c(query), ¶ms); + ctx->map = map; + return 1; +} + +static void sql_dict_iterate_callback(struct sql_result *result, + struct sql_dict_iterate_context *ctx) +{ + if (!ctx->destroyed) { + sql_result_ref(result); + ctx->result = result; + if (ctx->ctx.async_callback != NULL && !ctx->synchronous_result) + ctx->ctx.async_callback(ctx->ctx.async_context); + } + + pool_t pool_copy = ctx->pool; + pool_unref(&pool_copy); +} + +static int sql_dict_iterate_next_query(struct sql_dict_iterate_context *ctx) +{ + struct sql_statement *stmt; + const char *error; + int ret; + + ret = sql_dict_iterate_build_next_query(ctx, &stmt, &error); + if (ret <= 0) { + /* this is expected error */ + if (ret == 0) + return ret; + /* failed */ + ctx->error = p_strdup_printf(ctx->pool, + "sql dict iterate failed for %s: %s", + ctx->path, error); + return -1; + } + + if ((ctx->flags & DICT_ITERATE_FLAG_ASYNC) == 0) { + ctx->result = sql_statement_query_s(&stmt); + } else { + i_assert(ctx->result == NULL); + ctx->synchronous_result = TRUE; + pool_ref(ctx->pool); + sql_statement_query(&stmt, sql_dict_iterate_callback, ctx); + ctx->synchronous_result = FALSE; + } + return ret; +} + +static struct dict_iterate_context * +sql_dict_iterate_init(struct dict *_dict, + const struct dict_op_settings *set ATTR_UNUSED, + const char *path, enum dict_iterate_flags flags) +{ + struct sql_dict_iterate_context *ctx; + pool_t pool; + + pool = pool_alloconly_create("sql dict iterate", 512); + ctx = p_new(pool, struct sql_dict_iterate_context, 1); + ctx->ctx.dict = _dict; + ctx->pool = pool; + ctx->flags = flags; + + ctx->path = p_strdup(pool, path); + + ctx->key = str_new(pool, 256); + return &ctx->ctx; +} + +static bool sql_dict_iterate(struct dict_iterate_context *_ctx, + const char **key_r, const char *const **values_r) +{ + struct sql_dict_iterate_context *ctx = + (struct sql_dict_iterate_context *)_ctx; + const char *p, *value; + unsigned int i, sql_field_i, count; + int ret; + + _ctx->has_more = FALSE; + if (ctx->error != NULL) + return FALSE; + if (!ctx->iter_query_sent) { + ctx->iter_query_sent = TRUE; + if (sql_dict_iterate_next_query(ctx) <= 0) + return FALSE; + } + + if (ctx->result == NULL) { + /* wait for async lookup to finish */ + i_assert((ctx->flags & DICT_ITERATE_FLAG_ASYNC) != 0); + _ctx->has_more = TRUE; + return FALSE; + } + + ret = sql_result_next_row(ctx->result); + while (ret == SQL_RESULT_NEXT_MORE) { + if ((ctx->flags & DICT_ITERATE_FLAG_ASYNC) == 0) + sql_result_more_s(&ctx->result); + else { + /* get more results asynchronously */ + ctx->synchronous_result = TRUE; + pool_ref(ctx->pool); + sql_result_more(&ctx->result, sql_dict_iterate_callback, ctx); + ctx->synchronous_result = FALSE; + if (ctx->result == NULL) { + _ctx->has_more = TRUE; + return FALSE; + } + } + ret = sql_result_next_row(ctx->result); + } + if (ret == 0) { + /* see if there are more results in the next map. + don't do it if we're looking for an exact match, since we + already should have handled it. */ + if ((ctx->flags & DICT_ITERATE_FLAG_EXACT_KEY) != 0) + return FALSE; + ctx->iter_query_sent = FALSE; + /* we have gotten *SOME* results, so can allow + unmapped next key now. */ + ctx->allow_null_map = TRUE; + return sql_dict_iterate(_ctx, key_r, values_r); + } + if (ret < 0) { + ctx->error = p_strdup_printf(ctx->pool, + "dict sql iterate failed: %s", + sql_result_get_error(ctx->result)); + return FALSE; + } + + /* convert fetched row to dict key */ + str_truncate(ctx->key, ctx->key_prefix_len); + if (ctx->key_prefix_len > 0 && + str_c(ctx->key)[ctx->key_prefix_len-1] != '/') + str_append_c(ctx->key, '/'); + + count = sql_result_get_fields_count(ctx->result); + i = (ctx->flags & DICT_ITERATE_FLAG_NO_VALUE) != 0 ? 0 : + ctx->map->values_count; + sql_field_i = ctx->sql_fields_start_idx; + for (p = ctx->map->pattern + ctx->pattern_prefix_len; *p != '\0'; p++) { + if (*p != '$') + str_append_c(ctx->key, *p); + else { + i_assert(i < count); + value = sql_dict_result_unescape_field(ctx->map, + pool_datastack_create(), ctx->result, i, sql_field_i); + if (value != NULL) + str_append(ctx->key, value); + i++; sql_field_i++; + } + } + + *key_r = str_c(ctx->key); + if ((ctx->flags & DICT_ITERATE_FLAG_NO_VALUE) == 0) { + *values_r = sql_dict_result_unescape_values(ctx->map, + pool_datastack_create(), ctx->result); + } + return TRUE; +} + +static int sql_dict_iterate_deinit(struct dict_iterate_context *_ctx, + const char **error_r) +{ + struct sql_dict_iterate_context *ctx = + (struct sql_dict_iterate_context *)_ctx; + int ret = ctx->error != NULL ? -1 : 0; + + *error_r = t_strdup(ctx->error); + if (ctx->result != NULL) + sql_result_unref(ctx->result); + ctx->destroyed = TRUE; + + pool_t pool_copy = ctx->pool; + pool_unref(&pool_copy); + return ret; +} + +static struct dict_transaction_context * +sql_dict_transaction_init(struct dict *_dict) +{ + struct sql_dict *dict = (struct sql_dict *)_dict; + struct sql_dict_transaction_context *ctx; + + ctx = i_new(struct sql_dict_transaction_context, 1); + ctx->ctx.dict = _dict; + ctx->sql_ctx = sql_transaction_begin(dict->db); + + return &ctx->ctx; +} + +static void sql_dict_transaction_free(struct sql_dict_transaction_context *ctx) +{ + if (array_is_created(&ctx->prev_inc)) + sql_dict_prev_inc_free(ctx); + if (array_is_created(&ctx->prev_set)) + sql_dict_prev_set_free(ctx); + + pool_unref(&ctx->inc_row_pool); + i_free(ctx->error); + i_free(ctx); +} + +static bool +sql_dict_transaction_has_nonexistent(struct sql_dict_transaction_context *ctx) +{ + struct sql_dict_inc_row *inc_row; + + for (inc_row = ctx->inc_row; inc_row != NULL; inc_row = inc_row->prev) { + i_assert(inc_row->rows != UINT_MAX); + if (inc_row->rows == 0) + return TRUE; + } + return FALSE; +} + +static void +sql_dict_transaction_commit_callback(const struct sql_commit_result *sql_result, + struct sql_dict_transaction_context *ctx) +{ + struct dict_commit_result result; + + i_zero(&result); + if (sql_result->error == NULL) + result.ret = sql_dict_transaction_has_nonexistent(ctx) ? + DICT_COMMIT_RET_NOTFOUND : DICT_COMMIT_RET_OK; + else { + result.error = t_strdup_printf("sql dict: commit failed: %s", + sql_result->error); + switch (sql_result->error_type) { + case SQL_RESULT_ERROR_TYPE_UNKNOWN: + default: + result.ret = DICT_COMMIT_RET_FAILED; + break; + case SQL_RESULT_ERROR_TYPE_WRITE_UNCERTAIN: + result.ret = DICT_COMMIT_RET_WRITE_UNCERTAIN; + break; + } + } + + if (ctx->async_callback != NULL) + ctx->async_callback(&result, ctx->async_context); + else if (result.ret < 0) + i_error("%s", result.error); + sql_dict_transaction_free(ctx); +} + +static void +sql_dict_transaction_commit(struct dict_transaction_context *_ctx, bool async, + dict_transaction_commit_callback_t *callback, + void *context) +{ + struct sql_dict_transaction_context *ctx = + (struct sql_dict_transaction_context *)_ctx; + const char *error; + struct dict_commit_result result; + + /* flush any pending set/inc */ + if (array_is_created(&ctx->prev_inc)) + sql_dict_prev_inc_flush(ctx); + if (array_is_created(&ctx->prev_set)) + sql_dict_prev_set_flush(ctx); + + /* note that the above calls might still set ctx->error */ + i_zero(&result); + result.ret = DICT_COMMIT_RET_FAILED; + result.error = t_strdup(ctx->error); + + if (ctx->error != NULL) { + sql_transaction_rollback(&ctx->sql_ctx); + } else if (!_ctx->changed) { + /* nothing changed, no need to commit */ + sql_transaction_rollback(&ctx->sql_ctx); + result.ret = DICT_COMMIT_RET_OK; + } else if (async) { + ctx->async_callback = callback; + ctx->async_context = context; + sql_transaction_commit(&ctx->sql_ctx, + sql_dict_transaction_commit_callback, ctx); + return; + } else if (sql_transaction_commit_s(&ctx->sql_ctx, &error) < 0) { + result.error = t_strdup_printf( + "sql dict: commit failed: %s", error); + } else { + if (sql_dict_transaction_has_nonexistent(ctx)) + result.ret = DICT_COMMIT_RET_NOTFOUND; + else + result.ret = DICT_COMMIT_RET_OK; + } + sql_dict_transaction_free(ctx); + + callback(&result, context); +} + +static void sql_dict_transaction_rollback(struct dict_transaction_context *_ctx) +{ + struct sql_dict_transaction_context *ctx = + (struct sql_dict_transaction_context *)_ctx; + + sql_transaction_rollback(&ctx->sql_ctx); + sql_dict_transaction_free(ctx); +} + +static struct sql_statement * +sql_dict_transaction_stmt_init(struct sql_dict_transaction_context *ctx, + const char *query, + const ARRAY_TYPE(sql_dict_param) *params) +{ + struct sql_dict *dict = (struct sql_dict *)ctx->ctx.dict; + struct sql_statement *stmt = + sql_dict_statement_init(dict, query, params); + + if (ctx->ctx.timestamp.tv_sec != 0) + sql_statement_set_timestamp(stmt, &ctx->ctx.timestamp); + return stmt; +} + +struct dict_sql_build_query_field { + const struct dict_sql_map *map; + const char *value; +}; + +struct dict_sql_build_query { + struct sql_dict *dict; + + ARRAY(struct dict_sql_build_query_field) fields; + const ARRAY_TYPE(const_string) *pattern_values; + bool add_username; +}; + +static int sql_dict_set_query(struct sql_dict_transaction_context *ctx, + const struct dict_sql_build_query *build, + struct sql_statement **stmt_r, + const char **error_r) +{ + struct sql_dict *dict = build->dict; + const struct dict_sql_build_query_field *fields; + const struct dict_sql_field *pattern_fields; + ARRAY_TYPE(sql_dict_param) params; + const char *const *pattern_values; + unsigned int i, field_count, count, count2; + string_t *prefix, *suffix; + + fields = array_get(&build->fields, &field_count); + i_assert(field_count > 0); + + t_array_init(¶ms, 4); + prefix = t_str_new(64); + suffix = t_str_new(256); + /* SQL table is guaranteed to be the same for all fields. + Build all the SQL field names into prefix and '?' placeholders for + each value into the suffix. The actual field values will be added + into params[]. */ + str_printfa(prefix, "INSERT INTO %s", fields[0].map->table); + str_append(prefix, " ("); + str_append(suffix, ") VALUES ("); + for (i = 0; i < field_count; i++) { + if (i > 0) { + str_append_c(prefix, ','); + str_append_c(suffix, ','); + } + str_append(prefix, t_strcut(fields[i].map->value_field, ',')); + + enum dict_sql_type value_type = + fields[i].map->value_types[0]; + str_append_c(suffix, '?'); + if (sql_dict_value_get(fields[i].map, + value_type, "value", fields[i].value, + "", ¶ms, error_r) < 0) + return -1; + } + if (build->add_username) { + struct sql_dict_param *param = array_append_space(¶ms); + str_printfa(prefix, ",%s", fields[0].map->username_field); + str_append(suffix, ",?"); + param->value_type = DICT_SQL_TYPE_STRING; + param->value_str = ctx->ctx.set.username; + } + + /* add the variable fields that were parsed from the path */ + pattern_fields = array_get(&fields[0].map->pattern_fields, &count); + pattern_values = array_get(build->pattern_values, &count2); + i_assert(count == count2); + for (i = 0; i < count; i++) { + str_printfa(prefix, ",%s", pattern_fields[i].name); + str_append(suffix, ",?"); + if (sql_dict_field_get_value(fields[0].map, &pattern_fields[i], + pattern_values[i], "", + ¶ms, error_r) < 0) + return -1; + } + + str_append_str(prefix, suffix); + str_append_c(prefix, ')'); + + enum sql_db_flags flags = sql_get_flags(dict->db); + if ((flags & SQL_DB_FLAG_ON_DUPLICATE_KEY) != 0) + str_append(prefix, " ON DUPLICATE KEY UPDATE "); + else if ((flags & SQL_DB_FLAG_ON_CONFLICT_DO) != 0) { + str_append(prefix, " ON CONFLICT ("); + for (i = 0; i < count; i++) { + if (i > 0) + str_append_c(prefix, ','); + str_append(prefix, pattern_fields[i].name); + } + if (build->add_username) { + if (count > 0) + str_append_c(prefix, ','); + str_append(prefix, fields[0].map->username_field); + } + str_append(prefix, ") DO UPDATE SET "); + } else { + *stmt_r = sql_dict_transaction_stmt_init(ctx, str_c(prefix), ¶ms); + return 0; + } + + /* If the row already exists, UPDATE it instead. The pattern_values + don't need to be updated here, because they are expected to be part + of the row's primary key. */ + for (i = 0; i < field_count; i++) { + const char *first_value_field = + t_strcut(fields[i].map->value_field, ','); + if (i > 0) + str_append_c(prefix, ','); + str_append(prefix, first_value_field); + str_append_c(prefix, '='); + + enum dict_sql_type value_type = + fields[i].map->value_types[0]; + str_append_c(prefix, '?'); + if (sql_dict_value_get(fields[i].map, + value_type, "value", fields[i].value, + "", ¶ms, error_r) < 0) + return -1; + } + *stmt_r = sql_dict_transaction_stmt_init(ctx, str_c(prefix), ¶ms); + return 0; +} + +static int +sql_dict_update_query(const struct dict_sql_build_query *build, + const struct dict_op_settings_private *set, + const char **query_r, ARRAY_TYPE(sql_dict_param) *params, + const char **error_r) +{ + const struct dict_sql_build_query_field *fields; + unsigned int i, field_count; + string_t *query; + + fields = array_get(&build->fields, &field_count); + i_assert(field_count > 0); + + query = t_str_new(64); + str_printfa(query, "UPDATE %s SET ", fields[0].map->table); + for (i = 0; i < field_count; i++) { + const char *first_value_field = + t_strcut(fields[i].map->value_field, ','); + if (i > 0) + str_append_c(query, ','); + str_printfa(query, "%s=%s+?", first_value_field, + first_value_field); + } + + if (sql_dict_where_build(set->username, fields[0].map, build->pattern_values, + build->add_username, SQL_DICT_RECURSE_NONE, + query, params, error_r) < 0) + return -1; + *query_r = str_c(query); + return 0; +} + +static void sql_dict_prev_set_free(struct sql_dict_transaction_context *ctx) +{ + struct sql_dict_prev *prev_set; + + array_foreach_modifiable(&ctx->prev_set, prev_set) { + i_free(prev_set->value.str); + i_free(prev_set->key); + } + array_free(&ctx->prev_set); +} + +static void sql_dict_prev_set_flush(struct sql_dict_transaction_context *ctx) +{ + struct sql_dict *dict = (struct sql_dict *)ctx->ctx.dict; + const struct sql_dict_prev *prev_sets; + unsigned int count; + struct sql_statement *stmt; + ARRAY_TYPE(const_string) pattern_values; + struct dict_sql_build_query build; + struct dict_sql_build_query_field *field; + const char *error; + + i_assert(array_is_created(&ctx->prev_set)); + + if (ctx->error != NULL) { + sql_dict_prev_set_free(ctx); + return; + } + + prev_sets = array_get(&ctx->prev_set, &count); + i_assert(count > 0); + + /* Get the variable values from the dict path. We already verified that + these are all exactly the same for everything in prev_sets. */ + if (sql_dict_find_map(dict, prev_sets[0].key, &pattern_values) == NULL) + i_unreached(); /* this was already checked */ + + i_zero(&build); + build.dict = dict; + build.pattern_values = &pattern_values; + build.add_username = (prev_sets[0].key[0] == DICT_PATH_PRIVATE[0]); + + /* build.fields[] is used to get the map { value_field } for the + SQL field names, as well as the values for them. + + Example: INSERT INTO ... (build.fields[0].map->value_field, + ...[1], ...) VALUES (build.fields[0].value, ...[1], ...) */ + t_array_init(&build.fields, count); + for (unsigned int i = 0; i < count; i++) { + i_assert(build.add_username == + (prev_sets[i].key[0] == DICT_PATH_PRIVATE[0])); + field = array_append_space(&build.fields); + field->map = prev_sets[i].map; + field->value = prev_sets[i].value.str; + } + + if (sql_dict_set_query(ctx, &build, &stmt, &error) < 0) { + ctx->error = i_strdup_printf( + "dict-sql: Failed to set %u fields (first %s): %s", + count, prev_sets[0].key, error); + } else { + sql_update_stmt(ctx->sql_ctx, &stmt); + } + sql_dict_prev_set_free(ctx); +} + +static void sql_dict_unset(struct dict_transaction_context *_ctx, + const char *key) +{ + struct sql_dict_transaction_context *ctx = + (struct sql_dict_transaction_context *)_ctx; + struct sql_dict *dict = (struct sql_dict *)_ctx->dict; + const struct dict_op_settings_private *set = &_ctx->set; + const struct dict_sql_map *map; + ARRAY_TYPE(const_string) pattern_values; + string_t *query = t_str_new(256); + ARRAY_TYPE(sql_dict_param) params; + const char *error; + + if (ctx->error != NULL) + return; + + /* In theory we could unset one of the previous set/incs in this + same transaction, so flush them first. */ + if (array_is_created(&ctx->prev_inc)) + sql_dict_prev_inc_flush(ctx); + if (array_is_created(&ctx->prev_set)) + sql_dict_prev_set_flush(ctx); + + map = sql_dict_find_map(dict, key, &pattern_values); + if (map == NULL) { + ctx->error = i_strdup_printf("dict-sql: Invalid/unmapped key: %s", key); + return; + } + + str_printfa(query, "DELETE FROM %s", map->table); + t_array_init(¶ms, 4); + if (sql_dict_where_build(set->username, map, &pattern_values, + key[0] == DICT_PATH_PRIVATE[0], + SQL_DICT_RECURSE_NONE, query, + ¶ms, &error) < 0) { + ctx->error = i_strdup_printf( + "dict-sql: Failed to delete %s: %s", key, error); + } else { + struct sql_statement *stmt = + sql_dict_transaction_stmt_init(ctx, str_c(query), ¶ms); + sql_update_stmt(ctx->sql_ctx, &stmt); + } +} + +static unsigned int * +sql_dict_next_inc_row(struct sql_dict_transaction_context *ctx) +{ + struct sql_dict_inc_row *row; + + if (ctx->inc_row_pool == NULL) { + ctx->inc_row_pool = + pool_alloconly_create("sql dict inc rows", 128); + } + row = p_new(ctx->inc_row_pool, struct sql_dict_inc_row, 1); + row->prev = ctx->inc_row; + row->rows = UINT_MAX; + ctx->inc_row = row; + return &row->rows; +} + +static void sql_dict_prev_inc_free(struct sql_dict_transaction_context *ctx) +{ + struct sql_dict_prev *prev_inc; + + array_foreach_modifiable(&ctx->prev_inc, prev_inc) + i_free(prev_inc->key); + array_free(&ctx->prev_inc); +} + +static void sql_dict_prev_inc_flush(struct sql_dict_transaction_context *ctx) +{ + struct sql_dict *dict = (struct sql_dict *)ctx->ctx.dict; + const struct dict_op_settings_private *set = &ctx->ctx.set; + const struct sql_dict_prev *prev_incs; + unsigned int count; + ARRAY_TYPE(const_string) pattern_values; + struct dict_sql_build_query build; + struct dict_sql_build_query_field *field; + ARRAY_TYPE(sql_dict_param) params; + struct sql_dict_param *param; + const char *query, *error; + + i_assert(array_is_created(&ctx->prev_inc)); + + if (ctx->error != NULL) { + sql_dict_prev_inc_free(ctx); + return; + } + + prev_incs = array_get(&ctx->prev_inc, &count); + i_assert(count > 0); + + /* Get the variable values from the dict path. We already verified that + these are all exactly the same for everything in prev_incs. */ + if (sql_dict_find_map(dict, prev_incs[0].key, &pattern_values) == NULL) + i_unreached(); /* this was already checked */ + + i_zero(&build); + build.dict = dict; + build.pattern_values = &pattern_values; + build.add_username = (prev_incs[0].key[0] == DICT_PATH_PRIVATE[0]); + + /* build.fields[] is an array of maps, which are used to get the + map { value_field } for the SQL field names. + + params[] specifies the list of values to use for each field. + + Example: UPDATE .. SET build.fields[0].map->value_field = + ...->value_field + params[0]->value_int64, ...[1]... */ + t_array_init(&build.fields, count); + t_array_init(¶ms, count); + for (unsigned int i = 0; i < count; i++) { + i_assert(build.add_username == + (prev_incs[i].key[0] == DICT_PATH_PRIVATE[0])); + field = array_append_space(&build.fields); + field->map = prev_incs[i].map; + field->value = NULL; /* unused */ + + param = array_append_space(¶ms); + param->value_type = DICT_SQL_TYPE_INT; + param->value_int64 = prev_incs[i].value.diff; + } + + if (sql_dict_update_query(&build, set, &query, ¶ms, &error) < 0) { + ctx->error = i_strdup_printf( + "dict-sql: Failed to increase %u fields (first %s): %s", + count, prev_incs[0].key, error); + } else { + struct sql_statement *stmt = + sql_dict_transaction_stmt_init(ctx, query, ¶ms); + sql_update_stmt_get_rows(ctx->sql_ctx, &stmt, + sql_dict_next_inc_row(ctx)); + } + sql_dict_prev_inc_free(ctx); +} + +static bool +sql_dict_maps_are_mergeable(struct sql_dict *dict, + const struct sql_dict_prev *prev1, + const struct dict_sql_map *map2, + const char *map2_key, + const ARRAY_TYPE(const_string) *map2_pattern_values) +{ + const struct dict_sql_map *map3; + ARRAY_TYPE(const_string) map1_pattern_values; + + /* sql table names must equal */ + if (strcmp(prev1->map->table, map2->table) != 0) + return FALSE; + /* private vs shared prefix must equal */ + if (prev1->key[0] != map2_key[0]) + return FALSE; + if (prev1->key[0] == DICT_PATH_PRIVATE[0]) { + /* for private keys, username must equal */ + if (strcmp(prev1->map->username_field, map2->username_field) != 0) + return FALSE; + } + + /* variable values in the paths must equal exactly */ + map3 = sql_dict_find_map(dict, prev1->key, &map1_pattern_values); + i_assert(map3 == prev1->map); + + return array_equal_fn(&map1_pattern_values, map2_pattern_values, + i_strcmp_p); +} + +static void sql_dict_set(struct dict_transaction_context *_ctx, + const char *key, const char *value) +{ + struct sql_dict_transaction_context *ctx = + (struct sql_dict_transaction_context *)_ctx; + struct sql_dict *dict = (struct sql_dict *)_ctx->dict; + const struct dict_sql_map *map; + ARRAY_TYPE(const_string) pattern_values; + + if (ctx->error != NULL) + return; + + /* In theory we could set the previous inc in this same transaction, + so flush it first. */ + if (array_is_created(&ctx->prev_inc)) + sql_dict_prev_inc_flush(ctx); + + map = sql_dict_find_map(dict, key, &pattern_values); + if (map == NULL) { + ctx->error = i_strdup_printf( + "sql dict set: Invalid/unmapped key: %s", key); + return; + } + + if (array_is_created(&ctx->prev_set) && + !sql_dict_maps_are_mergeable(dict, array_front(&ctx->prev_set), + map, key, &pattern_values)) { + /* couldn't merge to the previous set - flush it */ + sql_dict_prev_set_flush(ctx); + } + + if (!array_is_created(&ctx->prev_set)) + i_array_init(&ctx->prev_set, 4); + /* Either this is the first set, or this can be merged with the + previous set. */ + struct sql_dict_prev *prev_set = array_append_space(&ctx->prev_set); + prev_set->map = map; + prev_set->key = i_strdup(key); + prev_set->value.str = i_strdup(value); +} + +static void sql_dict_atomic_inc(struct dict_transaction_context *_ctx, + const char *key, long long diff) +{ + struct sql_dict_transaction_context *ctx = + (struct sql_dict_transaction_context *)_ctx; + struct sql_dict *dict = (struct sql_dict *)_ctx->dict; + const struct dict_sql_map *map; + ARRAY_TYPE(const_string) pattern_values; + + if (ctx->error != NULL) + return; + + /* In theory we could inc the previous set in this same transaction, + so flush it first. */ + if (array_is_created(&ctx->prev_set)) + sql_dict_prev_set_flush(ctx); + + map = sql_dict_find_map(dict, key, &pattern_values); + if (map == NULL) { + ctx->error = i_strdup_printf( + "sql dict atomic inc: Invalid/unmapped key: %s", key); + return; + } + + if (array_is_created(&ctx->prev_inc) && + !sql_dict_maps_are_mergeable(dict, array_front(&ctx->prev_inc), + map, key, &pattern_values)) { + /* couldn't merge to the previous inc - flush it */ + sql_dict_prev_inc_flush(ctx); + } + + if (!array_is_created(&ctx->prev_inc)) + i_array_init(&ctx->prev_inc, 4); + /* Either this is the first inc, or this can be merged with the + previous inc. */ + struct sql_dict_prev *prev_inc = array_append_space(&ctx->prev_inc); + prev_inc->map = map; + prev_inc->key = i_strdup(key); + prev_inc->value.diff = diff; +} + +static struct dict sql_dict = { + .name = "sql", + + { + .init = sql_dict_init, + .deinit = sql_dict_deinit, + .wait = sql_dict_wait, + .lookup = sql_dict_lookup, + .iterate_init = sql_dict_iterate_init, + .iterate = sql_dict_iterate, + .iterate_deinit = sql_dict_iterate_deinit, + .transaction_init = sql_dict_transaction_init, + .transaction_commit = sql_dict_transaction_commit, + .transaction_rollback = sql_dict_transaction_rollback, + .set = sql_dict_set, + .unset = sql_dict_unset, + .atomic_inc = sql_dict_atomic_inc, + .lookup_async = sql_dict_lookup_async, + } +}; + +static struct dict *dict_sql_drivers; + +void dict_sql_register(void) +{ + const struct sql_db *const *drivers; + unsigned int i, count; + + dict_sql_db_cache = sql_db_cache_init(DICT_SQL_MAX_UNUSED_CONNECTIONS); + + /* @UNSAFE */ + drivers = array_get(&sql_drivers, &count); + dict_sql_drivers = i_new(struct dict, count + 1); + + for (i = 0; i < count; i++) { + dict_sql_drivers[i] = sql_dict; + dict_sql_drivers[i].name = drivers[i]->name; + + dict_driver_register(&dict_sql_drivers[i]); + } +} + +void dict_sql_unregister(void) +{ + int i; + + for (i = 0; dict_sql_drivers[i].name != NULL; i++) + dict_driver_unregister(&dict_sql_drivers[i]); + i_free(dict_sql_drivers); + sql_db_cache_deinit(&dict_sql_db_cache); + dict_sql_settings_deinit(); +} diff --git a/src/lib-dict-backend/dict-sql.h b/src/lib-dict-backend/dict-sql.h new file mode 100644 index 0000000..418a650 --- /dev/null +++ b/src/lib-dict-backend/dict-sql.h @@ -0,0 +1,7 @@ +#ifndef DICT_SQL_H +#define DICT_SQL_H + +void dict_sql_register(void); +void dict_sql_unregister(void); + +#endif diff --git a/src/lib-dict-backend/dict.conf b/src/lib-dict-backend/dict.conf new file mode 100644 index 0000000..04dd63a --- /dev/null +++ b/src/lib-dict-backend/dict.conf @@ -0,0 +1,49 @@ +connect = host=localhost + +map { + pattern = shared/dictmap/$key1/$key2 + table = table + value_field = value + value_type = string + + fields { + a = $key1 + b = $key2 + } +} + +map { + pattern = shared/counters/$class/$name + table = counters + value_field = value + value_type = uint + + fields { + class = $class + name = $name + } +} + +map { + pattern = priv/quota/bytes + table = quota + username_field = username + value_field = bytes + value_type = uint +} + +map { + pattern = priv/quota/count + table = quota + username_field = username + value_field = count + value_type = uint +} + +map { + pattern = priv/quota/folders + table = quota + username_field = username + value_field = folders + value_type = uint +} diff --git a/src/lib-dict-backend/test-dict-sql.c b/src/lib-dict-backend/test-dict-sql.c new file mode 100644 index 0000000..4de45eb --- /dev/null +++ b/src/lib-dict-backend/test-dict-sql.c @@ -0,0 +1,314 @@ +/* Copyright (c) 2017-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "test-lib.h" +#include "sql-api.h" +#include "dict.h" +#include "dict-private.h" +#include "dict-sql.h" +#include "dict-sql-private.h" +#include "driver-test.h" + +struct dict_op_settings dict_op_settings = { + .username = "testuser", +}; + +static void test_setup(struct dict **dict_r) +{ + const char *error = NULL; + struct dict_settings set = { + .base_dir = "." + }; + struct dict *dict = NULL; + + if (dict_init("mysql:" DICT_SRC_DIR "/dict.conf", &set, &dict, &error) < 0) + i_fatal("cannot initialize dict: %s", error); + + *dict_r = dict; +} + +static void test_teardown(struct dict **_dict) +{ + struct dict *dict = *_dict; + *_dict = NULL; + if (dict != NULL) { + dict_deinit(&dict); + } +} + +static void test_set_expected(struct dict *_dict, + const struct test_driver_result *result) +{ + struct sql_dict *dict = + (struct sql_dict *)_dict; + + sql_driver_test_add_expected_result(dict->db, result); +} + +static void test_lookup_one(void) +{ + const char *value = NULL, *error = NULL; + struct test_driver_result_set rset = { + .rows = 1, + .cols = 1, + .col_names = (const char *[]){"value", NULL}, + .row_data = (const char **[]){(const char*[]){"one", NULL}}, + }; + struct test_driver_result res = { + .nqueries = 1, + .queries = (const char *[]){"SELECT value FROM table WHERE a = 'hello' AND b = 'world'", NULL}, + .result = &rset, + }; + const struct dict_op_settings set = { + .username = "testuser", + }; + struct dict *dict; + pool_t pool = pool_datastack_create(); + + test_begin("dict lookup one"); + test_setup(&dict); + + test_set_expected(dict, &res); + + test_assert(dict_lookup(dict, &set, pool, "shared/dictmap/hello/world", &value, &error) == 1); + test_assert_strcmp(value, "one"); + if (error != NULL) + i_error("dict_lookup failed: %s", error); + test_teardown(&dict); + test_end(); +} + +static void test_atomic_inc(void) +{ + const char *error; + struct test_driver_result res = { + .nqueries = 3, + .queries = (const char *[]){ + "UPDATE counters SET value=value+128 WHERE class = 'global' AND name = 'counter'", + "UPDATE quota SET bytes=bytes+128,count=count+1 WHERE username = 'testuser'", + "UPDATE quota SET bytes=bytes+128,count=count+1,folders=folders+123 WHERE username = 'testuser'", + NULL}, + .result = NULL, + }; + struct dict_op_settings set = { + .username = "testuser", + }; + struct dict *dict; + + test_begin("dict atomic inc"); + test_setup(&dict); + + test_set_expected(dict, &res); + + /* 1 field */ + struct dict_transaction_context *ctx = dict_transaction_begin(dict, &set); + dict_atomic_inc(ctx, "shared/counters/global/counter", 128); + test_assert(dict_transaction_commit(&ctx, &error) == 0); + if (error != NULL) + i_error("dict_transaction_commit failed: %s", error); + error = NULL; + + /* 2 fields */ + ctx = dict_transaction_begin(dict, &set); + dict_atomic_inc(ctx, "priv/quota/bytes", 128); + dict_atomic_inc(ctx, "priv/quota/count", 1); + test_assert(dict_transaction_commit(&ctx, &error) == 0); + if (error != NULL) + i_error("dict_transaction_commit failed: %s", error); + error = NULL; + + /* 3 fields */ + ctx = dict_transaction_begin(dict, &set); + dict_atomic_inc(ctx, "priv/quota/bytes", 128); + dict_atomic_inc(ctx, "priv/quota/count", 1); + dict_atomic_inc(ctx, "priv/quota/folders", 123); + test_assert(dict_transaction_commit(&ctx, &error) == 0); + if (error != NULL) + i_error("dict_transaction_commit failed: %s", error); + test_teardown(&dict); + test_end(); +} + +static void test_set(void) +{ + const char *error; + struct test_driver_result res = { + .affected_rows = 1, + .nqueries = 3, + .queries = (const char *[]){ + "INSERT INTO counters (value,class,name) VALUES (128,'global','counter') ON DUPLICATE KEY UPDATE value=128", + "INSERT INTO quota (bytes,count,username) VALUES (128,1,'testuser') ON DUPLICATE KEY UPDATE bytes=128,count=1", + "INSERT INTO quota (bytes,count,folders,username) VALUES (128,1,123,'testuser') ON DUPLICATE KEY UPDATE bytes=128,count=1,folders=123", + NULL}, + .result = NULL, + }; + struct dict *dict; + + test_begin("dict set"); + test_setup(&dict); + + test_set_expected(dict, &res); + + /* 1 field */ + struct dict_transaction_context *ctx = dict_transaction_begin(dict, &dict_op_settings); + dict_set(ctx, "shared/counters/global/counter", "128"); + test_assert(dict_transaction_commit(&ctx, &error) == 1); + if (error != NULL) + i_error("dict_transaction_commit failed: %s", error); + error = NULL; + + /* 2 fields */ + ctx = dict_transaction_begin(dict, &dict_op_settings); + dict_set(ctx, "priv/quota/bytes", "128"); + dict_set(ctx, "priv/quota/count", "1"); + test_assert(dict_transaction_commit(&ctx, &error) == 1); + if (error != NULL) + i_error("dict_transaction_commit failed: %s", error); + error = NULL; + + /* 3 fields */ + ctx = dict_transaction_begin(dict, &dict_op_settings); + dict_set(ctx, "priv/quota/bytes", "128"); + dict_set(ctx, "priv/quota/count", "1"); + dict_set(ctx, "priv/quota/folders", "123"); + test_assert(dict_transaction_commit(&ctx, &error) == 1); + if (error != NULL) + i_error("dict_transaction_commit failed: %s", error); + test_teardown(&dict); + test_end(); +} + +static void test_unset(void) +{ + const char *error; + struct test_driver_result res = { + .affected_rows = 1, + .nqueries = 3, + .queries = (const char *[]){ + "DELETE FROM counters WHERE class = 'global' AND name = 'counter'", + "DELETE FROM quota WHERE username = 'testuser'", + "DELETE FROM quota WHERE username = 'testuser'", + NULL}, + .result = NULL, + }; + struct dict *dict; + + test_begin("dict unset"); + test_setup(&dict); + + test_set_expected(dict, &res); + + struct dict_transaction_context *ctx = dict_transaction_begin(dict, &dict_op_settings); + dict_unset(ctx, "shared/counters/global/counter"); + test_assert(dict_transaction_commit(&ctx, &error) == 1); + if (error != NULL) + i_error("dict_transaction_commit failed: %s", error); + error = NULL; + ctx = dict_transaction_begin(dict, &dict_op_settings); + dict_unset(ctx, "priv/quota/bytes"); + dict_unset(ctx, "priv/quota/count"); + test_assert(dict_transaction_commit(&ctx, &error) == 1); + if (error != NULL) + i_error("dict_transaction_commit failed: %s", error); + test_teardown(&dict); + test_end(); +} + +static void test_iterate(void) +{ + const char *key = NULL, *value = NULL, *error; + struct test_driver_result_set rset = { + .rows = 5, + .cols = 2, + .col_names = (const char *[]){"value", "name", NULL}, + .row_data = (const char **[]){ + (const char*[]){"one", "counter", NULL}, + (const char*[]){"two", "counter", NULL}, + (const char*[]){"three", "counter", NULL}, + (const char*[]){"four", "counter", NULL}, + (const char*[]){"five", "counter", NULL}, + }, + }; + struct test_driver_result res = { + .nqueries = 1, + .queries = (const char *[]){ + "SELECT value,name FROM counters WHERE class = 'global' AND name = 'counter'", + NULL}, + .result = &rset, + }; + struct dict *dict; + + test_begin("dict iterate"); + test_setup(&dict); + + test_set_expected(dict, &res); + + struct dict_iterate_context *iter = + dict_iterate_init(dict, &dict_op_settings, "shared/counters/global/counter", + DICT_ITERATE_FLAG_EXACT_KEY); + + size_t idx = 0; + while(dict_iterate(iter, &key, &value)) { + i_assert(idx < rset.rows); + test_assert_strcmp_idx(key, "shared/counters/global/counter", idx); + test_assert_strcmp_idx(value, rset.row_data[idx][0], idx); + idx++; + } + + test_assert(idx == rset.rows); + test_assert(dict_iterate_deinit(&iter, &error) == 0); + if (error != NULL) + i_error("dict_iterate_deinit failed: %s", error); + error = NULL; + + res.queries = (const char*[]){ + "SELECT value,name FROM counters WHERE class = 'global' AND name LIKE '%' AND name NOT LIKE '%/%'", + NULL + }; + + res.cur = 0; + res.result->cur = 0; + + test_set_expected(dict, &res); + + iter = dict_iterate_init(dict, &dict_op_settings, "shared/counters/global/", 0); + + idx = 0; + + while(dict_iterate(iter, &key, &value)) { + i_assert(idx < rset.rows); + test_assert_strcmp_idx(key, "shared/counters/global/counter", idx); + test_assert_strcmp_idx(value, rset.row_data[idx][0], idx); + idx++; + } + + test_assert(idx == rset.rows); + test_assert(dict_iterate_deinit(&iter, &error) == 0); + if (error != NULL) + i_error("dict_iterate_deinit failed: %s", error); + test_teardown(&dict); + test_end(); +} + +int main(void) { + sql_drivers_init(); + sql_driver_test_register(); + dict_sql_register(); + + static void (*const test_functions[])(void) = { + test_lookup_one, + test_atomic_inc, + test_set, + test_unset, + test_iterate, + NULL + }; + + int ret = test_run(test_functions); + + dict_sql_unregister(); + sql_driver_test_unregister(); + sql_drivers_deinit(); + + return ret; +} diff --git a/src/lib-dict-extra/Makefile.am b/src/lib-dict-extra/Makefile.am new file mode 100644 index 0000000..ce4ec4f --- /dev/null +++ b/src/lib-dict-extra/Makefile.am @@ -0,0 +1,35 @@ +noinst_LTLIBRARIES = libdict_extra.la + +dict_drivers = @dict_drivers@ + +AM_CPPFLAGS = \ + -I$(top_srcdir)/src/lib \ + -I$(top_srcdir)/src/lib-test \ + -I$(top_srcdir)/src/lib-dict \ + -I$(top_srcdir)/src/lib-fs \ + -I$(top_srcdir)/src/lib-settings + +libdict_extra_la_SOURCES = \ + dict-fs.c \ + dict-register.c + +NOPLUGIN_LDFLAGS = + +test_programs = \ + test-dict-fs + +noinst_PROGRAMS = $(test_programs) + +test_libs = \ + ../lib-test/libtest.la \ + ../lib-dict/libdict.la \ + ../lib/liblib.la + +test_dict_fs_SOURCES = test-dict-fs.c +test_dict_fs_LDADD = $(noinst_LTLIBRARIES) ../lib-fs/libfs.la $(test_libs) $(MODULE_LIBS) +test_dict_fs_DEPENDENCIES = $(noinst_LTLIBRARIES) ../lib-fs/libfs.la $(test_libs) + +check-local: + for bin in $(test_programs) $(check_PROGRAMS); do \ + if ! $(RUN_TEST) ./$$bin; then exit 1; fi; \ + done diff --git a/src/lib-dict-extra/Makefile.in b/src/lib-dict-extra/Makefile.in new file mode 100644 index 0000000..39d3036 --- /dev/null +++ b/src/lib-dict-extra/Makefile.in @@ -0,0 +1,796 @@ +# Makefile.in generated by automake 1.16.1 from Makefile.am. +# @configure_input@ + +# Copyright (C) 1994-2018 Free Software Foundation, Inc. + +# This Makefile.in is free software; the Free Software Foundation +# gives unlimited permission to copy and/or distribute it, +# with or without modifications, as long as this notice is preserved. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY, to the extent permitted by law; without +# even the implied warranty of MERCHANTABILITY or FITNESS FOR A +# PARTICULAR PURPOSE. + +@SET_MAKE@ + + +VPATH = @srcdir@ +am__is_gnu_make = { \ + if test -z '$(MAKELEVEL)'; then \ + false; \ + elif test -n '$(MAKE_HOST)'; then \ + true; \ + elif test -n '$(MAKE_VERSION)' && test -n '$(CURDIR)'; then \ + true; \ + else \ + false; \ + fi; \ +} +am__make_running_with_option = \ + case $${target_option-} in \ + ?) ;; \ + *) echo "am__make_running_with_option: internal error: invalid" \ + "target option '$${target_option-}' specified" >&2; \ + exit 1;; \ + esac; \ + has_opt=no; \ + sane_makeflags=$$MAKEFLAGS; \ + if $(am__is_gnu_make); then \ + sane_makeflags=$$MFLAGS; \ + else \ + case $$MAKEFLAGS in \ + *\\[\ \ ]*) \ + bs=\\; \ + sane_makeflags=`printf '%s\n' "$$MAKEFLAGS" \ + | sed "s/$$bs$$bs[$$bs $$bs ]*//g"`;; \ + esac; \ + fi; \ + skip_next=no; \ + strip_trailopt () \ + { \ + flg=`printf '%s\n' "$$flg" | sed "s/$$1.*$$//"`; \ + }; \ + for flg in $$sane_makeflags; do \ + test $$skip_next = yes && { skip_next=no; continue; }; \ + case $$flg in \ + *=*|--*) continue;; \ + -*I) strip_trailopt 'I'; skip_next=yes;; \ + -*I?*) strip_trailopt 'I';; \ + -*O) strip_trailopt 'O'; skip_next=yes;; \ + -*O?*) strip_trailopt 'O';; \ + -*l) strip_trailopt 'l'; skip_next=yes;; \ + -*l?*) strip_trailopt 'l';; \ + -[dEDm]) skip_next=yes;; \ + -[JT]) skip_next=yes;; \ + esac; \ + case $$flg in \ + *$$target_option*) has_opt=yes; break;; \ + esac; \ + done; \ + test $$has_opt = yes +am__make_dryrun = (target_option=n; $(am__make_running_with_option)) +am__make_keepgoing = (target_option=k; $(am__make_running_with_option)) +pkgdatadir = $(datadir)/@PACKAGE@ +pkgincludedir = $(includedir)/@PACKAGE@ +pkglibdir = $(libdir)/@PACKAGE@ +pkglibexecdir = $(libexecdir)/@PACKAGE@ +am__cd = CDPATH="$${ZSH_VERSION+.}$(PATH_SEPARATOR)" && cd +install_sh_DATA = $(install_sh) -c -m 644 +install_sh_PROGRAM = $(install_sh) -c +install_sh_SCRIPT = $(install_sh) -c +INSTALL_HEADER = $(INSTALL_DATA) +transform = $(program_transform_name) +NORMAL_INSTALL = : +PRE_INSTALL = : +POST_INSTALL = : +NORMAL_UNINSTALL = : +PRE_UNINSTALL = : +POST_UNINSTALL = : +build_triplet = @build@ +host_triplet = @host@ +noinst_PROGRAMS = $(am__EXEEXT_1) +subdir = src/lib-dict-extra +ACLOCAL_M4 = $(top_srcdir)/aclocal.m4 +am__aclocal_m4_deps = $(top_srcdir)/m4/ac_checktype2.m4 \ + $(top_srcdir)/m4/ac_typeof.m4 $(top_srcdir)/m4/arc4random.m4 \ + $(top_srcdir)/m4/blockdev.m4 $(top_srcdir)/m4/c99_vsnprintf.m4 \ + $(top_srcdir)/m4/clock_gettime.m4 $(top_srcdir)/m4/crypt.m4 \ + $(top_srcdir)/m4/crypt_xpg6.m4 $(top_srcdir)/m4/dbqlk.m4 \ + $(top_srcdir)/m4/dirent_dtype.m4 $(top_srcdir)/m4/dovecot.m4 \ + $(top_srcdir)/m4/fd_passing.m4 $(top_srcdir)/m4/fdatasync.m4 \ + $(top_srcdir)/m4/flexible_array_member.m4 \ + $(top_srcdir)/m4/glibc.m4 $(top_srcdir)/m4/gmtime_max.m4 \ + $(top_srcdir)/m4/gmtime_tm_gmtoff.m4 \ + $(top_srcdir)/m4/ioloop.m4 $(top_srcdir)/m4/iovec.m4 \ + $(top_srcdir)/m4/ipv6.m4 $(top_srcdir)/m4/libcap.m4 \ + $(top_srcdir)/m4/libtool.m4 $(top_srcdir)/m4/libwrap.m4 \ + $(top_srcdir)/m4/linux_mremap.m4 $(top_srcdir)/m4/ltoptions.m4 \ + $(top_srcdir)/m4/ltsugar.m4 $(top_srcdir)/m4/ltversion.m4 \ + $(top_srcdir)/m4/lt~obsolete.m4 $(top_srcdir)/m4/mmap_write.m4 \ + $(top_srcdir)/m4/mntctl.m4 $(top_srcdir)/m4/modules.m4 \ + $(top_srcdir)/m4/notify.m4 $(top_srcdir)/m4/nsl.m4 \ + $(top_srcdir)/m4/off_t_max.m4 $(top_srcdir)/m4/pkg.m4 \ + $(top_srcdir)/m4/pr_set_dumpable.m4 \ + $(top_srcdir)/m4/q_quotactl.m4 $(top_srcdir)/m4/quota.m4 \ + $(top_srcdir)/m4/random.m4 $(top_srcdir)/m4/rlimit.m4 \ + $(top_srcdir)/m4/sendfile.m4 $(top_srcdir)/m4/size_t_signed.m4 \ + $(top_srcdir)/m4/sockpeercred.m4 $(top_srcdir)/m4/sql.m4 \ + $(top_srcdir)/m4/ssl.m4 $(top_srcdir)/m4/st_tim.m4 \ + $(top_srcdir)/m4/static_array.m4 $(top_srcdir)/m4/test_with.m4 \ + $(top_srcdir)/m4/time_t.m4 $(top_srcdir)/m4/typeof.m4 \ + $(top_srcdir)/m4/typeof_dev_t.m4 \ + $(top_srcdir)/m4/uoff_t_max.m4 $(top_srcdir)/m4/vararg.m4 \ + $(top_srcdir)/m4/want_apparmor.m4 \ + $(top_srcdir)/m4/want_bsdauth.m4 \ + $(top_srcdir)/m4/want_bzlib.m4 \ + $(top_srcdir)/m4/want_cassandra.m4 \ + $(top_srcdir)/m4/want_cdb.m4 \ + $(top_srcdir)/m4/want_checkpassword.m4 \ + $(top_srcdir)/m4/want_clucene.m4 $(top_srcdir)/m4/want_db.m4 \ + $(top_srcdir)/m4/want_gssapi.m4 $(top_srcdir)/m4/want_icu.m4 \ + $(top_srcdir)/m4/want_ldap.m4 $(top_srcdir)/m4/want_lua.m4 \ + $(top_srcdir)/m4/want_lz4.m4 $(top_srcdir)/m4/want_lzma.m4 \ + $(top_srcdir)/m4/want_mysql.m4 $(top_srcdir)/m4/want_pam.m4 \ + $(top_srcdir)/m4/want_passwd.m4 $(top_srcdir)/m4/want_pgsql.m4 \ + $(top_srcdir)/m4/want_prefetch.m4 \ + $(top_srcdir)/m4/want_shadow.m4 \ + $(top_srcdir)/m4/want_sodium.m4 $(top_srcdir)/m4/want_solr.m4 \ + $(top_srcdir)/m4/want_sqlite.m4 \ + $(top_srcdir)/m4/want_stemmer.m4 \ + $(top_srcdir)/m4/want_systemd.m4 \ + $(top_srcdir)/m4/want_textcat.m4 \ + $(top_srcdir)/m4/want_unwind.m4 $(top_srcdir)/m4/want_zlib.m4 \ + $(top_srcdir)/m4/want_zstd.m4 $(top_srcdir)/configure.ac +am__configure_deps = $(am__aclocal_m4_deps) $(CONFIGURE_DEPENDENCIES) \ + $(ACLOCAL_M4) +DIST_COMMON = $(srcdir)/Makefile.am $(am__DIST_COMMON) +mkinstalldirs = $(install_sh) -d +CONFIG_HEADER = $(top_builddir)/config.h +CONFIG_CLEAN_FILES = +CONFIG_CLEAN_VPATH_FILES = +am__EXEEXT_1 = test-dict-fs$(EXEEXT) +PROGRAMS = $(noinst_PROGRAMS) +LTLIBRARIES = $(noinst_LTLIBRARIES) +libdict_extra_la_LIBADD = +am_libdict_extra_la_OBJECTS = dict-fs.lo dict-register.lo +libdict_extra_la_OBJECTS = $(am_libdict_extra_la_OBJECTS) +AM_V_lt = $(am__v_lt_@AM_V@) +am__v_lt_ = $(am__v_lt_@AM_DEFAULT_V@) +am__v_lt_0 = --silent +am__v_lt_1 = +am_test_dict_fs_OBJECTS = test-dict-fs.$(OBJEXT) +test_dict_fs_OBJECTS = $(am_test_dict_fs_OBJECTS) +am__DEPENDENCIES_1 = +AM_V_P = $(am__v_P_@AM_V@) +am__v_P_ = $(am__v_P_@AM_DEFAULT_V@) +am__v_P_0 = false +am__v_P_1 = : +AM_V_GEN = $(am__v_GEN_@AM_V@) +am__v_GEN_ = $(am__v_GEN_@AM_DEFAULT_V@) +am__v_GEN_0 = @echo " GEN " $@; +am__v_GEN_1 = +AM_V_at = $(am__v_at_@AM_V@) +am__v_at_ = $(am__v_at_@AM_DEFAULT_V@) +am__v_at_0 = @ +am__v_at_1 = +DEFAULT_INCLUDES = -I.@am__isrc@ -I$(top_builddir) +depcomp = $(SHELL) $(top_srcdir)/depcomp +am__maybe_remake_depfiles = depfiles +am__depfiles_remade = ./$(DEPDIR)/dict-fs.Plo \ + ./$(DEPDIR)/dict-register.Plo ./$(DEPDIR)/test-dict-fs.Po +am__mv = mv -f +COMPILE = $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) \ + $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) +LTCOMPILE = $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) \ + $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) \ + $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) \ + $(AM_CFLAGS) $(CFLAGS) +AM_V_CC = $(am__v_CC_@AM_V@) +am__v_CC_ = $(am__v_CC_@AM_DEFAULT_V@) +am__v_CC_0 = @echo " CC " $@; +am__v_CC_1 = +CCLD = $(CC) +LINK = $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) \ + $(LIBTOOLFLAGS) --mode=link $(CCLD) $(AM_CFLAGS) $(CFLAGS) \ + $(AM_LDFLAGS) $(LDFLAGS) -o $@ +AM_V_CCLD = $(am__v_CCLD_@AM_V@) +am__v_CCLD_ = $(am__v_CCLD_@AM_DEFAULT_V@) +am__v_CCLD_0 = @echo " CCLD " $@; +am__v_CCLD_1 = +SOURCES = $(libdict_extra_la_SOURCES) $(test_dict_fs_SOURCES) +DIST_SOURCES = $(libdict_extra_la_SOURCES) $(test_dict_fs_SOURCES) +am__can_run_installinfo = \ + case $$AM_UPDATE_INFO_DIR in \ + n|no|NO) false;; \ + *) (install-info --version) >/dev/null 2>&1;; \ + esac +am__tagged_files = $(HEADERS) $(SOURCES) $(TAGS_FILES) $(LISP) +# Read a list of newline-separated strings from the standard input, +# and print each of them once, without duplicates. Input order is +# *not* preserved. +am__uniquify_input = $(AWK) '\ + BEGIN { nonempty = 0; } \ + { items[$$0] = 1; nonempty = 1; } \ + END { if (nonempty) { for (i in items) print i; }; } \ +' +# Make sure the list of sources is unique. This is necessary because, +# e.g., the same source file might be shared among _SOURCES variables +# for different programs/libraries. +am__define_uniq_tagged_files = \ + list='$(am__tagged_files)'; \ + unique=`for i in $$list; do \ + if test -f "$$i"; then echo $$i; else echo $(srcdir)/$$i; fi; \ + done | $(am__uniquify_input)` +ETAGS = etags +CTAGS = ctags +am__DIST_COMMON = $(srcdir)/Makefile.in $(top_srcdir)/depcomp +DISTFILES = $(DIST_COMMON) $(DIST_SOURCES) $(TEXINFOS) $(EXTRA_DIST) +ACLOCAL = @ACLOCAL@ +ACLOCAL_AMFLAGS = @ACLOCAL_AMFLAGS@ +AMTAR = @AMTAR@ +AM_DEFAULT_VERBOSITY = @AM_DEFAULT_VERBOSITY@ +APPARMOR_LIBS = @APPARMOR_LIBS@ +AR = @AR@ +AUTH_CFLAGS = @AUTH_CFLAGS@ +AUTH_LIBS = @AUTH_LIBS@ +AUTOCONF = @AUTOCONF@ +AUTOHEADER = @AUTOHEADER@ +AUTOMAKE = @AUTOMAKE@ +AWK = @AWK@ +BINARY_CFLAGS = @BINARY_CFLAGS@ +BINARY_LDFLAGS = @BINARY_LDFLAGS@ +BISON = @BISON@ +CASSANDRA_CFLAGS = @CASSANDRA_CFLAGS@ +CASSANDRA_LIBS = @CASSANDRA_LIBS@ +CC = @CC@ +CCDEPMODE = @CCDEPMODE@ +CDB_LIBS = @CDB_LIBS@ +CFLAGS = @CFLAGS@ +CLUCENE_CFLAGS = @CLUCENE_CFLAGS@ +CLUCENE_LIBS = @CLUCENE_LIBS@ +COMPRESS_LIBS = @COMPRESS_LIBS@ +CPP = @CPP@ +CPPFLAGS = @CPPFLAGS@ +CRYPT_LIBS = @CRYPT_LIBS@ +CXX = @CXX@ +CXXCPP = @CXXCPP@ +CXXDEPMODE = @CXXDEPMODE@ +CXXFLAGS = @CXXFLAGS@ +CYGPATH_W = @CYGPATH_W@ +DEFS = @DEFS@ +DEPDIR = @DEPDIR@ +DICT_LIBS = @DICT_LIBS@ +DLLIB = @DLLIB@ +DLLTOOL = @DLLTOOL@ +DSYMUTIL = @DSYMUTIL@ +DUMPBIN = @DUMPBIN@ +ECHO_C = @ECHO_C@ +ECHO_N = @ECHO_N@ +ECHO_T = @ECHO_T@ +EGREP = @EGREP@ +EXEEXT = @EXEEXT@ +FGREP = @FGREP@ +FLEX = @FLEX@ +FUZZER_CPPFLAGS = @FUZZER_CPPFLAGS@ +FUZZER_LDFLAGS = @FUZZER_LDFLAGS@ +GREP = @GREP@ +INSTALL = @INSTALL@ +INSTALL_DATA = @INSTALL_DATA@ +INSTALL_PROGRAM = @INSTALL_PROGRAM@ +INSTALL_SCRIPT = @INSTALL_SCRIPT@ +INSTALL_STRIP_PROGRAM = @INSTALL_STRIP_PROGRAM@ +KRB5CONFIG = @KRB5CONFIG@ +KRB5_CFLAGS = @KRB5_CFLAGS@ +KRB5_LIBS = @KRB5_LIBS@ +LD = @LD@ +LDAP_LIBS = @LDAP_LIBS@ +LDFLAGS = @LDFLAGS@ +LD_NO_WHOLE_ARCHIVE = @LD_NO_WHOLE_ARCHIVE@ +LD_WHOLE_ARCHIVE = @LD_WHOLE_ARCHIVE@ +LIBCAP = @LIBCAP@ +LIBDOVECOT = @LIBDOVECOT@ +LIBDOVECOT_COMPRESS = @LIBDOVECOT_COMPRESS@ +LIBDOVECOT_DEPS = @LIBDOVECOT_DEPS@ +LIBDOVECOT_DSYNC = @LIBDOVECOT_DSYNC@ +LIBDOVECOT_LA_LIBS = @LIBDOVECOT_LA_LIBS@ +LIBDOVECOT_LDA = @LIBDOVECOT_LDA@ +LIBDOVECOT_LDAP = @LIBDOVECOT_LDAP@ +LIBDOVECOT_LIBFTS = @LIBDOVECOT_LIBFTS@ +LIBDOVECOT_LIBFTS_DEPS = @LIBDOVECOT_LIBFTS_DEPS@ +LIBDOVECOT_LOGIN = @LIBDOVECOT_LOGIN@ +LIBDOVECOT_LUA = @LIBDOVECOT_LUA@ +LIBDOVECOT_LUA_DEPS = @LIBDOVECOT_LUA_DEPS@ +LIBDOVECOT_SQL = @LIBDOVECOT_SQL@ +LIBDOVECOT_STORAGE = @LIBDOVECOT_STORAGE@ +LIBDOVECOT_STORAGE_DEPS = @LIBDOVECOT_STORAGE_DEPS@ +LIBEXTTEXTCAT_CFLAGS = @LIBEXTTEXTCAT_CFLAGS@ +LIBEXTTEXTCAT_LIBS = @LIBEXTTEXTCAT_LIBS@ +LIBICONV = @LIBICONV@ +LIBICU_CFLAGS = @LIBICU_CFLAGS@ +LIBICU_LIBS = @LIBICU_LIBS@ +LIBOBJS = @LIBOBJS@ +LIBS = @LIBS@ +LIBSODIUM_CFLAGS = @LIBSODIUM_CFLAGS@ +LIBSODIUM_LIBS = @LIBSODIUM_LIBS@ +LIBTIRPC_CFLAGS = @LIBTIRPC_CFLAGS@ +LIBTIRPC_LIBS = @LIBTIRPC_LIBS@ +LIBTOOL = @LIBTOOL@ +LIBUNWIND_CFLAGS = @LIBUNWIND_CFLAGS@ +LIBUNWIND_LIBS = @LIBUNWIND_LIBS@ +LIBWRAP_LIBS = @LIBWRAP_LIBS@ +LINKED_STORAGE_LDADD = @LINKED_STORAGE_LDADD@ +LIPO = @LIPO@ +LN_S = @LN_S@ +LTLIBICONV = @LTLIBICONV@ +LTLIBOBJS = @LTLIBOBJS@ +LT_SYS_LIBRARY_PATH = @LT_SYS_LIBRARY_PATH@ +LUA_CFLAGS = @LUA_CFLAGS@ +LUA_LIBS = @LUA_LIBS@ +MAINT = @MAINT@ +MAKEINFO = @MAKEINFO@ +MANIFEST_TOOL = @MANIFEST_TOOL@ +MKDIR_P = @MKDIR_P@ +MODULE_LIBS = @MODULE_LIBS@ +MODULE_SUFFIX = @MODULE_SUFFIX@ +MYSQL_CFLAGS = @MYSQL_CFLAGS@ +MYSQL_CONFIG = @MYSQL_CONFIG@ +MYSQL_LIBS = @MYSQL_LIBS@ +NM = @NM@ +NMEDIT = @NMEDIT@ +NOPLUGIN_LDFLAGS = +OBJDUMP = @OBJDUMP@ +OBJEXT = @OBJEXT@ +OTOOL = @OTOOL@ +OTOOL64 = @OTOOL64@ +PACKAGE = @PACKAGE@ +PACKAGE_BUGREPORT = @PACKAGE_BUGREPORT@ +PACKAGE_NAME = @PACKAGE_NAME@ +PACKAGE_STRING = @PACKAGE_STRING@ +PACKAGE_TARNAME = @PACKAGE_TARNAME@ +PACKAGE_URL = @PACKAGE_URL@ +PACKAGE_VERSION = @PACKAGE_VERSION@ +PANDOC = @PANDOC@ +PATH_SEPARATOR = @PATH_SEPARATOR@ +PGSQL_CFLAGS = @PGSQL_CFLAGS@ +PGSQL_LIBS = @PGSQL_LIBS@ +PG_CONFIG = @PG_CONFIG@ +PIE_CFLAGS = @PIE_CFLAGS@ +PIE_LDFLAGS = @PIE_LDFLAGS@ +PKG_CONFIG = @PKG_CONFIG@ +PKG_CONFIG_LIBDIR = @PKG_CONFIG_LIBDIR@ +PKG_CONFIG_PATH = @PKG_CONFIG_PATH@ +QUOTA_LIBS = @QUOTA_LIBS@ +RANLIB = @RANLIB@ +RELRO_LDFLAGS = @RELRO_LDFLAGS@ +RPCGEN = @RPCGEN@ +RUN_TEST = @RUN_TEST@ +SED = @SED@ +SETTING_FILES = @SETTING_FILES@ +SET_MAKE = @SET_MAKE@ +SHELL = @SHELL@ +SQLITE_CFLAGS = @SQLITE_CFLAGS@ +SQLITE_LIBS = @SQLITE_LIBS@ +SQL_CFLAGS = @SQL_CFLAGS@ +SQL_LIBS = @SQL_LIBS@ +SSL_CFLAGS = @SSL_CFLAGS@ +SSL_LIBS = @SSL_LIBS@ +STRIP = @STRIP@ +SYSTEMD_CFLAGS = @SYSTEMD_CFLAGS@ +SYSTEMD_LIBS = @SYSTEMD_LIBS@ +VALGRIND = @VALGRIND@ +VERSION = @VERSION@ +ZSTD_CFLAGS = @ZSTD_CFLAGS@ +ZSTD_LIBS = @ZSTD_LIBS@ +abs_builddir = @abs_builddir@ +abs_srcdir = @abs_srcdir@ +abs_top_builddir = @abs_top_builddir@ +abs_top_srcdir = @abs_top_srcdir@ +ac_ct_AR = @ac_ct_AR@ +ac_ct_CC = @ac_ct_CC@ +ac_ct_CXX = @ac_ct_CXX@ +ac_ct_DUMPBIN = @ac_ct_DUMPBIN@ +am__include = @am__include@ +am__leading_dot = @am__leading_dot@ +am__quote = @am__quote@ +am__tar = @am__tar@ +am__untar = @am__untar@ +bindir = @bindir@ +build = @build@ +build_alias = @build_alias@ +build_cpu = @build_cpu@ +build_os = @build_os@ +build_vendor = @build_vendor@ +builddir = @builddir@ +datadir = @datadir@ +datarootdir = @datarootdir@ +dict_drivers = @dict_drivers@ +docdir = @docdir@ +dvidir = @dvidir@ +exec_prefix = @exec_prefix@ +host = @host@ +host_alias = @host_alias@ +host_cpu = @host_cpu@ +host_os = @host_os@ +host_vendor = @host_vendor@ +htmldir = @htmldir@ +includedir = @includedir@ +infodir = @infodir@ +install_sh = @install_sh@ +libdir = @libdir@ +libexecdir = @libexecdir@ +localedir = @localedir@ +localstatedir = @localstatedir@ +mandir = @mandir@ +mkdir_p = @mkdir_p@ +moduledir = @moduledir@ +oldincludedir = @oldincludedir@ +pdfdir = @pdfdir@ +prefix = @prefix@ +program_transform_name = @program_transform_name@ +psdir = @psdir@ +rundir = @rundir@ +runstatedir = @runstatedir@ +sbindir = @sbindir@ +sharedstatedir = @sharedstatedir@ +sql_drivers = @sql_drivers@ +srcdir = @srcdir@ +ssldir = @ssldir@ +statedir = @statedir@ +sysconfdir = @sysconfdir@ +systemdservicetype = @systemdservicetype@ +systemdsystemunitdir = @systemdsystemunitdir@ +target_alias = @target_alias@ +top_build_prefix = @top_build_prefix@ +top_builddir = @top_builddir@ +top_srcdir = @top_srcdir@ +noinst_LTLIBRARIES = libdict_extra.la +AM_CPPFLAGS = \ + -I$(top_srcdir)/src/lib \ + -I$(top_srcdir)/src/lib-test \ + -I$(top_srcdir)/src/lib-dict \ + -I$(top_srcdir)/src/lib-fs \ + -I$(top_srcdir)/src/lib-settings + +libdict_extra_la_SOURCES = \ + dict-fs.c \ + dict-register.c + +test_programs = \ + test-dict-fs + +test_libs = \ + ../lib-test/libtest.la \ + ../lib-dict/libdict.la \ + ../lib/liblib.la + +test_dict_fs_SOURCES = test-dict-fs.c +test_dict_fs_LDADD = $(noinst_LTLIBRARIES) ../lib-fs/libfs.la $(test_libs) $(MODULE_LIBS) +test_dict_fs_DEPENDENCIES = $(noinst_LTLIBRARIES) ../lib-fs/libfs.la $(test_libs) +all: all-am + +.SUFFIXES: +.SUFFIXES: .c .lo .o .obj +$(srcdir)/Makefile.in: @MAINTAINER_MODE_TRUE@ $(srcdir)/Makefile.am $(am__configure_deps) + @for dep in $?; do \ + case '$(am__configure_deps)' in \ + *$$dep*) \ + ( cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh ) \ + && { if test -f $@; then exit 0; else break; fi; }; \ + exit 1;; \ + esac; \ + done; \ + echo ' cd $(top_srcdir) && $(AUTOMAKE) --foreign src/lib-dict-extra/Makefile'; \ + $(am__cd) $(top_srcdir) && \ + $(AUTOMAKE) --foreign src/lib-dict-extra/Makefile +Makefile: $(srcdir)/Makefile.in $(top_builddir)/config.status + @case '$?' in \ + *config.status*) \ + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh;; \ + *) \ + echo ' cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__maybe_remake_depfiles)'; \ + cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__maybe_remake_depfiles);; \ + esac; + +$(top_builddir)/config.status: $(top_srcdir)/configure $(CONFIG_STATUS_DEPENDENCIES) + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh + +$(top_srcdir)/configure: @MAINTAINER_MODE_TRUE@ $(am__configure_deps) + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh +$(ACLOCAL_M4): @MAINTAINER_MODE_TRUE@ $(am__aclocal_m4_deps) + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh +$(am__aclocal_m4_deps): + +clean-noinstPROGRAMS: + @list='$(noinst_PROGRAMS)'; test -n "$$list" || exit 0; \ + echo " rm -f" $$list; \ + rm -f $$list || exit $$?; \ + test -n "$(EXEEXT)" || exit 0; \ + list=`for p in $$list; do echo "$$p"; done | sed 's/$(EXEEXT)$$//'`; \ + echo " rm -f" $$list; \ + rm -f $$list + +clean-noinstLTLIBRARIES: + -test -z "$(noinst_LTLIBRARIES)" || rm -f $(noinst_LTLIBRARIES) + @list='$(noinst_LTLIBRARIES)'; \ + locs=`for p in $$list; do echo $$p; done | \ + sed 's|^[^/]*$$|.|; s|/[^/]*$$||; s|$$|/so_locations|' | \ + sort -u`; \ + test -z "$$locs" || { \ + echo rm -f $${locs}; \ + rm -f $${locs}; \ + } + +libdict_extra.la: $(libdict_extra_la_OBJECTS) $(libdict_extra_la_DEPENDENCIES) $(EXTRA_libdict_extra_la_DEPENDENCIES) + $(AM_V_CCLD)$(LINK) $(libdict_extra_la_OBJECTS) $(libdict_extra_la_LIBADD) $(LIBS) + +test-dict-fs$(EXEEXT): $(test_dict_fs_OBJECTS) $(test_dict_fs_DEPENDENCIES) $(EXTRA_test_dict_fs_DEPENDENCIES) + @rm -f test-dict-fs$(EXEEXT) + $(AM_V_CCLD)$(LINK) $(test_dict_fs_OBJECTS) $(test_dict_fs_LDADD) $(LIBS) + +mostlyclean-compile: + -rm -f *.$(OBJEXT) + +distclean-compile: + -rm -f *.tab.c + +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/dict-fs.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/dict-register.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/test-dict-fs.Po@am__quote@ # am--include-marker + +$(am__depfiles_remade): + @$(MKDIR_P) $(@D) + @echo '# dummy' >$@-t && $(am__mv) $@-t $@ + +am--depfiles: $(am__depfiles_remade) + +.c.o: +@am__fastdepCC_TRUE@ $(AM_V_CC)$(COMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ $< +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='$<' object='$@' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(COMPILE) -c -o $@ $< + +.c.obj: +@am__fastdepCC_TRUE@ $(AM_V_CC)$(COMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ `$(CYGPATH_W) '$<'` +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='$<' object='$@' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(COMPILE) -c -o $@ `$(CYGPATH_W) '$<'` + +.c.lo: +@am__fastdepCC_TRUE@ $(AM_V_CC)$(LTCOMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ $< +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Plo +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='$<' object='$@' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(LTCOMPILE) -c -o $@ $< + +mostlyclean-libtool: + -rm -f *.lo + +clean-libtool: + -rm -rf .libs _libs + +ID: $(am__tagged_files) + $(am__define_uniq_tagged_files); mkid -fID $$unique +tags: tags-am +TAGS: tags + +tags-am: $(TAGS_DEPENDENCIES) $(am__tagged_files) + set x; \ + here=`pwd`; \ + $(am__define_uniq_tagged_files); \ + shift; \ + if test -z "$(ETAGS_ARGS)$$*$$unique"; then :; else \ + test -n "$$unique" || unique=$$empty_fix; \ + if test $$# -gt 0; then \ + $(ETAGS) $(ETAGSFLAGS) $(AM_ETAGSFLAGS) $(ETAGS_ARGS) \ + "$$@" $$unique; \ + else \ + $(ETAGS) $(ETAGSFLAGS) $(AM_ETAGSFLAGS) $(ETAGS_ARGS) \ + $$unique; \ + fi; \ + fi +ctags: ctags-am + +CTAGS: ctags +ctags-am: $(TAGS_DEPENDENCIES) $(am__tagged_files) + $(am__define_uniq_tagged_files); \ + test -z "$(CTAGS_ARGS)$$unique" \ + || $(CTAGS) $(CTAGSFLAGS) $(AM_CTAGSFLAGS) $(CTAGS_ARGS) \ + $$unique + +GTAGS: + here=`$(am__cd) $(top_builddir) && pwd` \ + && $(am__cd) $(top_srcdir) \ + && gtags -i $(GTAGS_ARGS) "$$here" +cscopelist: cscopelist-am + +cscopelist-am: $(am__tagged_files) + list='$(am__tagged_files)'; \ + case "$(srcdir)" in \ + [\\/]* | ?:[\\/]*) sdir="$(srcdir)" ;; \ + *) sdir=$(subdir)/$(srcdir) ;; \ + esac; \ + for i in $$list; do \ + if test -f "$$i"; then \ + echo "$(subdir)/$$i"; \ + else \ + echo "$$sdir/$$i"; \ + fi; \ + done >> $(top_builddir)/cscope.files + +distclean-tags: + -rm -f TAGS ID GTAGS GRTAGS GSYMS GPATH tags + +distdir: $(BUILT_SOURCES) + $(MAKE) $(AM_MAKEFLAGS) distdir-am + +distdir-am: $(DISTFILES) + @srcdirstrip=`echo "$(srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \ + topsrcdirstrip=`echo "$(top_srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \ + list='$(DISTFILES)'; \ + dist_files=`for file in $$list; do echo $$file; done | \ + sed -e "s|^$$srcdirstrip/||;t" \ + -e "s|^$$topsrcdirstrip/|$(top_builddir)/|;t"`; \ + case $$dist_files in \ + */*) $(MKDIR_P) `echo "$$dist_files" | \ + sed '/\//!d;s|^|$(distdir)/|;s,/[^/]*$$,,' | \ + sort -u` ;; \ + esac; \ + for file in $$dist_files; do \ + if test -f $$file || test -d $$file; then d=.; else d=$(srcdir); fi; \ + if test -d $$d/$$file; then \ + dir=`echo "/$$file" | sed -e 's,/[^/]*$$,,'`; \ + if test -d "$(distdir)/$$file"; then \ + find "$(distdir)/$$file" -type d ! -perm -700 -exec chmod u+rwx {} \;; \ + fi; \ + if test -d $(srcdir)/$$file && test $$d != $(srcdir); then \ + cp -fpR $(srcdir)/$$file "$(distdir)$$dir" || exit 1; \ + find "$(distdir)/$$file" -type d ! -perm -700 -exec chmod u+rwx {} \;; \ + fi; \ + cp -fpR $$d/$$file "$(distdir)$$dir" || exit 1; \ + else \ + test -f "$(distdir)/$$file" \ + || cp -p $$d/$$file "$(distdir)/$$file" \ + || exit 1; \ + fi; \ + done +check-am: all-am + $(MAKE) $(AM_MAKEFLAGS) check-local +check: check-am +all-am: Makefile $(PROGRAMS) $(LTLIBRARIES) +installdirs: +install: install-am +install-exec: install-exec-am +install-data: install-data-am +uninstall: uninstall-am + +install-am: all-am + @$(MAKE) $(AM_MAKEFLAGS) install-exec-am install-data-am + +installcheck: installcheck-am +install-strip: + if test -z '$(STRIP)'; then \ + $(MAKE) $(AM_MAKEFLAGS) INSTALL_PROGRAM="$(INSTALL_STRIP_PROGRAM)" \ + install_sh_PROGRAM="$(INSTALL_STRIP_PROGRAM)" INSTALL_STRIP_FLAG=-s \ + install; \ + else \ + $(MAKE) $(AM_MAKEFLAGS) INSTALL_PROGRAM="$(INSTALL_STRIP_PROGRAM)" \ + install_sh_PROGRAM="$(INSTALL_STRIP_PROGRAM)" INSTALL_STRIP_FLAG=-s \ + "INSTALL_PROGRAM_ENV=STRIPPROG='$(STRIP)'" install; \ + fi +mostlyclean-generic: + +clean-generic: + +distclean-generic: + -test -z "$(CONFIG_CLEAN_FILES)" || rm -f $(CONFIG_CLEAN_FILES) + -test . = "$(srcdir)" || test -z "$(CONFIG_CLEAN_VPATH_FILES)" || rm -f $(CONFIG_CLEAN_VPATH_FILES) + +maintainer-clean-generic: + @echo "This command is intended for maintainers to use" + @echo "it deletes files that may require special tools to rebuild." +clean: clean-am + +clean-am: clean-generic clean-libtool clean-noinstLTLIBRARIES \ + clean-noinstPROGRAMS mostlyclean-am + +distclean: distclean-am + -rm -f ./$(DEPDIR)/dict-fs.Plo + -rm -f ./$(DEPDIR)/dict-register.Plo + -rm -f ./$(DEPDIR)/test-dict-fs.Po + -rm -f Makefile +distclean-am: clean-am distclean-compile distclean-generic \ + distclean-tags + +dvi: dvi-am + +dvi-am: + +html: html-am + +html-am: + +info: info-am + +info-am: + +install-data-am: + +install-dvi: install-dvi-am + +install-dvi-am: + +install-exec-am: + +install-html: install-html-am + +install-html-am: + +install-info: install-info-am + +install-info-am: + +install-man: + +install-pdf: install-pdf-am + +install-pdf-am: + +install-ps: install-ps-am + +install-ps-am: + +installcheck-am: + +maintainer-clean: maintainer-clean-am + -rm -f ./$(DEPDIR)/dict-fs.Plo + -rm -f ./$(DEPDIR)/dict-register.Plo + -rm -f ./$(DEPDIR)/test-dict-fs.Po + -rm -f Makefile +maintainer-clean-am: distclean-am maintainer-clean-generic + +mostlyclean: mostlyclean-am + +mostlyclean-am: mostlyclean-compile mostlyclean-generic \ + mostlyclean-libtool + +pdf: pdf-am + +pdf-am: + +ps: ps-am + +ps-am: + +uninstall-am: + +.MAKE: check-am install-am install-strip + +.PHONY: CTAGS GTAGS TAGS all all-am am--depfiles check check-am \ + check-local clean clean-generic clean-libtool \ + clean-noinstLTLIBRARIES clean-noinstPROGRAMS cscopelist-am \ + ctags ctags-am distclean distclean-compile distclean-generic \ + distclean-libtool distclean-tags distdir dvi dvi-am html \ + html-am info info-am install install-am install-data \ + install-data-am install-dvi install-dvi-am install-exec \ + install-exec-am install-html install-html-am install-info \ + install-info-am install-man install-pdf install-pdf-am \ + install-ps install-ps-am install-strip installcheck \ + installcheck-am installdirs maintainer-clean \ + maintainer-clean-generic mostlyclean mostlyclean-compile \ + mostlyclean-generic mostlyclean-libtool pdf pdf-am ps ps-am \ + tags tags-am uninstall uninstall-am + +.PRECIOUS: Makefile + + +check-local: + for bin in $(test_programs) $(check_PROGRAMS); do \ + if ! $(RUN_TEST) ./$$bin; then exit 1; fi; \ + done + +# Tell versions [3.59,3.63) of GNU make to not export all variables. +# Otherwise a system limit (for SysV at least) may be exceeded. +.NOEXPORT: diff --git a/src/lib-dict-extra/dict-fs.c b/src/lib-dict-extra/dict-fs.c new file mode 100644 index 0000000..4c173e3 --- /dev/null +++ b/src/lib-dict-extra/dict-fs.c @@ -0,0 +1,330 @@ +/* Copyright (c) 2013-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "fs-api.h" +#include "istream.h" +#include "str.h" +#include "dict-transaction-memory.h" +#include "dict-private.h" + +struct fs_dict { + struct dict dict; + struct fs *fs; +}; + +struct fs_dict_iterate_context { + struct dict_iterate_context ctx; + char *path; + enum dict_iterate_flags flags; + pool_t value_pool; + struct fs_iter *fs_iter; + const char *values[2]; + char *error; +}; + +static int +fs_dict_init(struct dict *driver, const char *uri, + const struct dict_settings *set, + struct dict **dict_r, const char **error_r) +{ + struct fs_settings fs_set; + struct fs *fs; + struct fs_dict *dict; + const char *p, *fs_driver, *fs_args; + + p = strchr(uri, ':'); + if (p == NULL) { + fs_driver = uri; + fs_args = ""; + } else { + fs_driver = t_strdup_until(uri, p); + fs_args = p+1; + } + + i_zero(&fs_set); + fs_set.base_dir = set->base_dir; + if (fs_init(fs_driver, fs_args, &fs_set, &fs, error_r) < 0) + return -1; + + dict = i_new(struct fs_dict, 1); + dict->dict = *driver; + dict->fs = fs; + + *dict_r = &dict->dict; + return 0; +} + +static void fs_dict_deinit(struct dict *_dict) +{ + struct fs_dict *dict = (struct fs_dict *)_dict; + + fs_deinit(&dict->fs); + i_free(dict); +} + +/* Remove unsafe paths */ +static const char *fs_dict_escape_key(const char *key) +{ + const char *ptr; + string_t *new_key = NULL; + /* we take the slow path always if we see potential + need for escaping */ + while ((ptr = strstr(key, "/.")) != NULL) { + /* move to the first dot */ + const char *ptr2 = ptr + 1; + /* find position of non-dot */ + while (*ptr2 == '.') ptr2++; + if (new_key == NULL) + new_key = t_str_new(strlen(key)); + str_append_data(new_key, key, ptr - key); + /* if ptr2 is / or end of string, escape */ + if (*ptr2 == '/' || *ptr2 == '\0') + str_append(new_key, "/..."); + else + str_append(new_key, "/."); + key = ptr + 2; + } + if (new_key == NULL) + return key; + str_append(new_key, key); + return str_c(new_key); +} + +static const char *fs_dict_get_full_key(const char *username, const char *key) +{ + key = fs_dict_escape_key(key); + if (str_begins(key, DICT_PATH_SHARED)) + return key + strlen(DICT_PATH_SHARED); + else if (str_begins(key, DICT_PATH_PRIVATE)) { + return t_strdup_printf("%s/%s", username, + key + strlen(DICT_PATH_PRIVATE)); + } else { + i_unreached(); + } +} + +static int fs_dict_lookup(struct dict *_dict, const struct dict_op_settings *set, + pool_t pool, const char *key, + const char **value_r, const char **error_r) +{ + struct fs_dict *dict = (struct fs_dict *)_dict; + struct fs_file *file; + struct istream *input; + const unsigned char *data; + size_t size; + const char *path; + string_t *str; + int ret; + + path = fs_dict_get_full_key(set->username, key); + file = fs_file_init(dict->fs, path, FS_OPEN_MODE_READONLY); + input = fs_read_stream(file, IO_BLOCK_SIZE); + (void)i_stream_read(input); + + str = str_new(pool, i_stream_get_data_size(input)+1); + while ((ret = i_stream_read_more(input, &data, &size)) > 0) { + str_append_data(str, data, size); + i_stream_skip(input, size); + } + i_assert(ret == -1); + + if (input->stream_errno == 0) { + *value_r = str_c(str); + ret = 1; + } else { + *value_r = NULL; + if (input->stream_errno == ENOENT) + ret = 0; + else { + *error_r = t_strdup_printf("read(%s) failed: %s", + path, i_stream_get_error(input)); + } + } + + i_stream_unref(&input); + fs_file_deinit(&file); + return ret; +} + +static struct dict_iterate_context * +fs_dict_iterate_init(struct dict *_dict, const struct dict_op_settings *set, + const char *path, enum dict_iterate_flags flags) +{ + struct fs_dict *dict = (struct fs_dict *)_dict; + struct fs_dict_iterate_context *iter; + + /* these flags are not supported for now */ + i_assert((flags & DICT_ITERATE_FLAG_RECURSE) == 0); + i_assert((flags & DICT_ITERATE_FLAG_EXACT_KEY) == 0); + i_assert((flags & (DICT_ITERATE_FLAG_SORT_BY_KEY | + DICT_ITERATE_FLAG_SORT_BY_VALUE)) == 0); + + iter = i_new(struct fs_dict_iterate_context, 1); + iter->ctx.dict = _dict; + iter->path = i_strdup(path); + iter->flags = flags; + iter->value_pool = pool_alloconly_create("iterate value pool", 128); + iter->fs_iter = fs_iter_init(dict->fs, + fs_dict_get_full_key(set->username, path), 0); + return &iter->ctx; +} + +static bool fs_dict_iterate(struct dict_iterate_context *ctx, + const char **key_r, const char *const **values_r) +{ + struct fs_dict_iterate_context *iter = + (struct fs_dict_iterate_context *)ctx; + struct fs_dict *dict = (struct fs_dict *)ctx->dict; + const char *path, *error; + int ret; + + if (iter->error != NULL) + return FALSE; + + *key_r = fs_iter_next(iter->fs_iter); + if (*key_r == NULL) { + if (fs_iter_deinit(&iter->fs_iter, &error) < 0) { + iter->error = i_strdup(error); + return FALSE; + } + if (iter->path == NULL) + return FALSE; + path = fs_dict_get_full_key(ctx->set.username, iter->path); + iter->fs_iter = fs_iter_init(dict->fs, path, 0); + return fs_dict_iterate(ctx, key_r, values_r); + } + path = t_strconcat(iter->path, *key_r, NULL); + if ((iter->flags & DICT_ITERATE_FLAG_NO_VALUE) != 0) { + iter->values[0] = NULL; + *key_r = path; + return TRUE; + } + p_clear(iter->value_pool); + struct dict_op_settings set = { + .username = ctx->set.username, + }; + ret = fs_dict_lookup(ctx->dict, &set, iter->value_pool, path, + &iter->values[0], &error); + if (ret < 0) { + /* I/O error */ + iter->error = i_strdup(error); + return FALSE; + } else if (ret == 0) { + /* file was just deleted, just skip to next one */ + return fs_dict_iterate(ctx, key_r, values_r); + } + *key_r = path; + *values_r = iter->values; + return TRUE; +} + +static int fs_dict_iterate_deinit(struct dict_iterate_context *ctx, + const char **error_r) +{ + struct fs_dict_iterate_context *iter = + (struct fs_dict_iterate_context *)ctx; + const char *error; + int ret; + + if (fs_iter_deinit(&iter->fs_iter, &error) < 0 && iter->error == NULL) + iter->error = i_strdup(error); + + ret = iter->error != NULL ? -1 : 0; + *error_r = t_strdup(iter->error); + + pool_unref(&iter->value_pool); + i_free(iter->path); + i_free(iter->error); + i_free(iter); + return ret; +} + +static struct dict_transaction_context * +fs_dict_transaction_init(struct dict *_dict) +{ + struct dict_transaction_memory_context *ctx; + pool_t pool; + + pool = pool_alloconly_create("file dict transaction", 2048); + ctx = p_new(pool, struct dict_transaction_memory_context, 1); + dict_transaction_memory_init(ctx, _dict, pool); + return &ctx->ctx; +} + +static int fs_dict_write_changes(struct dict_transaction_memory_context *ctx, + const char **error_r) +{ + struct fs_dict *dict = (struct fs_dict *)ctx->ctx.dict; + struct fs_file *file; + const struct dict_transaction_memory_change *change; + const char *key; + int ret = 0; + + array_foreach(&ctx->changes, change) { + key = fs_dict_get_full_key(ctx->ctx.set.username, change->key); + switch (change->type) { + case DICT_CHANGE_TYPE_SET: + file = fs_file_init(dict->fs, key, + FS_OPEN_MODE_REPLACE); + if (fs_write(file, change->value.str, strlen(change->value.str)) < 0) { + *error_r = t_strdup_printf( + "fs_write(%s) failed: %s", key, + fs_file_last_error(file)); + ret = -1; + } + fs_file_deinit(&file); + break; + case DICT_CHANGE_TYPE_UNSET: + file = fs_file_init(dict->fs, key, FS_OPEN_MODE_READONLY); + if (fs_delete(file) < 0) { + *error_r = t_strdup_printf( + "fs_delete(%s) failed: %s", key, + fs_file_last_error(file)); + ret = -1; + } + fs_file_deinit(&file); + break; + case DICT_CHANGE_TYPE_INC: + i_unreached(); + } + if (ret < 0) + return -1; + } + return 0; +} + +static void +fs_dict_transaction_commit(struct dict_transaction_context *_ctx, + bool async ATTR_UNUSED, + dict_transaction_commit_callback_t *callback, + void *context) +{ + struct dict_transaction_memory_context *ctx = + (struct dict_transaction_memory_context *)_ctx; + struct dict_commit_result result = { .ret = 1 }; + + + if (fs_dict_write_changes(ctx, &result.error) < 0) + result.ret = -1; + pool_unref(&ctx->pool); + + callback(&result, context); +} + +struct dict dict_driver_fs = { + .name = "fs", + { + .init = fs_dict_init, + .deinit = fs_dict_deinit, + .lookup = fs_dict_lookup, + .iterate_init = fs_dict_iterate_init, + .iterate = fs_dict_iterate, + .iterate_deinit = fs_dict_iterate_deinit, + .transaction_init = fs_dict_transaction_init, + .transaction_commit = fs_dict_transaction_commit, + .transaction_rollback = dict_transaction_memory_rollback, + .set = dict_transaction_memory_set, + .unset = dict_transaction_memory_unset, + } +}; diff --git a/src/lib-dict-extra/dict-register.c b/src/lib-dict-extra/dict-register.c new file mode 100644 index 0000000..96d2bef --- /dev/null +++ b/src/lib-dict-extra/dict-register.c @@ -0,0 +1,30 @@ +/* Copyright (c) 2013-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "dict-private.h" + +static int refcount = 0; + +void dict_drivers_register_builtin(void) +{ + if (refcount++ > 0) + return; + dict_driver_register(&dict_driver_client); + dict_driver_register(&dict_driver_file); + dict_driver_register(&dict_driver_fs); + dict_driver_register(&dict_driver_memcached); + dict_driver_register(&dict_driver_memcached_ascii); + dict_driver_register(&dict_driver_redis); +} + +void dict_drivers_unregister_builtin(void) +{ + if (--refcount > 0) + return; + dict_driver_unregister(&dict_driver_client); + dict_driver_unregister(&dict_driver_file); + dict_driver_unregister(&dict_driver_fs); + dict_driver_unregister(&dict_driver_memcached); + dict_driver_unregister(&dict_driver_memcached_ascii); + dict_driver_unregister(&dict_driver_redis); +} diff --git a/src/lib-dict-extra/test-dict-fs.c b/src/lib-dict-extra/test-dict-fs.c new file mode 100644 index 0000000..a7b9905 --- /dev/null +++ b/src/lib-dict-extra/test-dict-fs.c @@ -0,0 +1,106 @@ +#include <errno.h> +#include <sys/stat.h> +#include "lib.h" +#include "unlink-directory.h" +#include "test-common.h" +#include "dict-private.h" + +static void test_dict_set_get(struct dict *dict, const char *key, + const char *value) +{ + const char *got_value, *error; + struct dict_op_settings set = { + .username = "testuser", + }; + struct dict_transaction_context *t = dict_transaction_begin(dict, &set); + dict_set(t, key, value); + if (dict_transaction_commit(&t, &error) < 0) + i_fatal("dict_transaction_commit(%s) failed: %s", key, error); + if (dict_lookup(dict, &set, pool_datastack_create(), key, &got_value, + &error) < 0) + i_fatal("dict_lookup(%s) failed: %s", key, error); + test_assert_strcmp(got_value, value); +} + +static bool test_file_exists(const char *path) +{ + struct stat st; + if (stat(path, &st) < 0) { + if (ENOTFOUND(errno)) return FALSE; + i_fatal("stat(%s) failed: %m", path); + } + return TRUE; +} + +static void test_dict_fs_set_get(void) +{ + test_begin("dict-fs get/set"); + const char *error; + struct dict *dict; + struct dict_settings set = { + .base_dir = ".", + }; + if (dict_init("fs:posix:prefix=.test-dict/", &set, &dict, &error) < 0) + i_fatal("dict_init() failed: %s", error); + + /* shared paths */ + struct { + const char *key; + const char *path; + } test_cases[] = { + { "shared/./key", ".test-dict/.../key" }, + { "shared/../key", ".test-dict/..../key" }, + { "shared/.../key", ".test-dict/...../key" }, + { "shared/..../key", ".test-dict/....../key" }, + { "shared/...../key", ".test-dict/......./key" }, + { "shared/key/.", ".test-dict/key/..." }, + { "shared/key/..", ".test-dict/key/...." }, + { "shared/key/...", ".test-dict/key/....." }, + { "shared/key/....", ".test-dict/key/......" }, + { "shared/key/.....", ".test-dict/key/......." }, + { "shared/key/.key", ".test-dict/key/.key" }, + { "shared/key/..key", ".test-dict/key/..key" }, + { "shared/key/...key", ".test-dict/key/...key" }, + { "shared/.key/key", ".test-dict/.key/key" }, + { "shared/..key/key", ".test-dict/..key/key" }, + { "shared/...key/key", ".test-dict/...key/key" }, + }; + for (size_t i = 0; i < N_ELEMENTS(test_cases); i++) { + test_dict_set_get(dict, test_cases[i].key, "1"); + test_assert(test_file_exists(test_cases[i].path)); + } + + /* per user paths */ + test_dict_set_get(dict, "priv/value", "priv1"); + test_assert(test_file_exists(".test-dict/testuser/value")); + test_dict_set_get(dict, "priv/path/with/value", "priv2"); + test_assert(test_file_exists(".test-dict/testuser/path/with/value")); + + /* check that dots work correctly */ + test_dict_set_get(dict, "shared/../test-dict-fs.c", "3"); + test_assert(test_file_exists(".test-dict/..../test-dict-fs.c")); + test_dict_set_get(dict, "shared/./test", "4"); + test_assert(test_file_exists(".test-dict/.../test")); + test_dict_set_get(dict, "shared/.test", "5"); + test_assert(test_file_exists(".test-dict/.test")); + test_dict_set_get(dict, "shared/..test", "6"); + test_assert(test_file_exists(".test-dict/..test")); + dict_deinit(&dict); + + if (unlink_directory(".test-dict", UNLINK_DIRECTORY_FLAG_RMDIR, &error) < 0) + i_fatal("unlink_directory(.test_dict) failed: %s", error); + test_end(); +} + +int main(void) +{ + static void (*const test_functions[])(void) = { + test_dict_fs_set_get, + NULL + }; + int ret; + dict_driver_register(&dict_driver_fs); + ret = test_run(test_functions); + dict_driver_unregister(&dict_driver_fs); + return ret; +} diff --git a/src/lib-dict/Makefile.am b/src/lib-dict/Makefile.am new file mode 100644 index 0000000..97e5eb6 --- /dev/null +++ b/src/lib-dict/Makefile.am @@ -0,0 +1,73 @@ +noinst_LTLIBRARIES = \ + libdict.la + +AM_CPPFLAGS = \ + -I$(top_srcdir)/src/lib \ + -I$(top_srcdir)/src/lib-test \ + -I$(top_srcdir)/src/lib-settings + +base_sources = \ + dict.c \ + dict-client.c \ + dict-file.c \ + dict-memcached.c \ + dict-memcached-ascii.c \ + dict-redis.c \ + dict-fail.c \ + dict-transaction-memory.c + +libdict_la_SOURCES = \ + $(base_sources) + +headers = \ + dict.h \ + dict-client.h \ + dict-private.h \ + dict-transaction-memory.h + +# Internally, the dict methods yield via lua_yieldk() as implemented in Lua +# 5.3 and newer. +if DLUA_WITH_YIELDS +noinst_LTLIBRARIES += libdict_lua.la + +libdict_lua_la_SOURCES = \ + dict-lua.c \ + dict-iter-lua.c \ + dict-txn-lua.c +libdict_lua_la_CPPFLAGS = \ + $(AM_CPPFLAGS) \ + $(LUA_CFLAGS) \ + -I$(top_srcdir)/src/lib-lua +libdict_lua_la_LIBADD = +libdict_lua_la_DEPENDENCIES = \ + libdict.la + +headers += \ + dict-lua.h \ + dict-lua-private.h +endif + +pkginc_libdir=$(pkgincludedir) +pkginc_lib_HEADERS = $(headers) + +test_programs = \ + test-dict + +noinst_PROGRAMS = $(test_programs) test-dict-client + +test_libs = \ + ../lib-test/libtest.la \ + ../lib/liblib.la + +test_dict_SOURCES = test-dict.c +test_dict_LDADD = libdict.la $(test_libs) +test_dict_DEPENDENCIES = $(noinst_LTLIBRARIES) $(test_libs) + +test_dict_client_SOURCES = test-dict-client.c +test_dict_client_LDADD = $(noinst_LTLIBRARIES) ../lib/liblib.la +test_dict_client_DEPENDENCIES = $(noinst_LTLIBRARIES) $(test_libs) + +check-local: + for bin in $(test_programs) $(check_PROGRAMS); do \ + if ! $(RUN_TEST) ./$$bin; then exit 1; fi; \ + done diff --git a/src/lib-dict/Makefile.in b/src/lib-dict/Makefile.in new file mode 100644 index 0000000..aa8f17e --- /dev/null +++ b/src/lib-dict/Makefile.in @@ -0,0 +1,970 @@ +# Makefile.in generated by automake 1.16.1 from Makefile.am. +# @configure_input@ + +# Copyright (C) 1994-2018 Free Software Foundation, Inc. + +# This Makefile.in is free software; the Free Software Foundation +# gives unlimited permission to copy and/or distribute it, +# with or without modifications, as long as this notice is preserved. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY, to the extent permitted by law; without +# even the implied warranty of MERCHANTABILITY or FITNESS FOR A +# PARTICULAR PURPOSE. + +@SET_MAKE@ + + + +VPATH = @srcdir@ +am__is_gnu_make = { \ + if test -z '$(MAKELEVEL)'; then \ + false; \ + elif test -n '$(MAKE_HOST)'; then \ + true; \ + elif test -n '$(MAKE_VERSION)' && test -n '$(CURDIR)'; then \ + true; \ + else \ + false; \ + fi; \ +} +am__make_running_with_option = \ + case $${target_option-} in \ + ?) ;; \ + *) echo "am__make_running_with_option: internal error: invalid" \ + "target option '$${target_option-}' specified" >&2; \ + exit 1;; \ + esac; \ + has_opt=no; \ + sane_makeflags=$$MAKEFLAGS; \ + if $(am__is_gnu_make); then \ + sane_makeflags=$$MFLAGS; \ + else \ + case $$MAKEFLAGS in \ + *\\[\ \ ]*) \ + bs=\\; \ + sane_makeflags=`printf '%s\n' "$$MAKEFLAGS" \ + | sed "s/$$bs$$bs[$$bs $$bs ]*//g"`;; \ + esac; \ + fi; \ + skip_next=no; \ + strip_trailopt () \ + { \ + flg=`printf '%s\n' "$$flg" | sed "s/$$1.*$$//"`; \ + }; \ + for flg in $$sane_makeflags; do \ + test $$skip_next = yes && { skip_next=no; continue; }; \ + case $$flg in \ + *=*|--*) continue;; \ + -*I) strip_trailopt 'I'; skip_next=yes;; \ + -*I?*) strip_trailopt 'I';; \ + -*O) strip_trailopt 'O'; skip_next=yes;; \ + -*O?*) strip_trailopt 'O';; \ + -*l) strip_trailopt 'l'; skip_next=yes;; \ + -*l?*) strip_trailopt 'l';; \ + -[dEDm]) skip_next=yes;; \ + -[JT]) skip_next=yes;; \ + esac; \ + case $$flg in \ + *$$target_option*) has_opt=yes; break;; \ + esac; \ + done; \ + test $$has_opt = yes +am__make_dryrun = (target_option=n; $(am__make_running_with_option)) +am__make_keepgoing = (target_option=k; $(am__make_running_with_option)) +pkgdatadir = $(datadir)/@PACKAGE@ +pkgincludedir = $(includedir)/@PACKAGE@ +pkglibdir = $(libdir)/@PACKAGE@ +pkglibexecdir = $(libexecdir)/@PACKAGE@ +am__cd = CDPATH="$${ZSH_VERSION+.}$(PATH_SEPARATOR)" && cd +install_sh_DATA = $(install_sh) -c -m 644 +install_sh_PROGRAM = $(install_sh) -c +install_sh_SCRIPT = $(install_sh) -c +INSTALL_HEADER = $(INSTALL_DATA) +transform = $(program_transform_name) +NORMAL_INSTALL = : +PRE_INSTALL = : +POST_INSTALL = : +NORMAL_UNINSTALL = : +PRE_UNINSTALL = : +POST_UNINSTALL = : +build_triplet = @build@ +host_triplet = @host@ + +# Internally, the dict methods yield via lua_yieldk() as implemented in Lua +# 5.3 and newer. +@DLUA_WITH_YIELDS_TRUE@am__append_1 = libdict_lua.la +@DLUA_WITH_YIELDS_FALSE@libdict_lua_la_DEPENDENCIES = +@DLUA_WITH_YIELDS_TRUE@am__append_2 = \ +@DLUA_WITH_YIELDS_TRUE@ dict-lua.h \ +@DLUA_WITH_YIELDS_TRUE@ dict-lua-private.h + +noinst_PROGRAMS = $(am__EXEEXT_1) test-dict-client$(EXEEXT) +subdir = src/lib-dict +ACLOCAL_M4 = $(top_srcdir)/aclocal.m4 +am__aclocal_m4_deps = $(top_srcdir)/m4/ac_checktype2.m4 \ + $(top_srcdir)/m4/ac_typeof.m4 $(top_srcdir)/m4/arc4random.m4 \ + $(top_srcdir)/m4/blockdev.m4 $(top_srcdir)/m4/c99_vsnprintf.m4 \ + $(top_srcdir)/m4/clock_gettime.m4 $(top_srcdir)/m4/crypt.m4 \ + $(top_srcdir)/m4/crypt_xpg6.m4 $(top_srcdir)/m4/dbqlk.m4 \ + $(top_srcdir)/m4/dirent_dtype.m4 $(top_srcdir)/m4/dovecot.m4 \ + $(top_srcdir)/m4/fd_passing.m4 $(top_srcdir)/m4/fdatasync.m4 \ + $(top_srcdir)/m4/flexible_array_member.m4 \ + $(top_srcdir)/m4/glibc.m4 $(top_srcdir)/m4/gmtime_max.m4 \ + $(top_srcdir)/m4/gmtime_tm_gmtoff.m4 \ + $(top_srcdir)/m4/ioloop.m4 $(top_srcdir)/m4/iovec.m4 \ + $(top_srcdir)/m4/ipv6.m4 $(top_srcdir)/m4/libcap.m4 \ + $(top_srcdir)/m4/libtool.m4 $(top_srcdir)/m4/libwrap.m4 \ + $(top_srcdir)/m4/linux_mremap.m4 $(top_srcdir)/m4/ltoptions.m4 \ + $(top_srcdir)/m4/ltsugar.m4 $(top_srcdir)/m4/ltversion.m4 \ + $(top_srcdir)/m4/lt~obsolete.m4 $(top_srcdir)/m4/mmap_write.m4 \ + $(top_srcdir)/m4/mntctl.m4 $(top_srcdir)/m4/modules.m4 \ + $(top_srcdir)/m4/notify.m4 $(top_srcdir)/m4/nsl.m4 \ + $(top_srcdir)/m4/off_t_max.m4 $(top_srcdir)/m4/pkg.m4 \ + $(top_srcdir)/m4/pr_set_dumpable.m4 \ + $(top_srcdir)/m4/q_quotactl.m4 $(top_srcdir)/m4/quota.m4 \ + $(top_srcdir)/m4/random.m4 $(top_srcdir)/m4/rlimit.m4 \ + $(top_srcdir)/m4/sendfile.m4 $(top_srcdir)/m4/size_t_signed.m4 \ + $(top_srcdir)/m4/sockpeercred.m4 $(top_srcdir)/m4/sql.m4 \ + $(top_srcdir)/m4/ssl.m4 $(top_srcdir)/m4/st_tim.m4 \ + $(top_srcdir)/m4/static_array.m4 $(top_srcdir)/m4/test_with.m4 \ + $(top_srcdir)/m4/time_t.m4 $(top_srcdir)/m4/typeof.m4 \ + $(top_srcdir)/m4/typeof_dev_t.m4 \ + $(top_srcdir)/m4/uoff_t_max.m4 $(top_srcdir)/m4/vararg.m4 \ + $(top_srcdir)/m4/want_apparmor.m4 \ + $(top_srcdir)/m4/want_bsdauth.m4 \ + $(top_srcdir)/m4/want_bzlib.m4 \ + $(top_srcdir)/m4/want_cassandra.m4 \ + $(top_srcdir)/m4/want_cdb.m4 \ + $(top_srcdir)/m4/want_checkpassword.m4 \ + $(top_srcdir)/m4/want_clucene.m4 $(top_srcdir)/m4/want_db.m4 \ + $(top_srcdir)/m4/want_gssapi.m4 $(top_srcdir)/m4/want_icu.m4 \ + $(top_srcdir)/m4/want_ldap.m4 $(top_srcdir)/m4/want_lua.m4 \ + $(top_srcdir)/m4/want_lz4.m4 $(top_srcdir)/m4/want_lzma.m4 \ + $(top_srcdir)/m4/want_mysql.m4 $(top_srcdir)/m4/want_pam.m4 \ + $(top_srcdir)/m4/want_passwd.m4 $(top_srcdir)/m4/want_pgsql.m4 \ + $(top_srcdir)/m4/want_prefetch.m4 \ + $(top_srcdir)/m4/want_shadow.m4 \ + $(top_srcdir)/m4/want_sodium.m4 $(top_srcdir)/m4/want_solr.m4 \ + $(top_srcdir)/m4/want_sqlite.m4 \ + $(top_srcdir)/m4/want_stemmer.m4 \ + $(top_srcdir)/m4/want_systemd.m4 \ + $(top_srcdir)/m4/want_textcat.m4 \ + $(top_srcdir)/m4/want_unwind.m4 $(top_srcdir)/m4/want_zlib.m4 \ + $(top_srcdir)/m4/want_zstd.m4 $(top_srcdir)/configure.ac +am__configure_deps = $(am__aclocal_m4_deps) $(CONFIGURE_DEPENDENCIES) \ + $(ACLOCAL_M4) +DIST_COMMON = $(srcdir)/Makefile.am $(am__pkginc_lib_HEADERS_DIST) \ + $(am__DIST_COMMON) +mkinstalldirs = $(install_sh) -d +CONFIG_HEADER = $(top_builddir)/config.h +CONFIG_CLEAN_FILES = +CONFIG_CLEAN_VPATH_FILES = +am__EXEEXT_1 = test-dict$(EXEEXT) +PROGRAMS = $(noinst_PROGRAMS) +LTLIBRARIES = $(noinst_LTLIBRARIES) +libdict_la_LIBADD = +am__objects_1 = dict.lo dict-client.lo dict-file.lo dict-memcached.lo \ + dict-memcached-ascii.lo dict-redis.lo dict-fail.lo \ + dict-transaction-memory.lo +am_libdict_la_OBJECTS = $(am__objects_1) +libdict_la_OBJECTS = $(am_libdict_la_OBJECTS) +AM_V_lt = $(am__v_lt_@AM_V@) +am__v_lt_ = $(am__v_lt_@AM_DEFAULT_V@) +am__v_lt_0 = --silent +am__v_lt_1 = +am__libdict_lua_la_SOURCES_DIST = dict-lua.c dict-iter-lua.c \ + dict-txn-lua.c +@DLUA_WITH_YIELDS_TRUE@am_libdict_lua_la_OBJECTS = \ +@DLUA_WITH_YIELDS_TRUE@ libdict_lua_la-dict-lua.lo \ +@DLUA_WITH_YIELDS_TRUE@ libdict_lua_la-dict-iter-lua.lo \ +@DLUA_WITH_YIELDS_TRUE@ libdict_lua_la-dict-txn-lua.lo +libdict_lua_la_OBJECTS = $(am_libdict_lua_la_OBJECTS) +@DLUA_WITH_YIELDS_TRUE@am_libdict_lua_la_rpath = +am_test_dict_OBJECTS = test-dict.$(OBJEXT) +test_dict_OBJECTS = $(am_test_dict_OBJECTS) +am_test_dict_client_OBJECTS = test-dict-client.$(OBJEXT) +test_dict_client_OBJECTS = $(am_test_dict_client_OBJECTS) +AM_V_P = $(am__v_P_@AM_V@) +am__v_P_ = $(am__v_P_@AM_DEFAULT_V@) +am__v_P_0 = false +am__v_P_1 = : +AM_V_GEN = $(am__v_GEN_@AM_V@) +am__v_GEN_ = $(am__v_GEN_@AM_DEFAULT_V@) +am__v_GEN_0 = @echo " GEN " $@; +am__v_GEN_1 = +AM_V_at = $(am__v_at_@AM_V@) +am__v_at_ = $(am__v_at_@AM_DEFAULT_V@) +am__v_at_0 = @ +am__v_at_1 = +DEFAULT_INCLUDES = -I.@am__isrc@ -I$(top_builddir) +depcomp = $(SHELL) $(top_srcdir)/depcomp +am__maybe_remake_depfiles = depfiles +am__depfiles_remade = ./$(DEPDIR)/dict-client.Plo \ + ./$(DEPDIR)/dict-fail.Plo ./$(DEPDIR)/dict-file.Plo \ + ./$(DEPDIR)/dict-memcached-ascii.Plo \ + ./$(DEPDIR)/dict-memcached.Plo ./$(DEPDIR)/dict-redis.Plo \ + ./$(DEPDIR)/dict-transaction-memory.Plo ./$(DEPDIR)/dict.Plo \ + ./$(DEPDIR)/libdict_lua_la-dict-iter-lua.Plo \ + ./$(DEPDIR)/libdict_lua_la-dict-lua.Plo \ + ./$(DEPDIR)/libdict_lua_la-dict-txn-lua.Plo \ + ./$(DEPDIR)/test-dict-client.Po ./$(DEPDIR)/test-dict.Po +am__mv = mv -f +COMPILE = $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) \ + $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) +LTCOMPILE = $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) \ + $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) \ + $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) \ + $(AM_CFLAGS) $(CFLAGS) +AM_V_CC = $(am__v_CC_@AM_V@) +am__v_CC_ = $(am__v_CC_@AM_DEFAULT_V@) +am__v_CC_0 = @echo " CC " $@; +am__v_CC_1 = +CCLD = $(CC) +LINK = $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) \ + $(LIBTOOLFLAGS) --mode=link $(CCLD) $(AM_CFLAGS) $(CFLAGS) \ + $(AM_LDFLAGS) $(LDFLAGS) -o $@ +AM_V_CCLD = $(am__v_CCLD_@AM_V@) +am__v_CCLD_ = $(am__v_CCLD_@AM_DEFAULT_V@) +am__v_CCLD_0 = @echo " CCLD " $@; +am__v_CCLD_1 = +SOURCES = $(libdict_la_SOURCES) $(libdict_lua_la_SOURCES) \ + $(test_dict_SOURCES) $(test_dict_client_SOURCES) +DIST_SOURCES = $(libdict_la_SOURCES) \ + $(am__libdict_lua_la_SOURCES_DIST) $(test_dict_SOURCES) \ + $(test_dict_client_SOURCES) +am__can_run_installinfo = \ + case $$AM_UPDATE_INFO_DIR in \ + n|no|NO) false;; \ + *) (install-info --version) >/dev/null 2>&1;; \ + esac +am__pkginc_lib_HEADERS_DIST = dict.h dict-client.h dict-private.h \ + dict-transaction-memory.h dict-lua.h dict-lua-private.h +am__vpath_adj_setup = srcdirstrip=`echo "$(srcdir)" | sed 's|.|.|g'`; +am__vpath_adj = case $$p in \ + $(srcdir)/*) f=`echo "$$p" | sed "s|^$$srcdirstrip/||"`;; \ + *) f=$$p;; \ + esac; +am__strip_dir = f=`echo $$p | sed -e 's|^.*/||'`; +am__install_max = 40 +am__nobase_strip_setup = \ + srcdirstrip=`echo "$(srcdir)" | sed 's/[].[^$$\\*|]/\\\\&/g'` +am__nobase_strip = \ + for p in $$list; do echo "$$p"; done | sed -e "s|$$srcdirstrip/||" +am__nobase_list = $(am__nobase_strip_setup); \ + for p in $$list; do echo "$$p $$p"; done | \ + sed "s| $$srcdirstrip/| |;"' / .*\//!s/ .*/ ./; s,\( .*\)/[^/]*$$,\1,' | \ + $(AWK) 'BEGIN { files["."] = "" } { files[$$2] = files[$$2] " " $$1; \ + if (++n[$$2] == $(am__install_max)) \ + { print $$2, files[$$2]; n[$$2] = 0; files[$$2] = "" } } \ + END { for (dir in files) print dir, files[dir] }' +am__base_list = \ + sed '$$!N;$$!N;$$!N;$$!N;$$!N;$$!N;$$!N;s/\n/ /g' | \ + sed '$$!N;$$!N;$$!N;$$!N;s/\n/ /g' +am__uninstall_files_from_dir = { \ + test -z "$$files" \ + || { test ! -d "$$dir" && test ! -f "$$dir" && test ! -r "$$dir"; } \ + || { echo " ( cd '$$dir' && rm -f" $$files ")"; \ + $(am__cd) "$$dir" && rm -f $$files; }; \ + } +am__installdirs = "$(DESTDIR)$(pkginc_libdir)" +HEADERS = $(pkginc_lib_HEADERS) +am__tagged_files = $(HEADERS) $(SOURCES) $(TAGS_FILES) $(LISP) +# Read a list of newline-separated strings from the standard input, +# and print each of them once, without duplicates. Input order is +# *not* preserved. +am__uniquify_input = $(AWK) '\ + BEGIN { nonempty = 0; } \ + { items[$$0] = 1; nonempty = 1; } \ + END { if (nonempty) { for (i in items) print i; }; } \ +' +# Make sure the list of sources is unique. This is necessary because, +# e.g., the same source file might be shared among _SOURCES variables +# for different programs/libraries. +am__define_uniq_tagged_files = \ + list='$(am__tagged_files)'; \ + unique=`for i in $$list; do \ + if test -f "$$i"; then echo $$i; else echo $(srcdir)/$$i; fi; \ + done | $(am__uniquify_input)` +ETAGS = etags +CTAGS = ctags +am__DIST_COMMON = $(srcdir)/Makefile.in $(top_srcdir)/depcomp +DISTFILES = $(DIST_COMMON) $(DIST_SOURCES) $(TEXINFOS) $(EXTRA_DIST) +ACLOCAL = @ACLOCAL@ +ACLOCAL_AMFLAGS = @ACLOCAL_AMFLAGS@ +AMTAR = @AMTAR@ +AM_DEFAULT_VERBOSITY = @AM_DEFAULT_VERBOSITY@ +APPARMOR_LIBS = @APPARMOR_LIBS@ +AR = @AR@ +AUTH_CFLAGS = @AUTH_CFLAGS@ +AUTH_LIBS = @AUTH_LIBS@ +AUTOCONF = @AUTOCONF@ +AUTOHEADER = @AUTOHEADER@ +AUTOMAKE = @AUTOMAKE@ +AWK = @AWK@ +BINARY_CFLAGS = @BINARY_CFLAGS@ +BINARY_LDFLAGS = @BINARY_LDFLAGS@ +BISON = @BISON@ +CASSANDRA_CFLAGS = @CASSANDRA_CFLAGS@ +CASSANDRA_LIBS = @CASSANDRA_LIBS@ +CC = @CC@ +CCDEPMODE = @CCDEPMODE@ +CDB_LIBS = @CDB_LIBS@ +CFLAGS = @CFLAGS@ +CLUCENE_CFLAGS = @CLUCENE_CFLAGS@ +CLUCENE_LIBS = @CLUCENE_LIBS@ +COMPRESS_LIBS = @COMPRESS_LIBS@ +CPP = @CPP@ +CPPFLAGS = @CPPFLAGS@ +CRYPT_LIBS = @CRYPT_LIBS@ +CXX = @CXX@ +CXXCPP = @CXXCPP@ +CXXDEPMODE = @CXXDEPMODE@ +CXXFLAGS = @CXXFLAGS@ +CYGPATH_W = @CYGPATH_W@ +DEFS = @DEFS@ +DEPDIR = @DEPDIR@ +DICT_LIBS = @DICT_LIBS@ +DLLIB = @DLLIB@ +DLLTOOL = @DLLTOOL@ +DSYMUTIL = @DSYMUTIL@ +DUMPBIN = @DUMPBIN@ +ECHO_C = @ECHO_C@ +ECHO_N = @ECHO_N@ +ECHO_T = @ECHO_T@ +EGREP = @EGREP@ +EXEEXT = @EXEEXT@ +FGREP = @FGREP@ +FLEX = @FLEX@ +FUZZER_CPPFLAGS = @FUZZER_CPPFLAGS@ +FUZZER_LDFLAGS = @FUZZER_LDFLAGS@ +GREP = @GREP@ +INSTALL = @INSTALL@ +INSTALL_DATA = @INSTALL_DATA@ +INSTALL_PROGRAM = @INSTALL_PROGRAM@ +INSTALL_SCRIPT = @INSTALL_SCRIPT@ +INSTALL_STRIP_PROGRAM = @INSTALL_STRIP_PROGRAM@ +KRB5CONFIG = @KRB5CONFIG@ +KRB5_CFLAGS = @KRB5_CFLAGS@ +KRB5_LIBS = @KRB5_LIBS@ +LD = @LD@ +LDAP_LIBS = @LDAP_LIBS@ +LDFLAGS = @LDFLAGS@ +LD_NO_WHOLE_ARCHIVE = @LD_NO_WHOLE_ARCHIVE@ +LD_WHOLE_ARCHIVE = @LD_WHOLE_ARCHIVE@ +LIBCAP = @LIBCAP@ +LIBDOVECOT = @LIBDOVECOT@ +LIBDOVECOT_COMPRESS = @LIBDOVECOT_COMPRESS@ +LIBDOVECOT_DEPS = @LIBDOVECOT_DEPS@ +LIBDOVECOT_DSYNC = @LIBDOVECOT_DSYNC@ +LIBDOVECOT_LA_LIBS = @LIBDOVECOT_LA_LIBS@ +LIBDOVECOT_LDA = @LIBDOVECOT_LDA@ +LIBDOVECOT_LDAP = @LIBDOVECOT_LDAP@ +LIBDOVECOT_LIBFTS = @LIBDOVECOT_LIBFTS@ +LIBDOVECOT_LIBFTS_DEPS = @LIBDOVECOT_LIBFTS_DEPS@ +LIBDOVECOT_LOGIN = @LIBDOVECOT_LOGIN@ +LIBDOVECOT_LUA = @LIBDOVECOT_LUA@ +LIBDOVECOT_LUA_DEPS = @LIBDOVECOT_LUA_DEPS@ +LIBDOVECOT_SQL = @LIBDOVECOT_SQL@ +LIBDOVECOT_STORAGE = @LIBDOVECOT_STORAGE@ +LIBDOVECOT_STORAGE_DEPS = @LIBDOVECOT_STORAGE_DEPS@ +LIBEXTTEXTCAT_CFLAGS = @LIBEXTTEXTCAT_CFLAGS@ +LIBEXTTEXTCAT_LIBS = @LIBEXTTEXTCAT_LIBS@ +LIBICONV = @LIBICONV@ +LIBICU_CFLAGS = @LIBICU_CFLAGS@ +LIBICU_LIBS = @LIBICU_LIBS@ +LIBOBJS = @LIBOBJS@ +LIBS = @LIBS@ +LIBSODIUM_CFLAGS = @LIBSODIUM_CFLAGS@ +LIBSODIUM_LIBS = @LIBSODIUM_LIBS@ +LIBTIRPC_CFLAGS = @LIBTIRPC_CFLAGS@ +LIBTIRPC_LIBS = @LIBTIRPC_LIBS@ +LIBTOOL = @LIBTOOL@ +LIBUNWIND_CFLAGS = @LIBUNWIND_CFLAGS@ +LIBUNWIND_LIBS = @LIBUNWIND_LIBS@ +LIBWRAP_LIBS = @LIBWRAP_LIBS@ +LINKED_STORAGE_LDADD = @LINKED_STORAGE_LDADD@ +LIPO = @LIPO@ +LN_S = @LN_S@ +LTLIBICONV = @LTLIBICONV@ +LTLIBOBJS = @LTLIBOBJS@ +LT_SYS_LIBRARY_PATH = @LT_SYS_LIBRARY_PATH@ +LUA_CFLAGS = @LUA_CFLAGS@ +LUA_LIBS = @LUA_LIBS@ +MAINT = @MAINT@ +MAKEINFO = @MAKEINFO@ +MANIFEST_TOOL = @MANIFEST_TOOL@ +MKDIR_P = @MKDIR_P@ +MODULE_LIBS = @MODULE_LIBS@ +MODULE_SUFFIX = @MODULE_SUFFIX@ +MYSQL_CFLAGS = @MYSQL_CFLAGS@ +MYSQL_CONFIG = @MYSQL_CONFIG@ +MYSQL_LIBS = @MYSQL_LIBS@ +NM = @NM@ +NMEDIT = @NMEDIT@ +NOPLUGIN_LDFLAGS = @NOPLUGIN_LDFLAGS@ +OBJDUMP = @OBJDUMP@ +OBJEXT = @OBJEXT@ +OTOOL = @OTOOL@ +OTOOL64 = @OTOOL64@ +PACKAGE = @PACKAGE@ +PACKAGE_BUGREPORT = @PACKAGE_BUGREPORT@ +PACKAGE_NAME = @PACKAGE_NAME@ +PACKAGE_STRING = @PACKAGE_STRING@ +PACKAGE_TARNAME = @PACKAGE_TARNAME@ +PACKAGE_URL = @PACKAGE_URL@ +PACKAGE_VERSION = @PACKAGE_VERSION@ +PANDOC = @PANDOC@ +PATH_SEPARATOR = @PATH_SEPARATOR@ +PGSQL_CFLAGS = @PGSQL_CFLAGS@ +PGSQL_LIBS = @PGSQL_LIBS@ +PG_CONFIG = @PG_CONFIG@ +PIE_CFLAGS = @PIE_CFLAGS@ +PIE_LDFLAGS = @PIE_LDFLAGS@ +PKG_CONFIG = @PKG_CONFIG@ +PKG_CONFIG_LIBDIR = @PKG_CONFIG_LIBDIR@ +PKG_CONFIG_PATH = @PKG_CONFIG_PATH@ +QUOTA_LIBS = @QUOTA_LIBS@ +RANLIB = @RANLIB@ +RELRO_LDFLAGS = @RELRO_LDFLAGS@ +RPCGEN = @RPCGEN@ +RUN_TEST = @RUN_TEST@ +SED = @SED@ +SETTING_FILES = @SETTING_FILES@ +SET_MAKE = @SET_MAKE@ +SHELL = @SHELL@ +SQLITE_CFLAGS = @SQLITE_CFLAGS@ +SQLITE_LIBS = @SQLITE_LIBS@ +SQL_CFLAGS = @SQL_CFLAGS@ +SQL_LIBS = @SQL_LIBS@ +SSL_CFLAGS = @SSL_CFLAGS@ +SSL_LIBS = @SSL_LIBS@ +STRIP = @STRIP@ +SYSTEMD_CFLAGS = @SYSTEMD_CFLAGS@ +SYSTEMD_LIBS = @SYSTEMD_LIBS@ +VALGRIND = @VALGRIND@ +VERSION = @VERSION@ +ZSTD_CFLAGS = @ZSTD_CFLAGS@ +ZSTD_LIBS = @ZSTD_LIBS@ +abs_builddir = @abs_builddir@ +abs_srcdir = @abs_srcdir@ +abs_top_builddir = @abs_top_builddir@ +abs_top_srcdir = @abs_top_srcdir@ +ac_ct_AR = @ac_ct_AR@ +ac_ct_CC = @ac_ct_CC@ +ac_ct_CXX = @ac_ct_CXX@ +ac_ct_DUMPBIN = @ac_ct_DUMPBIN@ +am__include = @am__include@ +am__leading_dot = @am__leading_dot@ +am__quote = @am__quote@ +am__tar = @am__tar@ +am__untar = @am__untar@ +bindir = @bindir@ +build = @build@ +build_alias = @build_alias@ +build_cpu = @build_cpu@ +build_os = @build_os@ +build_vendor = @build_vendor@ +builddir = @builddir@ +datadir = @datadir@ +datarootdir = @datarootdir@ +dict_drivers = @dict_drivers@ +docdir = @docdir@ +dvidir = @dvidir@ +exec_prefix = @exec_prefix@ +host = @host@ +host_alias = @host_alias@ +host_cpu = @host_cpu@ +host_os = @host_os@ +host_vendor = @host_vendor@ +htmldir = @htmldir@ +includedir = @includedir@ +infodir = @infodir@ +install_sh = @install_sh@ +libdir = @libdir@ +libexecdir = @libexecdir@ +localedir = @localedir@ +localstatedir = @localstatedir@ +mandir = @mandir@ +mkdir_p = @mkdir_p@ +moduledir = @moduledir@ +oldincludedir = @oldincludedir@ +pdfdir = @pdfdir@ +prefix = @prefix@ +program_transform_name = @program_transform_name@ +psdir = @psdir@ +rundir = @rundir@ +runstatedir = @runstatedir@ +sbindir = @sbindir@ +sharedstatedir = @sharedstatedir@ +sql_drivers = @sql_drivers@ +srcdir = @srcdir@ +ssldir = @ssldir@ +statedir = @statedir@ +sysconfdir = @sysconfdir@ +systemdservicetype = @systemdservicetype@ +systemdsystemunitdir = @systemdsystemunitdir@ +target_alias = @target_alias@ +top_build_prefix = @top_build_prefix@ +top_builddir = @top_builddir@ +top_srcdir = @top_srcdir@ +noinst_LTLIBRARIES = libdict.la $(am__append_1) +AM_CPPFLAGS = \ + -I$(top_srcdir)/src/lib \ + -I$(top_srcdir)/src/lib-test \ + -I$(top_srcdir)/src/lib-settings + +base_sources = \ + dict.c \ + dict-client.c \ + dict-file.c \ + dict-memcached.c \ + dict-memcached-ascii.c \ + dict-redis.c \ + dict-fail.c \ + dict-transaction-memory.c + +libdict_la_SOURCES = \ + $(base_sources) + +headers = dict.h dict-client.h dict-private.h \ + dict-transaction-memory.h $(am__append_2) +@DLUA_WITH_YIELDS_TRUE@libdict_lua_la_SOURCES = \ +@DLUA_WITH_YIELDS_TRUE@ dict-lua.c \ +@DLUA_WITH_YIELDS_TRUE@ dict-iter-lua.c \ +@DLUA_WITH_YIELDS_TRUE@ dict-txn-lua.c + +@DLUA_WITH_YIELDS_TRUE@libdict_lua_la_CPPFLAGS = \ +@DLUA_WITH_YIELDS_TRUE@ $(AM_CPPFLAGS) \ +@DLUA_WITH_YIELDS_TRUE@ $(LUA_CFLAGS) \ +@DLUA_WITH_YIELDS_TRUE@ -I$(top_srcdir)/src/lib-lua + +@DLUA_WITH_YIELDS_TRUE@libdict_lua_la_LIBADD = +@DLUA_WITH_YIELDS_TRUE@libdict_lua_la_DEPENDENCIES = \ +@DLUA_WITH_YIELDS_TRUE@ libdict.la + +pkginc_libdir = $(pkgincludedir) +pkginc_lib_HEADERS = $(headers) +test_programs = \ + test-dict + +test_libs = \ + ../lib-test/libtest.la \ + ../lib/liblib.la + +test_dict_SOURCES = test-dict.c +test_dict_LDADD = libdict.la $(test_libs) +test_dict_DEPENDENCIES = $(noinst_LTLIBRARIES) $(test_libs) +test_dict_client_SOURCES = test-dict-client.c +test_dict_client_LDADD = $(noinst_LTLIBRARIES) ../lib/liblib.la +test_dict_client_DEPENDENCIES = $(noinst_LTLIBRARIES) $(test_libs) +all: all-am + +.SUFFIXES: +.SUFFIXES: .c .lo .o .obj +$(srcdir)/Makefile.in: @MAINTAINER_MODE_TRUE@ $(srcdir)/Makefile.am $(am__configure_deps) + @for dep in $?; do \ + case '$(am__configure_deps)' in \ + *$$dep*) \ + ( cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh ) \ + && { if test -f $@; then exit 0; else break; fi; }; \ + exit 1;; \ + esac; \ + done; \ + echo ' cd $(top_srcdir) && $(AUTOMAKE) --foreign src/lib-dict/Makefile'; \ + $(am__cd) $(top_srcdir) && \ + $(AUTOMAKE) --foreign src/lib-dict/Makefile +Makefile: $(srcdir)/Makefile.in $(top_builddir)/config.status + @case '$?' in \ + *config.status*) \ + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh;; \ + *) \ + echo ' cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__maybe_remake_depfiles)'; \ + cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__maybe_remake_depfiles);; \ + esac; + +$(top_builddir)/config.status: $(top_srcdir)/configure $(CONFIG_STATUS_DEPENDENCIES) + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh + +$(top_srcdir)/configure: @MAINTAINER_MODE_TRUE@ $(am__configure_deps) + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh +$(ACLOCAL_M4): @MAINTAINER_MODE_TRUE@ $(am__aclocal_m4_deps) + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh +$(am__aclocal_m4_deps): + +clean-noinstPROGRAMS: + @list='$(noinst_PROGRAMS)'; test -n "$$list" || exit 0; \ + echo " rm -f" $$list; \ + rm -f $$list || exit $$?; \ + test -n "$(EXEEXT)" || exit 0; \ + list=`for p in $$list; do echo "$$p"; done | sed 's/$(EXEEXT)$$//'`; \ + echo " rm -f" $$list; \ + rm -f $$list + +clean-noinstLTLIBRARIES: + -test -z "$(noinst_LTLIBRARIES)" || rm -f $(noinst_LTLIBRARIES) + @list='$(noinst_LTLIBRARIES)'; \ + locs=`for p in $$list; do echo $$p; done | \ + sed 's|^[^/]*$$|.|; s|/[^/]*$$||; s|$$|/so_locations|' | \ + sort -u`; \ + test -z "$$locs" || { \ + echo rm -f $${locs}; \ + rm -f $${locs}; \ + } + +libdict.la: $(libdict_la_OBJECTS) $(libdict_la_DEPENDENCIES) $(EXTRA_libdict_la_DEPENDENCIES) + $(AM_V_CCLD)$(LINK) $(libdict_la_OBJECTS) $(libdict_la_LIBADD) $(LIBS) + +libdict_lua.la: $(libdict_lua_la_OBJECTS) $(libdict_lua_la_DEPENDENCIES) $(EXTRA_libdict_lua_la_DEPENDENCIES) + $(AM_V_CCLD)$(LINK) $(am_libdict_lua_la_rpath) $(libdict_lua_la_OBJECTS) $(libdict_lua_la_LIBADD) $(LIBS) + +test-dict$(EXEEXT): $(test_dict_OBJECTS) $(test_dict_DEPENDENCIES) $(EXTRA_test_dict_DEPENDENCIES) + @rm -f test-dict$(EXEEXT) + $(AM_V_CCLD)$(LINK) $(test_dict_OBJECTS) $(test_dict_LDADD) $(LIBS) + +test-dict-client$(EXEEXT): $(test_dict_client_OBJECTS) $(test_dict_client_DEPENDENCIES) $(EXTRA_test_dict_client_DEPENDENCIES) + @rm -f test-dict-client$(EXEEXT) + $(AM_V_CCLD)$(LINK) $(test_dict_client_OBJECTS) $(test_dict_client_LDADD) $(LIBS) + +mostlyclean-compile: + -rm -f *.$(OBJEXT) + +distclean-compile: + -rm -f *.tab.c + +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/dict-client.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/dict-fail.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/dict-file.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/dict-memcached-ascii.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/dict-memcached.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/dict-redis.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/dict-transaction-memory.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/dict.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libdict_lua_la-dict-iter-lua.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libdict_lua_la-dict-lua.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libdict_lua_la-dict-txn-lua.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/test-dict-client.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/test-dict.Po@am__quote@ # am--include-marker + +$(am__depfiles_remade): + @$(MKDIR_P) $(@D) + @echo '# dummy' >$@-t && $(am__mv) $@-t $@ + +am--depfiles: $(am__depfiles_remade) + +.c.o: +@am__fastdepCC_TRUE@ $(AM_V_CC)$(COMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ $< +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='$<' object='$@' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(COMPILE) -c -o $@ $< + +.c.obj: +@am__fastdepCC_TRUE@ $(AM_V_CC)$(COMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ `$(CYGPATH_W) '$<'` +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='$<' object='$@' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(COMPILE) -c -o $@ `$(CYGPATH_W) '$<'` + +.c.lo: +@am__fastdepCC_TRUE@ $(AM_V_CC)$(LTCOMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ $< +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Plo +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='$<' object='$@' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(LTCOMPILE) -c -o $@ $< + +libdict_lua_la-dict-lua.lo: dict-lua.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libdict_lua_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT libdict_lua_la-dict-lua.lo -MD -MP -MF $(DEPDIR)/libdict_lua_la-dict-lua.Tpo -c -o libdict_lua_la-dict-lua.lo `test -f 'dict-lua.c' || echo '$(srcdir)/'`dict-lua.c +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/libdict_lua_la-dict-lua.Tpo $(DEPDIR)/libdict_lua_la-dict-lua.Plo +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='dict-lua.c' object='libdict_lua_la-dict-lua.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libdict_lua_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o libdict_lua_la-dict-lua.lo `test -f 'dict-lua.c' || echo '$(srcdir)/'`dict-lua.c + +libdict_lua_la-dict-iter-lua.lo: dict-iter-lua.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libdict_lua_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT libdict_lua_la-dict-iter-lua.lo -MD -MP -MF $(DEPDIR)/libdict_lua_la-dict-iter-lua.Tpo -c -o libdict_lua_la-dict-iter-lua.lo `test -f 'dict-iter-lua.c' || echo '$(srcdir)/'`dict-iter-lua.c +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/libdict_lua_la-dict-iter-lua.Tpo $(DEPDIR)/libdict_lua_la-dict-iter-lua.Plo +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='dict-iter-lua.c' object='libdict_lua_la-dict-iter-lua.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libdict_lua_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o libdict_lua_la-dict-iter-lua.lo `test -f 'dict-iter-lua.c' || echo '$(srcdir)/'`dict-iter-lua.c + +libdict_lua_la-dict-txn-lua.lo: dict-txn-lua.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libdict_lua_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT libdict_lua_la-dict-txn-lua.lo -MD -MP -MF $(DEPDIR)/libdict_lua_la-dict-txn-lua.Tpo -c -o libdict_lua_la-dict-txn-lua.lo `test -f 'dict-txn-lua.c' || echo '$(srcdir)/'`dict-txn-lua.c +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/libdict_lua_la-dict-txn-lua.Tpo $(DEPDIR)/libdict_lua_la-dict-txn-lua.Plo +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='dict-txn-lua.c' object='libdict_lua_la-dict-txn-lua.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libdict_lua_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o libdict_lua_la-dict-txn-lua.lo `test -f 'dict-txn-lua.c' || echo '$(srcdir)/'`dict-txn-lua.c + +mostlyclean-libtool: + -rm -f *.lo + +clean-libtool: + -rm -rf .libs _libs +install-pkginc_libHEADERS: $(pkginc_lib_HEADERS) + @$(NORMAL_INSTALL) + @list='$(pkginc_lib_HEADERS)'; test -n "$(pkginc_libdir)" || list=; \ + if test -n "$$list"; then \ + echo " $(MKDIR_P) '$(DESTDIR)$(pkginc_libdir)'"; \ + $(MKDIR_P) "$(DESTDIR)$(pkginc_libdir)" || exit 1; \ + fi; \ + for p in $$list; do \ + if test -f "$$p"; then d=; else d="$(srcdir)/"; fi; \ + echo "$$d$$p"; \ + done | $(am__base_list) | \ + while read files; do \ + echo " $(INSTALL_HEADER) $$files '$(DESTDIR)$(pkginc_libdir)'"; \ + $(INSTALL_HEADER) $$files "$(DESTDIR)$(pkginc_libdir)" || exit $$?; \ + done + +uninstall-pkginc_libHEADERS: + @$(NORMAL_UNINSTALL) + @list='$(pkginc_lib_HEADERS)'; test -n "$(pkginc_libdir)" || list=; \ + files=`for p in $$list; do echo $$p; done | sed -e 's|^.*/||'`; \ + dir='$(DESTDIR)$(pkginc_libdir)'; $(am__uninstall_files_from_dir) + +ID: $(am__tagged_files) + $(am__define_uniq_tagged_files); mkid -fID $$unique +tags: tags-am +TAGS: tags + +tags-am: $(TAGS_DEPENDENCIES) $(am__tagged_files) + set x; \ + here=`pwd`; \ + $(am__define_uniq_tagged_files); \ + shift; \ + if test -z "$(ETAGS_ARGS)$$*$$unique"; then :; else \ + test -n "$$unique" || unique=$$empty_fix; \ + if test $$# -gt 0; then \ + $(ETAGS) $(ETAGSFLAGS) $(AM_ETAGSFLAGS) $(ETAGS_ARGS) \ + "$$@" $$unique; \ + else \ + $(ETAGS) $(ETAGSFLAGS) $(AM_ETAGSFLAGS) $(ETAGS_ARGS) \ + $$unique; \ + fi; \ + fi +ctags: ctags-am + +CTAGS: ctags +ctags-am: $(TAGS_DEPENDENCIES) $(am__tagged_files) + $(am__define_uniq_tagged_files); \ + test -z "$(CTAGS_ARGS)$$unique" \ + || $(CTAGS) $(CTAGSFLAGS) $(AM_CTAGSFLAGS) $(CTAGS_ARGS) \ + $$unique + +GTAGS: + here=`$(am__cd) $(top_builddir) && pwd` \ + && $(am__cd) $(top_srcdir) \ + && gtags -i $(GTAGS_ARGS) "$$here" +cscopelist: cscopelist-am + +cscopelist-am: $(am__tagged_files) + list='$(am__tagged_files)'; \ + case "$(srcdir)" in \ + [\\/]* | ?:[\\/]*) sdir="$(srcdir)" ;; \ + *) sdir=$(subdir)/$(srcdir) ;; \ + esac; \ + for i in $$list; do \ + if test -f "$$i"; then \ + echo "$(subdir)/$$i"; \ + else \ + echo "$$sdir/$$i"; \ + fi; \ + done >> $(top_builddir)/cscope.files + +distclean-tags: + -rm -f TAGS ID GTAGS GRTAGS GSYMS GPATH tags + +distdir: $(BUILT_SOURCES) + $(MAKE) $(AM_MAKEFLAGS) distdir-am + +distdir-am: $(DISTFILES) + @srcdirstrip=`echo "$(srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \ + topsrcdirstrip=`echo "$(top_srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \ + list='$(DISTFILES)'; \ + dist_files=`for file in $$list; do echo $$file; done | \ + sed -e "s|^$$srcdirstrip/||;t" \ + -e "s|^$$topsrcdirstrip/|$(top_builddir)/|;t"`; \ + case $$dist_files in \ + */*) $(MKDIR_P) `echo "$$dist_files" | \ + sed '/\//!d;s|^|$(distdir)/|;s,/[^/]*$$,,' | \ + sort -u` ;; \ + esac; \ + for file in $$dist_files; do \ + if test -f $$file || test -d $$file; then d=.; else d=$(srcdir); fi; \ + if test -d $$d/$$file; then \ + dir=`echo "/$$file" | sed -e 's,/[^/]*$$,,'`; \ + if test -d "$(distdir)/$$file"; then \ + find "$(distdir)/$$file" -type d ! -perm -700 -exec chmod u+rwx {} \;; \ + fi; \ + if test -d $(srcdir)/$$file && test $$d != $(srcdir); then \ + cp -fpR $(srcdir)/$$file "$(distdir)$$dir" || exit 1; \ + find "$(distdir)/$$file" -type d ! -perm -700 -exec chmod u+rwx {} \;; \ + fi; \ + cp -fpR $$d/$$file "$(distdir)$$dir" || exit 1; \ + else \ + test -f "$(distdir)/$$file" \ + || cp -p $$d/$$file "$(distdir)/$$file" \ + || exit 1; \ + fi; \ + done +check-am: all-am + $(MAKE) $(AM_MAKEFLAGS) check-local +check: check-am +all-am: Makefile $(PROGRAMS) $(LTLIBRARIES) $(HEADERS) +installdirs: + for dir in "$(DESTDIR)$(pkginc_libdir)"; do \ + test -z "$$dir" || $(MKDIR_P) "$$dir"; \ + done +install: install-am +install-exec: install-exec-am +install-data: install-data-am +uninstall: uninstall-am + +install-am: all-am + @$(MAKE) $(AM_MAKEFLAGS) install-exec-am install-data-am + +installcheck: installcheck-am +install-strip: + if test -z '$(STRIP)'; then \ + $(MAKE) $(AM_MAKEFLAGS) INSTALL_PROGRAM="$(INSTALL_STRIP_PROGRAM)" \ + install_sh_PROGRAM="$(INSTALL_STRIP_PROGRAM)" INSTALL_STRIP_FLAG=-s \ + install; \ + else \ + $(MAKE) $(AM_MAKEFLAGS) INSTALL_PROGRAM="$(INSTALL_STRIP_PROGRAM)" \ + install_sh_PROGRAM="$(INSTALL_STRIP_PROGRAM)" INSTALL_STRIP_FLAG=-s \ + "INSTALL_PROGRAM_ENV=STRIPPROG='$(STRIP)'" install; \ + fi +mostlyclean-generic: + +clean-generic: + +distclean-generic: + -test -z "$(CONFIG_CLEAN_FILES)" || rm -f $(CONFIG_CLEAN_FILES) + -test . = "$(srcdir)" || test -z "$(CONFIG_CLEAN_VPATH_FILES)" || rm -f $(CONFIG_CLEAN_VPATH_FILES) + +maintainer-clean-generic: + @echo "This command is intended for maintainers to use" + @echo "it deletes files that may require special tools to rebuild." +clean: clean-am + +clean-am: clean-generic clean-libtool clean-noinstLTLIBRARIES \ + clean-noinstPROGRAMS mostlyclean-am + +distclean: distclean-am + -rm -f ./$(DEPDIR)/dict-client.Plo + -rm -f ./$(DEPDIR)/dict-fail.Plo + -rm -f ./$(DEPDIR)/dict-file.Plo + -rm -f ./$(DEPDIR)/dict-memcached-ascii.Plo + -rm -f ./$(DEPDIR)/dict-memcached.Plo + -rm -f ./$(DEPDIR)/dict-redis.Plo + -rm -f ./$(DEPDIR)/dict-transaction-memory.Plo + -rm -f ./$(DEPDIR)/dict.Plo + -rm -f ./$(DEPDIR)/libdict_lua_la-dict-iter-lua.Plo + -rm -f ./$(DEPDIR)/libdict_lua_la-dict-lua.Plo + -rm -f ./$(DEPDIR)/libdict_lua_la-dict-txn-lua.Plo + -rm -f ./$(DEPDIR)/test-dict-client.Po + -rm -f ./$(DEPDIR)/test-dict.Po + -rm -f Makefile +distclean-am: clean-am distclean-compile distclean-generic \ + distclean-tags + +dvi: dvi-am + +dvi-am: + +html: html-am + +html-am: + +info: info-am + +info-am: + +install-data-am: install-pkginc_libHEADERS + +install-dvi: install-dvi-am + +install-dvi-am: + +install-exec-am: + +install-html: install-html-am + +install-html-am: + +install-info: install-info-am + +install-info-am: + +install-man: + +install-pdf: install-pdf-am + +install-pdf-am: + +install-ps: install-ps-am + +install-ps-am: + +installcheck-am: + +maintainer-clean: maintainer-clean-am + -rm -f ./$(DEPDIR)/dict-client.Plo + -rm -f ./$(DEPDIR)/dict-fail.Plo + -rm -f ./$(DEPDIR)/dict-file.Plo + -rm -f ./$(DEPDIR)/dict-memcached-ascii.Plo + -rm -f ./$(DEPDIR)/dict-memcached.Plo + -rm -f ./$(DEPDIR)/dict-redis.Plo + -rm -f ./$(DEPDIR)/dict-transaction-memory.Plo + -rm -f ./$(DEPDIR)/dict.Plo + -rm -f ./$(DEPDIR)/libdict_lua_la-dict-iter-lua.Plo + -rm -f ./$(DEPDIR)/libdict_lua_la-dict-lua.Plo + -rm -f ./$(DEPDIR)/libdict_lua_la-dict-txn-lua.Plo + -rm -f ./$(DEPDIR)/test-dict-client.Po + -rm -f ./$(DEPDIR)/test-dict.Po + -rm -f Makefile +maintainer-clean-am: distclean-am maintainer-clean-generic + +mostlyclean: mostlyclean-am + +mostlyclean-am: mostlyclean-compile mostlyclean-generic \ + mostlyclean-libtool + +pdf: pdf-am + +pdf-am: + +ps: ps-am + +ps-am: + +uninstall-am: uninstall-pkginc_libHEADERS + +.MAKE: check-am install-am install-strip + +.PHONY: CTAGS GTAGS TAGS all all-am am--depfiles check check-am \ + check-local clean clean-generic clean-libtool \ + clean-noinstLTLIBRARIES clean-noinstPROGRAMS cscopelist-am \ + ctags ctags-am distclean distclean-compile distclean-generic \ + distclean-libtool distclean-tags distdir dvi dvi-am html \ + html-am info info-am install install-am install-data \ + install-data-am install-dvi install-dvi-am install-exec \ + install-exec-am install-html install-html-am install-info \ + install-info-am install-man install-pdf install-pdf-am \ + install-pkginc_libHEADERS install-ps install-ps-am \ + install-strip installcheck installcheck-am installdirs \ + maintainer-clean maintainer-clean-generic mostlyclean \ + mostlyclean-compile mostlyclean-generic mostlyclean-libtool \ + pdf pdf-am ps ps-am tags tags-am uninstall uninstall-am \ + uninstall-pkginc_libHEADERS + +.PRECIOUS: Makefile + + +check-local: + for bin in $(test_programs) $(check_PROGRAMS); do \ + if ! $(RUN_TEST) ./$$bin; then exit 1; fi; \ + done + +# Tell versions [3.59,3.63) of GNU make to not export all variables. +# Otherwise a system limit (for SysV at least) may be exceeded. +.NOEXPORT: diff --git a/src/lib-dict/dict-client.c b/src/lib-dict/dict-client.c new file mode 100644 index 0000000..32b2da1 --- /dev/null +++ b/src/lib-dict/dict-client.c @@ -0,0 +1,1492 @@ +/* Copyright (c) 2005-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "llist.h" +#include "str.h" +#include "strescape.h" +#include "file-lock.h" +#include "time-util.h" +#include "connection.h" +#include "ostream.h" +#include "eacces-error.h" +#include "dict-private.h" +#include "dict-client.h" + +#include <unistd.h> +#include <fcntl.h> + +/* Disconnect from dict server after this many milliseconds of idling after + sending a command. Because dict server does blocking dict accesses, it can + handle only one client at a time. This is why the default timeout is zero, + so that there won't be many dict processes just doing nothing. Zero means + that the socket is disconnected immediately after returning to ioloop. */ +#define DICT_CLIENT_DEFAULT_TIMEOUT_MSECS 0 + +/* Abort dict lookup after this many seconds. */ +#define DICT_CLIENT_REQUEST_TIMEOUT_MSECS 30000 +/* When dict lookup timeout is reached, wait a bit longer if the last dict + ioloop wait was shorter than this. */ +#define DICT_CLIENT_REQUEST_TIMEOUT_MIN_LAST_IOLOOP_WAIT_MSECS 1000 +/* Log a warning if dict lookup takes longer than this many milliseconds. */ +#define DICT_CLIENT_DEFAULT_WARN_SLOW_MSECS 5000 + +struct client_dict_cmd { + int refcount; + struct client_dict *dict; + struct timeval start_time; + char *query; + unsigned int async_id; + struct timeval async_id_received_time; + + uint64_t start_global_ioloop_usecs; + uint64_t start_dict_ioloop_usecs; + uint64_t start_lock_usecs; + + bool reconnected; + bool retry_errors; + bool no_replies; + bool unfinished; + bool background; + + void (*callback)(struct client_dict_cmd *cmd, + enum dict_protocol_reply reply, const char *value, + const char *const *extra_args, const char *error, + bool disconnected); + struct client_dict_iterate_context *iter; + struct client_dict_transaction_context *trans; + + struct { + dict_lookup_callback_t *lookup; + dict_transaction_commit_callback_t *commit; + void *context; + } api_callback; +}; + +struct dict_client_connection { + struct connection conn; + struct client_dict *dict; +}; + +struct client_dict { + struct dict dict; + struct dict_client_connection conn; + + char *uri; + enum dict_data_type value_type; + unsigned warn_slow_msecs; + + time_t last_failed_connect; + char *last_connect_error; + + struct io_wait_timer *wait_timer; + uint64_t last_timer_switch_usecs; + struct timeout *to_requests; + struct timeout *to_idle; + unsigned int idle_msecs; + + ARRAY(struct client_dict_cmd *) cmds; + struct client_dict_transaction_context *transactions; + + unsigned int transaction_id_counter; +}; + +struct client_dict_iter_result { + const char *key, *const *values; +}; + +struct client_dict_iterate_context { + struct dict_iterate_context ctx; + char *error; + char *path; + enum dict_iterate_flags flags; + int refcount; + + pool_t results_pool; + ARRAY(struct client_dict_iter_result) results; + unsigned int result_idx; + + bool cmd_sent; + bool seen_results; + bool finished; + bool deinit; +}; + +struct client_dict_transaction_context { + struct dict_transaction_context ctx; + struct client_dict_transaction_context *prev, *next; + + char *first_query; + char *error; + + unsigned int id; + unsigned int query_count; + + bool sent_begin:1; +}; + +static struct connection_list *dict_connections; + +static int client_dict_connect(struct client_dict *dict, const char **error_r); +static int client_dict_reconnect(struct client_dict *dict, const char *reason, + const char **error_r); +static void client_dict_disconnect(struct client_dict *dict, const char *reason); +static const char *dict_wait_warnings(const struct client_dict_cmd *cmd); + +static struct client_dict_cmd * +client_dict_cmd_init(struct client_dict *dict, const char *query) +{ + struct client_dict_cmd *cmd; + + io_loop_time_refresh(); + + cmd = i_new(struct client_dict_cmd, 1); + cmd->refcount = 1; + cmd->dict = dict; + cmd->query = i_strdup(query); + cmd->start_time = ioloop_timeval; + cmd->start_global_ioloop_usecs = ioloop_global_wait_usecs; + cmd->start_dict_ioloop_usecs = io_wait_timer_get_usecs(dict->wait_timer); + cmd->start_lock_usecs = file_lock_wait_get_total_usecs(); + return cmd; +} + +static void client_dict_cmd_ref(struct client_dict_cmd *cmd) +{ + i_assert(cmd->refcount > 0); + cmd->refcount++; +} + +static bool client_dict_cmd_unref(struct client_dict_cmd *cmd) +{ + i_assert(cmd->refcount > 0); + if (--cmd->refcount > 0) + return TRUE; + + i_assert(cmd->trans == NULL); + + i_free(cmd->query); + i_free(cmd); + return FALSE; +} + +static bool +dict_cmd_callback_line(struct client_dict_cmd *cmd, const char *const *args) +{ + const char *value = args[0]; + enum dict_protocol_reply reply; + + if (value == NULL) { + /* "" is a valid iteration reply */ + reply = DICT_PROTOCOL_REPLY_ITER_FINISHED; + } else { + reply = value[0]; + value++; + args++; + } + + cmd->unfinished = FALSE; + cmd->callback(cmd, reply, value, args, NULL, FALSE); + return !cmd->unfinished; +} + +static void +dict_cmd_callback_error(struct client_dict_cmd *cmd, const char *error, + bool disconnected) +{ + const char *null_arg = NULL; + + cmd->unfinished = FALSE; + if (cmd->callback != NULL) { + cmd->callback(cmd, DICT_PROTOCOL_REPLY_ERROR, + "", &null_arg, error, disconnected); + } + i_assert(!cmd->unfinished); +} + +static struct client_dict_cmd * +client_dict_cmd_first_nonbg(struct client_dict *dict) +{ + struct client_dict_cmd *const *cmds; + unsigned int i, count; + + cmds = array_get(&dict->cmds, &count); + for (i = 0; i < count; i++) { + if (!cmds[i]->background) + return cmds[i]; + } + return NULL; +} + +static void client_dict_input_timeout(struct client_dict *dict) +{ + struct client_dict_cmd *cmd; + const char *error; + uint64_t msecs_in_last_dict_ioloop_wait; + int cmd_diff; + + /* find the first non-background command. there must be at least one. */ + cmd = client_dict_cmd_first_nonbg(dict); + i_assert(cmd != NULL); + + cmd_diff = timeval_diff_msecs(&ioloop_timeval, &cmd->start_time); + if (cmd_diff < DICT_CLIENT_REQUEST_TIMEOUT_MSECS) { + /* need to re-create this timeout. the currently-oldest + command was added when another command was still + running with an older timeout. */ + timeout_remove(&dict->to_requests); + dict->to_requests = + timeout_add(DICT_CLIENT_REQUEST_TIMEOUT_MSECS - cmd_diff, + client_dict_input_timeout, dict); + return; + } + + /* If we've gotten here because all the time was spent in other ioloops + or locks, make sure there's a bit of time waiting for the dict + ioloop as well. There's a good chance that the reply can be read. */ + msecs_in_last_dict_ioloop_wait = + (io_wait_timer_get_usecs(dict->wait_timer) - + dict->last_timer_switch_usecs + 999) / 1000; + if (msecs_in_last_dict_ioloop_wait < DICT_CLIENT_REQUEST_TIMEOUT_MIN_LAST_IOLOOP_WAIT_MSECS) { + timeout_remove(&dict->to_requests); + dict->to_requests = + timeout_add(DICT_CLIENT_REQUEST_TIMEOUT_MIN_LAST_IOLOOP_WAIT_MSECS - + msecs_in_last_dict_ioloop_wait, + client_dict_input_timeout, dict); + return; + } + + (void)client_dict_reconnect(dict, t_strdup_printf( + "Dict server timeout: %s " + "(%u commands pending, oldest sent %u.%03u secs ago: %s, %s)", + connection_input_timeout_reason(&dict->conn.conn), + array_count(&dict->cmds), + cmd_diff/1000, cmd_diff%1000, cmd->query, + dict_wait_warnings(cmd)), &error); +} + +static int +client_dict_cmd_query_send(struct client_dict *dict, const char *query) +{ + struct const_iovec iov[2]; + ssize_t ret; + + iov[0].iov_base = query; + iov[0].iov_len = strlen(query); + iov[1].iov_base = "\n"; + iov[1].iov_len = 1; + ret = o_stream_sendv(dict->conn.conn.output, iov, 2); + if (ret < 0) + return -1; + i_assert((size_t)ret == iov[0].iov_len + 1); + return 0; +} + +static bool +client_dict_cmd_send(struct client_dict *dict, struct client_dict_cmd **_cmd, + const char **error_r) +{ + struct client_dict_cmd *cmd = *_cmd; + const char *error = NULL; + bool retry = cmd->retry_errors; + int ret; + + *_cmd = NULL; + + /* we're no longer idling. even with no_replies=TRUE we're going to + wait for COMMIT/ROLLBACK. */ + timeout_remove(&dict->to_idle); + + if (client_dict_connect(dict, &error) < 0) { + retry = FALSE; + ret = -1; + } else { + ret = client_dict_cmd_query_send(dict, cmd->query); + if (ret < 0) { + error = t_strdup_printf("write(%s) failed: %s", dict->conn.conn.name, + o_stream_get_error(dict->conn.conn.output)); + } + } + if (ret < 0 && retry) { + /* Reconnect and try again. */ + if (client_dict_reconnect(dict, error, &error) < 0) + ; + else if (client_dict_cmd_query_send(dict, cmd->query) < 0) { + error = t_strdup_printf("write(%s) failed: %s", dict->conn.conn.name, + o_stream_get_error(dict->conn.conn.output)); + } else { + ret = 0; + } + } + + if (cmd->no_replies) { + /* just send and forget */ + client_dict_cmd_unref(cmd); + return TRUE; + } else if (ret < 0) { + i_assert(error != NULL); + /* we didn't successfully send this command to dict */ + dict_cmd_callback_error(cmd, error, cmd->reconnected); + client_dict_cmd_unref(cmd); + if (error_r != NULL) + *error_r = error; + return FALSE; + } else { + if (dict->to_requests == NULL && !cmd->background) { + dict->to_requests = + timeout_add(DICT_CLIENT_REQUEST_TIMEOUT_MSECS, + client_dict_input_timeout, dict); + } + array_push_back(&dict->cmds, &cmd); + return TRUE; + } +} + +static bool +client_dict_transaction_send_begin(struct client_dict_transaction_context *ctx, + const struct dict_op_settings_private *set) +{ + struct client_dict *dict = (struct client_dict *)ctx->ctx.dict; + struct client_dict_cmd *cmd; + const char *query, *error; + + i_assert(ctx->error == NULL); + + ctx->sent_begin = TRUE; + + /* transactions commands don't have replies. only COMMIT has. */ + query = t_strdup_printf("%c%u\t%s", DICT_PROTOCOL_CMD_BEGIN, + ctx->id, + set->username == NULL ? "" : str_tabescape(set->username)); + cmd = client_dict_cmd_init(dict, query); + cmd->no_replies = TRUE; + cmd->retry_errors = TRUE; + if (!client_dict_cmd_send(dict, &cmd, &error)) { + ctx->error = i_strdup(error); + return FALSE; + } + return TRUE; +} + +static void +client_dict_send_transaction_query(struct client_dict_transaction_context *ctx, + const char *query) +{ + struct client_dict *dict = (struct client_dict *)ctx->ctx.dict; + const struct dict_op_settings_private *set = &ctx->ctx.set; + struct client_dict_cmd *cmd; + const char *error; + + if (ctx->error != NULL) + return; + + if (!ctx->sent_begin) { + if (!client_dict_transaction_send_begin(ctx, set)) + return; + } + + ctx->query_count++; + if (ctx->first_query == NULL) + ctx->first_query = i_strdup(query); + + cmd = client_dict_cmd_init(dict, query); + cmd->no_replies = TRUE; + if (!client_dict_cmd_send(dict, &cmd, &error)) + ctx->error = i_strdup(error); +} + +static bool client_dict_is_finished(struct client_dict *dict) +{ + return dict->transactions == NULL && array_count(&dict->cmds) == 0; +} + +static void client_dict_timeout(struct client_dict *dict) +{ + if (client_dict_is_finished(dict)) + client_dict_disconnect(dict, "Idle disconnection"); + else + timeout_remove(&dict->to_idle); +} + +static bool client_dict_have_nonbackground_cmds(struct client_dict *dict) +{ + struct client_dict_cmd *cmd; + + array_foreach_elem(&dict->cmds, cmd) { + if (!cmd->background) + return TRUE; + } + return FALSE; +} + +static void client_dict_add_timeout(struct client_dict *dict) +{ + if (dict->to_idle != NULL) { + if (dict->idle_msecs > 0) + timeout_reset(dict->to_idle); + } else if (client_dict_is_finished(dict)) { + dict->to_idle = timeout_add(dict->idle_msecs, + client_dict_timeout, dict); + timeout_remove(&dict->to_requests); + } else if (dict->transactions == NULL && + !client_dict_have_nonbackground_cmds(dict)) { + /* we had non-background commands, but now we're back to + having only background commands. remove timeouts. */ + timeout_remove(&dict->to_requests); + } +} + +static void client_dict_cmd_backgrounded(struct client_dict *dict) +{ + if (dict->to_requests == NULL) + return; + + if (!client_dict_have_nonbackground_cmds(dict)) { + /* we only have background-commands. + remove the request timeout. */ + timeout_remove(&dict->to_requests); + } +} + +static int +dict_conn_assign_next_async_id(struct dict_client_connection *conn, + const char *line) +{ + struct client_dict_cmd *const *cmds; + unsigned int i, count, async_id; + + i_assert(line[0] == DICT_PROTOCOL_REPLY_ASYNC_ID); + + if (str_to_uint(line+1, &async_id) < 0 || async_id == 0) { + e_error(conn->conn.event, "Received invalid async-id line: %s", + line); + return -1; + } + cmds = array_get(&conn->dict->cmds, &count); + for (i = 0; i < count; i++) { + if (cmds[i]->async_id == 0) { + cmds[i]->async_id = async_id; + cmds[i]->async_id_received_time = ioloop_timeval; + return 0; + } + } + e_error(conn->conn.event, "Received async-id line, but all %u " + "commands already have it: %s", + count, line); + return -1; +} + +static int dict_conn_find_async_id(struct dict_client_connection *conn, + const char *async_arg, + const char *line, unsigned int *idx_r) +{ + struct client_dict_cmd *const *cmds; + unsigned int i, count, async_id; + + i_assert(async_arg[0] == DICT_PROTOCOL_REPLY_ASYNC_REPLY); + + if (str_to_uint(async_arg+1, &async_id) < 0 || async_id == 0) { + e_error(conn->conn.event, "Received invalid async-reply line: %s", + line); + return -1; + } + + cmds = array_get(&conn->dict->cmds, &count); + for (i = 0; i < count; i++) { + if (cmds[i]->async_id == async_id) { + *idx_r = i; + return 0; + } + } + e_error(conn->conn.event, "Received reply for nonexistent async-id %u: %s", + async_id, line); + return -1; +} + +static int dict_conn_input_line(struct connection *_conn, const char *line) +{ + struct dict_client_connection *conn = + (struct dict_client_connection *)_conn; + struct client_dict *dict = conn->dict; + struct client_dict_cmd *const *cmds; + const char *const *args; + unsigned int i, count; + bool finished; + + if (dict->to_requests != NULL) + timeout_reset(dict->to_requests); + + if (line[0] == DICT_PROTOCOL_REPLY_ASYNC_ID) + return dict_conn_assign_next_async_id(conn, line) < 0 ? -1 : 1; + + cmds = array_get(&conn->dict->cmds, &count); + if (count == 0) { + e_error(conn->conn.event, "Received reply without pending commands: %s", + line); + return -1; + } + + args = t_strsplit_tabescaped(line); + if (args[0] != NULL && args[0][0] == DICT_PROTOCOL_REPLY_ASYNC_REPLY) { + if (dict_conn_find_async_id(conn, args[0], line, &i) < 0) + return -1; + args++; + } else { + i = 0; + } + i_assert(!cmds[i]->no_replies); + + client_dict_cmd_ref(cmds[i]); + finished = dict_cmd_callback_line(cmds[i], args); + if (!client_dict_cmd_unref(cmds[i])) { + /* disconnected during command handling */ + return -1; + } + if (!finished) { + /* more lines needed for this command */ + return 1; + } + client_dict_cmd_unref(cmds[i]); + array_delete(&dict->cmds, i, 1); + + client_dict_add_timeout(dict); + return 1; +} + +static int client_dict_connect(struct client_dict *dict, const char **error_r) +{ + const char *query, *error; + + if (dict->conn.conn.fd_in != -1) + return 0; + if (dict->last_failed_connect == ioloop_time) { + /* Try again later */ + *error_r = dict->last_connect_error; + return -1; + } + + if (connection_client_connect(&dict->conn.conn) < 0) { + dict->last_failed_connect = ioloop_time; + if (errno == EACCES) { + error = eacces_error_get("net_connect_unix", + dict->conn.conn.name); + } else { + error = t_strdup_printf( + "net_connect_unix(%s) failed: %m", dict->conn.conn.name); + } + i_free(dict->last_connect_error); + dict->last_connect_error = i_strdup(error); + *error_r = error; + return -1; + } + + query = t_strdup_printf("%c%u\t%u\t%d\t%s\t%s\n", + DICT_PROTOCOL_CMD_HELLO, + DICT_CLIENT_PROTOCOL_MAJOR_VERSION, + DICT_CLIENT_PROTOCOL_MINOR_VERSION, + dict->value_type, + "", + str_tabescape(dict->uri)); + o_stream_nsend_str(dict->conn.conn.output, query); + client_dict_add_timeout(dict); + return 0; +} + +static void +client_dict_abort_commands(struct client_dict *dict, const char *reason) +{ + ARRAY(struct client_dict_cmd *) cmds_copy; + struct client_dict_cmd *cmd; + + /* abort all commands */ + t_array_init(&cmds_copy, array_count(&dict->cmds)); + array_append_array(&cmds_copy, &dict->cmds); + array_clear(&dict->cmds); + + array_foreach_elem(&cmds_copy, cmd) { + dict_cmd_callback_error(cmd, reason, TRUE); + client_dict_cmd_unref(cmd); + } +} + +static void client_dict_disconnect(struct client_dict *dict, const char *reason) +{ + struct client_dict_transaction_context *ctx, *next; + + client_dict_abort_commands(dict, reason); + + /* all transactions that have sent BEGIN are no longer valid */ + for (ctx = dict->transactions; ctx != NULL; ctx = next) { + next = ctx->next; + if (ctx->sent_begin && ctx->error == NULL) + ctx->error = i_strdup(reason); + } + + timeout_remove(&dict->to_idle); + timeout_remove(&dict->to_requests); + connection_disconnect(&dict->conn.conn); +} + +static int client_dict_reconnect(struct client_dict *dict, const char *reason, + const char **error_r) +{ + ARRAY(struct client_dict_cmd *) retry_cmds; + struct client_dict_cmd *cmd; + const char *error; + int ret; + + t_array_init(&retry_cmds, array_count(&dict->cmds)); + for (unsigned int i = 0; i < array_count(&dict->cmds); ) { + cmd = array_idx_elem(&dict->cmds, i); + if (!cmd->retry_errors) { + i++; + } else if (cmd->iter != NULL && + cmd->iter->seen_results) { + /* don't retry iteration that already returned + something to the caller. otherwise we'd return + duplicates. */ + i++; + } else { + array_push_back(&retry_cmds, &cmd); + array_delete(&dict->cmds, i, 1); + } + } + client_dict_disconnect(dict, reason); + if (client_dict_connect(dict, error_r) < 0) { + reason = t_strdup_printf("%s - reconnect failed: %s", + reason, *error_r); + array_foreach_elem(&retry_cmds, cmd) { + dict_cmd_callback_error(cmd, reason, TRUE); + client_dict_cmd_unref(cmd); + } + return -1; + } + if (array_count(&retry_cmds) == 0) + return 0; + e_warning(dict->conn.conn.event, "%s - reconnected", reason); + + ret = 0; error = ""; + array_foreach_elem(&retry_cmds, cmd) { + cmd->reconnected = TRUE; + cmd->async_id = 0; + /* if it fails again, don't retry anymore */ + cmd->retry_errors = FALSE; + if (ret < 0) { + dict_cmd_callback_error(cmd, error, TRUE); + client_dict_cmd_unref(cmd); + } else if (!client_dict_cmd_send(dict, &cmd, &error)) + ret = -1; + } + return ret; +} + +static void dict_conn_destroy(struct connection *_conn) +{ + struct dict_client_connection *conn = + (struct dict_client_connection *)_conn; + + client_dict_disconnect(conn->dict, connection_disconnect_reason(_conn)); +} + +static const struct connection_settings dict_conn_set = { + .input_max_size = SIZE_MAX, + .output_max_size = SIZE_MAX, + .unix_client_connect_msecs = 1000, + .client = TRUE +}; + +static const struct connection_vfuncs dict_conn_vfuncs = { + .destroy = dict_conn_destroy, + .input_line = dict_conn_input_line +}; + +static int +client_dict_init(struct dict *driver, const char *uri, + const struct dict_settings *set, + struct dict **dict_r, const char **error_r) +{ + struct ioloop *old_ioloop = current_ioloop; + struct client_dict *dict; + const char *p, *dest_uri, *path; + unsigned int idle_msecs = DICT_CLIENT_DEFAULT_TIMEOUT_MSECS; + unsigned int warn_slow_msecs = DICT_CLIENT_DEFAULT_WARN_SLOW_MSECS; + + /* uri = [idle_msecs=<n>:] [warn_slow_msecs=<n>:] [<path>] ":" <uri> */ + for (;;) { + if (str_begins(uri, "idle_msecs=")) { + p = strchr(uri+11, ':'); + if (p == NULL) { + *error_r = t_strdup_printf("Invalid URI: %s", uri); + return -1; + } + if (str_to_uint(t_strdup_until(uri+11, p), &idle_msecs) < 0) { + *error_r = "Invalid idle_msecs"; + return -1; + } + uri = p+1; + } else if (str_begins(uri, "warn_slow_msecs=")) { + p = strchr(uri+11, ':'); + if (p == NULL) { + *error_r = t_strdup_printf("Invalid URI: %s", uri); + return -1; + } + if (str_to_uint(t_strdup_until(uri+16, p), &warn_slow_msecs) < 0) { + *error_r = "Invalid warn_slow_msecs"; + return -1; + } + uri = p+1; + } else { + break; + } + } + dest_uri = strchr(uri, ':'); + if (dest_uri == NULL) { + *error_r = t_strdup_printf("Invalid URI: %s", uri); + return -1; + } + + if (dict_connections == NULL) { + dict_connections = connection_list_init(&dict_conn_set, + &dict_conn_vfuncs); + } + + dict = i_new(struct client_dict, 1); + dict->dict = *driver; + dict->conn.dict = dict; + dict->conn.conn.event_parent = set->event_parent; + dict->idle_msecs = idle_msecs; + dict->warn_slow_msecs = warn_slow_msecs; + i_array_init(&dict->cmds, 32); + + if (uri[0] == ':') { + /* default path */ + path = t_strconcat(set->base_dir, + "/"DEFAULT_DICT_SERVER_SOCKET_FNAME, NULL); + } else if (uri[0] == '/') { + /* absolute path */ + path = t_strdup_until(uri, dest_uri); + } else { + /* relative path to base_dir */ + path = t_strconcat(set->base_dir, "/", + t_strdup_until(uri, dest_uri), NULL); + } + connection_init_client_unix(dict_connections, &dict->conn.conn, path); + dict->uri = i_strdup(dest_uri + 1); + + dict->dict.ioloop = io_loop_create(); + dict->wait_timer = io_wait_timer_add(); + io_loop_set_current(old_ioloop); + *dict_r = &dict->dict; + return 0; +} + +static void client_dict_deinit(struct dict *_dict) +{ + struct client_dict *dict = (struct client_dict *)_dict; + struct ioloop *old_ioloop = current_ioloop; + + client_dict_disconnect(dict, "Deinit"); + connection_deinit(&dict->conn.conn); + io_wait_timer_remove(&dict->wait_timer); + + i_assert(dict->transactions == NULL); + i_assert(array_count(&dict->cmds) == 0); + + io_loop_set_current(dict->dict.ioloop); + io_loop_destroy(&dict->dict.ioloop); + io_loop_set_current(old_ioloop); + + array_free(&dict->cmds); + i_free(dict->last_connect_error); + i_free(dict->uri); + i_free(dict); + + if (dict_connections->connections == NULL) + connection_list_deinit(&dict_connections); +} + +static void client_dict_wait(struct dict *_dict) +{ + struct client_dict *dict = (struct client_dict *)_dict; + + if (array_count(&dict->cmds) == 0) + return; + + i_assert(io_loop_is_empty(dict->dict.ioloop)); + dict->dict.prev_ioloop = current_ioloop; + io_loop_set_current(dict->dict.ioloop); + dict_switch_ioloop(_dict); + while (array_count(&dict->cmds) > 0) + io_loop_run(dict->dict.ioloop); + + io_loop_set_current(dict->dict.prev_ioloop); + dict->dict.prev_ioloop = NULL; + + dict_switch_ioloop(_dict); + i_assert(io_loop_is_empty(dict->dict.ioloop)); +} + +static bool client_dict_switch_ioloop(struct dict *_dict) +{ + struct client_dict *dict = (struct client_dict *)_dict; + + dict->last_timer_switch_usecs = + io_wait_timer_get_usecs(dict->wait_timer); + dict->wait_timer = io_wait_timer_move(&dict->wait_timer); + if (dict->to_idle != NULL) + dict->to_idle = io_loop_move_timeout(&dict->to_idle); + if (dict->to_requests != NULL) + dict->to_requests = io_loop_move_timeout(&dict->to_requests); + connection_switch_ioloop(&dict->conn.conn); + return array_count(&dict->cmds) > 0; +} + +static const char *dict_wait_warnings(const struct client_dict_cmd *cmd) +{ + int global_ioloop_msecs = (ioloop_global_wait_usecs - + cmd->start_global_ioloop_usecs + 999) / 1000; + int dict_ioloop_msecs = (io_wait_timer_get_usecs(cmd->dict->wait_timer) - + cmd->start_dict_ioloop_usecs + 999) / 1000; + int other_ioloop_msecs = global_ioloop_msecs - dict_ioloop_msecs; + int lock_msecs = (file_lock_wait_get_total_usecs() - + cmd->start_lock_usecs + 999) / 1000; + + return t_strdup_printf( + "%d.%03d in dict wait, %d.%03d in other ioloops, %d.%03d in locks", + dict_ioloop_msecs/1000, dict_ioloop_msecs%1000, + other_ioloop_msecs/1000, other_ioloop_msecs%1000, + lock_msecs/1000, lock_msecs%1000); +} + +static const char * +dict_warnings_sec(const struct client_dict_cmd *cmd, int msecs, + const char *const *extra_args) +{ + string_t *str = t_str_new(64); + struct timeval tv_start, tv_end; + unsigned int tv_start_usec, tv_end_usec; + + str_printfa(str, "%d.%03d secs (%s", msecs/1000, msecs%1000, + dict_wait_warnings(cmd)); + if (cmd->reconnected) { + int reconnected_msecs = + timeval_diff_msecs(&ioloop_timeval, + &cmd->dict->conn.conn.connect_started); + str_printfa(str, ", reconnected %u.%03u secs ago", + reconnected_msecs/1000, reconnected_msecs%1000); + } + if (cmd->async_id != 0) { + int async_reply_msecs = + timeval_diff_msecs(&ioloop_timeval, &cmd->async_id_received_time); + str_printfa(str, ", async-id reply %u.%03u secs ago", + async_reply_msecs/1000, async_reply_msecs%1000); + } + if (extra_args != NULL && + str_array_length(extra_args) >= 4 && + str_to_time(extra_args[0], &tv_start.tv_sec) == 0 && + str_to_uint(extra_args[1], &tv_start_usec) == 0 && + str_to_time(extra_args[2], &tv_end.tv_sec) == 0 && + str_to_uint(extra_args[3], &tv_end_usec) == 0) { + tv_start.tv_usec = tv_start_usec; + tv_end.tv_usec = tv_end_usec; + + int server_msecs_since_start = + timeval_diff_msecs(&ioloop_timeval, &tv_start); + int server_msecs = timeval_diff_msecs(&tv_end, &tv_start); + str_printfa(str, ", started on dict-server %u.%03d secs ago, " + "took %u.%03d secs", + server_msecs_since_start/1000, + server_msecs_since_start%1000, + server_msecs/1000, server_msecs%1000); + } + str_append_c(str, ')'); + return str_c(str); +} + +static void +client_dict_lookup_async_callback(struct client_dict_cmd *cmd, + enum dict_protocol_reply reply, + const char *value, + const char *const *extra_args, + const char *error, + bool disconnected ATTR_UNUSED) +{ + struct client_dict *dict = cmd->dict; + struct dict_lookup_result result; + const char *const values[] = { value, NULL }; + + i_zero(&result); + if (error != NULL) { + result.ret = -1; + result.error = error; + } else switch (reply) { + case DICT_PROTOCOL_REPLY_OK: + result.value = value; + result.values = values; + result.ret = 1; + break; + case DICT_PROTOCOL_REPLY_MULTI_OK: + result.values = t_strsplit_tabescaped(value); + result.value = result.values[0]; + result.ret = 1; + break; + case DICT_PROTOCOL_REPLY_NOTFOUND: + result.ret = 0; + break; + case DICT_PROTOCOL_REPLY_FAIL: + result.error = value[0] == '\0' ? "dict-server returned failure" : + t_strdup_printf("dict-server returned failure: %s", + value); + result.ret = -1; + break; + default: + result.error = t_strdup_printf( + "dict-client: Invalid lookup '%s' reply: %c%s", + cmd->query, reply, value); + client_dict_disconnect(dict, result.error); + result.ret = -1; + break; + } + + int diff = timeval_diff_msecs(&ioloop_timeval, &cmd->start_time); + if (result.error != NULL) { + /* include timing info always in error messages */ + result.error = t_strdup_printf("%s (reply took %s)", + result.error, dict_warnings_sec(cmd, diff, extra_args)); + } else if (!cmd->background && + diff >= (int)dict->warn_slow_msecs) { + e_warning(dict->conn.conn.event, "dict lookup took %s: %s", + dict_warnings_sec(cmd, diff, extra_args), + cmd->query); + } + + dict_pre_api_callback(&dict->dict); + cmd->api_callback.lookup(&result, cmd->api_callback.context); + dict_post_api_callback(&dict->dict); +} + +static void +client_dict_lookup_async(struct dict *_dict, const struct dict_op_settings *set, + const char *key, dict_lookup_callback_t *callback, + void *context) +{ + struct client_dict *dict = (struct client_dict *)_dict; + struct client_dict_cmd *cmd; + const char *query; + + query = t_strdup_printf("%c%s\t%s", DICT_PROTOCOL_CMD_LOOKUP, + str_tabescape(key), + set->username == NULL ? "" : str_tabescape(set->username)); + cmd = client_dict_cmd_init(dict, query); + cmd->callback = client_dict_lookup_async_callback; + cmd->api_callback.lookup = callback; + cmd->api_callback.context = context; + cmd->retry_errors = TRUE; + + client_dict_cmd_send(dict, &cmd, NULL); +} + +struct client_dict_sync_lookup { + char *error; + char *value; + int ret; +}; + +static void client_dict_lookup_callback(const struct dict_lookup_result *result, + struct client_dict_sync_lookup *lookup) +{ + lookup->ret = result->ret; + if (result->ret == -1) + lookup->error = i_strdup(result->error); + else if (result->ret == 1) + lookup->value = i_strdup(result->value); +} + +static int client_dict_lookup(struct dict *_dict, + const struct dict_op_settings *set, + pool_t pool, const char *key, + const char **value_r, const char **error_r) +{ + struct client_dict_sync_lookup lookup; + + i_zero(&lookup); + lookup.ret = -2; + + dict_lookup_async(_dict, set, key, client_dict_lookup_callback, &lookup); + if (lookup.ret == -2) + client_dict_wait(_dict); + + switch (lookup.ret) { + case -1: + *error_r = t_strdup(lookup.error); + i_free(lookup.error); + return -1; + case 0: + i_assert(lookup.value == NULL); + *value_r = NULL; + return 0; + case 1: + *value_r = p_strdup(pool, lookup.value); + i_free(lookup.value); + return 1; + } + i_unreached(); +} + +static void client_dict_iterate_unref(struct client_dict_iterate_context *ctx) +{ + i_assert(ctx->refcount > 0); + if (--ctx->refcount > 0) + return; + i_free(ctx->error); + i_free(ctx); +} + +static void +client_dict_iter_api_callback(struct client_dict_iterate_context *ctx, + struct client_dict_cmd *cmd, + const char *const *extra_args) +{ + struct client_dict *dict = cmd->dict; + + if (ctx->deinit) { + /* Iterator was already deinitialized. Stop if we're in + client_dict_wait(). */ + dict_post_api_callback(&dict->dict); + return; + } + if (ctx->finished) { + int diff = timeval_diff_msecs(&ioloop_timeval, &cmd->start_time); + if (ctx->error != NULL) { + /* include timing info always in error messages */ + char *new_error = i_strdup_printf("%s (reply took %s)", + ctx->error, dict_warnings_sec(cmd, diff, extra_args)); + i_free(ctx->error); + ctx->error = new_error; + } else if (!cmd->background && + diff >= (int)dict->warn_slow_msecs) { + e_warning(dict->conn.conn.event, "dict iteration took %s: %s", + dict_warnings_sec(cmd, diff, extra_args), + cmd->query); + } + } + if (ctx->ctx.async_callback != NULL) { + dict_pre_api_callback(&dict->dict); + ctx->ctx.async_callback(ctx->ctx.async_context); + dict_post_api_callback(&dict->dict); + } else { + /* synchronous lookup */ + io_loop_stop(dict->dict.ioloop); + } +} + +static void +client_dict_iter_async_callback(struct client_dict_cmd *cmd, + enum dict_protocol_reply reply, + const char *value, + const char *const *extra_args, + const char *error, + bool disconnected ATTR_UNUSED) +{ + struct client_dict_iterate_context *ctx = cmd->iter; + struct client_dict *dict = cmd->dict; + struct client_dict_iter_result *result; + const char *iter_key = NULL, *const *iter_values = NULL; + + if (ctx->deinit) { + cmd->background = TRUE; + client_dict_cmd_backgrounded(dict); + } + + if (error != NULL) { + /* failed */ + } else switch (reply) { + case DICT_PROTOCOL_REPLY_ITER_FINISHED: + /* end of iteration */ + i_assert(!ctx->finished); + ctx->finished = TRUE; + client_dict_iter_api_callback(ctx, cmd, extra_args); + client_dict_iterate_unref(ctx); + return; + case DICT_PROTOCOL_REPLY_OK: + /* key \t value */ + iter_key = value; + iter_values = extra_args; + extra_args++; + break; + case DICT_PROTOCOL_REPLY_FAIL: + error = t_strdup_printf("dict-server returned failure: %s", value); + break; + default: + break; + } + if ((iter_values == NULL || iter_values[0] == NULL) && error == NULL) { + /* broken protocol */ + error = t_strdup_printf("dict client (%s) sent broken iterate reply: %c%s", + dict->conn.conn.name, reply, value); + client_dict_disconnect(dict, error); + } + + if (error != NULL) { + if (ctx->error == NULL) + ctx->error = i_strdup(error); + i_assert(!ctx->finished); + ctx->finished = TRUE; + client_dict_iter_api_callback(ctx, cmd, extra_args); + client_dict_iterate_unref(ctx); + return; + } + cmd->unfinished = TRUE; + + if (ctx->deinit) { + /* iterator was already deinitialized */ + return; + } + + result = array_append_space(&ctx->results); + result->key = p_strdup(ctx->results_pool, iter_key); + result->values = p_strarray_dup(ctx->results_pool, iter_values); + + client_dict_iter_api_callback(ctx, cmd, NULL); +} + +static struct dict_iterate_context * +client_dict_iterate_init(struct dict *_dict, + const struct dict_op_settings *set ATTR_UNUSED, + const char *path, enum dict_iterate_flags flags) +{ + struct client_dict_iterate_context *ctx; + + ctx = i_new(struct client_dict_iterate_context, 1); + ctx->ctx.dict = _dict; + ctx->results_pool = pool_alloconly_create("client dict iteration", 512); + ctx->flags = flags; + ctx->path = i_strdup(path); + ctx->refcount = 1; + i_array_init(&ctx->results, 64); + return &ctx->ctx; +} + +static void +client_dict_iterate_cmd_send(struct client_dict_iterate_context *ctx) +{ + struct client_dict *dict = (struct client_dict *)ctx->ctx.dict; + const struct dict_op_settings_private *set = &ctx->ctx.set; + struct client_dict_cmd *cmd; + string_t *query = t_str_new(256); + + /* we can't do this query in _iterate_init(), because + _set_limit() hasn't been called yet at that point. */ + str_printfa(query, "%c%d\t%"PRIu64, DICT_PROTOCOL_CMD_ITERATE, + ctx->flags, ctx->ctx.max_rows); + str_append_c(query, '\t'); + str_append_tabescaped(query, ctx->path); + str_append_c(query, '\t'); + str_append_tabescaped(query, set->username == NULL ? "" : set->username); + + cmd = client_dict_cmd_init(dict, str_c(query)); + cmd->iter = ctx; + cmd->callback = client_dict_iter_async_callback; + cmd->retry_errors = TRUE; + + ctx->refcount++; + client_dict_cmd_send(dict, &cmd, NULL); +} + +static bool client_dict_iterate(struct dict_iterate_context *_ctx, + const char **key_r, const char *const **values_r) +{ + struct client_dict_iterate_context *ctx = + (struct client_dict_iterate_context *)_ctx; + const struct client_dict_iter_result *results; + unsigned int count; + + if (ctx->error != NULL) { + ctx->ctx.has_more = FALSE; + return FALSE; + } + + results = array_get(&ctx->results, &count); + if (ctx->result_idx < count) { + *key_r = results[ctx->result_idx].key; + *values_r = results[ctx->result_idx].values; + ctx->ctx.has_more = TRUE; + ctx->result_idx++; + ctx->seen_results = TRUE; + return TRUE; + } + if (!ctx->cmd_sent) { + ctx->cmd_sent = TRUE; + client_dict_iterate_cmd_send(ctx); + return client_dict_iterate(_ctx, key_r, values_r); + } + ctx->ctx.has_more = !ctx->finished; + ctx->result_idx = 0; + array_clear(&ctx->results); + p_clear(ctx->results_pool); + + if ((ctx->flags & DICT_ITERATE_FLAG_ASYNC) == 0 && ctx->ctx.has_more) { + client_dict_wait(_ctx->dict); + return client_dict_iterate(_ctx, key_r, values_r); + } + return FALSE; +} + +static int client_dict_iterate_deinit(struct dict_iterate_context *_ctx, + const char **error_r) +{ + struct client_dict *dict = (struct client_dict *)_ctx->dict; + struct client_dict_iterate_context *ctx = + (struct client_dict_iterate_context *)_ctx; + int ret = ctx->error != NULL ? -1 : 0; + + i_assert(!ctx->deinit); + ctx->deinit = TRUE; + + *error_r = t_strdup(ctx->error); + array_free(&ctx->results); + pool_unref(&ctx->results_pool); + i_free(ctx->path); + client_dict_iterate_unref(ctx); + + client_dict_add_timeout(dict); + return ret; +} + +static struct dict_transaction_context * +client_dict_transaction_init(struct dict *_dict) +{ + struct client_dict *dict = (struct client_dict *)_dict; + struct client_dict_transaction_context *ctx; + + ctx = i_new(struct client_dict_transaction_context, 1); + ctx->ctx.dict = _dict; + ctx->id = ++dict->transaction_id_counter; + + DLLIST_PREPEND(&dict->transactions, ctx); + return &ctx->ctx; +} + +static void +client_dict_transaction_free(struct client_dict_transaction_context **_ctx) +{ + struct client_dict_transaction_context *ctx = *_ctx; + + *_ctx = NULL; + i_free(ctx->first_query); + i_free(ctx->error); + i_free(ctx); +} + +static void +client_dict_transaction_commit_callback(struct client_dict_cmd *cmd, + enum dict_protocol_reply reply, + const char *value, + const char *const *extra_args, + const char *error, bool disconnected) +{ + struct client_dict *dict = cmd->dict; + struct dict_commit_result result = { + .ret = DICT_COMMIT_RET_FAILED, .error = NULL + }; + + i_assert(cmd->trans != NULL); + + if (error != NULL) { + /* failed */ + if (disconnected) + result.ret = DICT_COMMIT_RET_WRITE_UNCERTAIN; + result.error = error; + } else switch (reply) { + case DICT_PROTOCOL_REPLY_OK: + result.ret = DICT_COMMIT_RET_OK; + break; + case DICT_PROTOCOL_REPLY_NOTFOUND: + result.ret = DICT_COMMIT_RET_NOTFOUND; + break; + case DICT_PROTOCOL_REPLY_WRITE_UNCERTAIN: + result.ret = DICT_COMMIT_RET_WRITE_UNCERTAIN; + /* fallthrough */ + case DICT_PROTOCOL_REPLY_FAIL: { + /* value contains the obsolete trans_id */ + const char *error = extra_args[0]; + + result.error = t_strdup_printf("dict-server returned failure: %s", + error != NULL ? t_str_tabunescape(error) : ""); + if (error != NULL) + extra_args++; + break; + } + default: + result.ret = DICT_COMMIT_RET_FAILED; + result.error = t_strdup_printf( + "dict-client: Invalid commit reply: %c%s", + reply, value); + client_dict_disconnect(dict, result.error); + break; + } + + int diff = timeval_diff_msecs(&ioloop_timeval, &cmd->start_time); + if (result.error != NULL) { + /* include timing info always in error messages */ + result.error = t_strdup_printf("%s (reply took %s)", + result.error, dict_warnings_sec(cmd, diff, extra_args)); + } else if (!cmd->background && !cmd->trans->ctx.no_slowness_warning && + diff >= (int)dict->warn_slow_msecs) { + e_warning(dict->conn.conn.event, "dict commit took %s: " + "%s (%u commands, first: %s)", + dict_warnings_sec(cmd, diff, extra_args), + cmd->query, cmd->trans->query_count, + cmd->trans->first_query); + } + client_dict_transaction_free(&cmd->trans); + + dict_pre_api_callback(&dict->dict); + cmd->api_callback.commit(&result, cmd->api_callback.context); + dict_post_api_callback(&dict->dict); +} + + +static void +client_dict_transaction_commit(struct dict_transaction_context *_ctx, + bool async, + dict_transaction_commit_callback_t *callback, + void *context) +{ + struct client_dict_transaction_context *ctx = + (struct client_dict_transaction_context *)_ctx; + struct client_dict *dict = (struct client_dict *)_ctx->dict; + struct client_dict_cmd *cmd; + const char *query; + + DLLIST_REMOVE(&dict->transactions, ctx); + + if (ctx->sent_begin && ctx->error == NULL) { + query = t_strdup_printf("%c%u", DICT_PROTOCOL_CMD_COMMIT, ctx->id); + cmd = client_dict_cmd_init(dict, query); + cmd->trans = ctx; + + cmd->callback = client_dict_transaction_commit_callback; + cmd->api_callback.commit = callback; + cmd->api_callback.context = context; + if (callback == dict_transaction_commit_async_noop_callback) + cmd->background = TRUE; + if (client_dict_cmd_send(dict, &cmd, NULL)) { + if (!async) + client_dict_wait(_ctx->dict); + } + } else if (ctx->error != NULL) { + /* already failed */ + struct dict_commit_result result = { + .ret = DICT_COMMIT_RET_FAILED, .error = ctx->error + }; + callback(&result, context); + client_dict_transaction_free(&ctx); + } else { + /* nothing changed */ + struct dict_commit_result result = { + .ret = DICT_COMMIT_RET_OK, .error = NULL + }; + callback(&result, context); + client_dict_transaction_free(&ctx); + } + + client_dict_add_timeout(dict); +} + +static void +client_dict_transaction_rollback(struct dict_transaction_context *_ctx) +{ + struct client_dict_transaction_context *ctx = + (struct client_dict_transaction_context *)_ctx; + struct client_dict *dict = (struct client_dict *)_ctx->dict; + + if (ctx->sent_begin) { + const char *query; + + query = t_strdup_printf("%c%u", DICT_PROTOCOL_CMD_ROLLBACK, + ctx->id); + client_dict_send_transaction_query(ctx, query); + } + + DLLIST_REMOVE(&dict->transactions, ctx); + client_dict_transaction_free(&ctx); + client_dict_add_timeout(dict); +} + +static void client_dict_set(struct dict_transaction_context *_ctx, + const char *key, const char *value) +{ + struct client_dict_transaction_context *ctx = + (struct client_dict_transaction_context *)_ctx; + const char *query; + + query = t_strdup_printf("%c%u\t%s\t%s", + DICT_PROTOCOL_CMD_SET, ctx->id, + str_tabescape(key), + str_tabescape(value)); + client_dict_send_transaction_query(ctx, query); +} + +static void client_dict_unset(struct dict_transaction_context *_ctx, + const char *key) +{ + struct client_dict_transaction_context *ctx = + (struct client_dict_transaction_context *)_ctx; + const char *query; + + query = t_strdup_printf("%c%u\t%s", + DICT_PROTOCOL_CMD_UNSET, ctx->id, + str_tabescape(key)); + client_dict_send_transaction_query(ctx, query); +} + +static void client_dict_atomic_inc(struct dict_transaction_context *_ctx, + const char *key, long long diff) +{ + struct client_dict_transaction_context *ctx = + (struct client_dict_transaction_context *)_ctx; + const char *query; + + query = t_strdup_printf("%c%u\t%s\t%lld", + DICT_PROTOCOL_CMD_ATOMIC_INC, + ctx->id, str_tabescape(key), diff); + client_dict_send_transaction_query(ctx, query); +} + +static void client_dict_set_timestamp(struct dict_transaction_context *_ctx, + const struct timespec *ts) +{ + struct client_dict_transaction_context *ctx = + (struct client_dict_transaction_context *)_ctx; + const char *query; + + query = t_strdup_printf("%c%u\t%s\t%u", + DICT_PROTOCOL_CMD_TIMESTAMP, + ctx->id, dec2str(ts->tv_sec), + (unsigned int)ts->tv_nsec); + client_dict_send_transaction_query(ctx, query); +} + +struct dict dict_driver_client = { + .name = "proxy", + + { + .init = client_dict_init, + .deinit = client_dict_deinit, + .wait = client_dict_wait, + .lookup = client_dict_lookup, + .iterate_init = client_dict_iterate_init, + .iterate = client_dict_iterate, + .iterate_deinit = client_dict_iterate_deinit, + .transaction_init = client_dict_transaction_init, + .transaction_commit = client_dict_transaction_commit, + .transaction_rollback = client_dict_transaction_rollback, + .set = client_dict_set, + .unset = client_dict_unset, + .atomic_inc = client_dict_atomic_inc, + .lookup_async = client_dict_lookup_async, + .switch_ioloop = client_dict_switch_ioloop, + .set_timestamp = client_dict_set_timestamp, + } +}; diff --git a/src/lib-dict/dict-client.h b/src/lib-dict/dict-client.h new file mode 100644 index 0000000..5c14f73 --- /dev/null +++ b/src/lib-dict/dict-client.h @@ -0,0 +1,47 @@ +#ifndef DICT_CLIENT_H +#define DICT_CLIENT_H + +#include "dict.h" + +#define DEFAULT_DICT_SERVER_SOCKET_FNAME "dict" + +#define DICT_CLIENT_PROTOCOL_MAJOR_VERSION 3 +#define DICT_CLIENT_PROTOCOL_MINOR_VERSION 2 + +#define DICT_CLIENT_PROTOCOL_VERSION_MIN_MULTI_OK 2 + +#define DICT_CLIENT_MAX_LINE_LENGTH (64*1024) + +enum dict_protocol_cmd { + /* <major-version> <minor-version> <value type> <user> <dict name> */ + DICT_PROTOCOL_CMD_HELLO = 'H', + + DICT_PROTOCOL_CMD_LOOKUP = 'L', /* <key> */ + DICT_PROTOCOL_CMD_ITERATE = 'I', /* <flags> <path> */ + + DICT_PROTOCOL_CMD_BEGIN = 'B', /* <id> */ + DICT_PROTOCOL_CMD_COMMIT = 'C', /* <id> */ + DICT_PROTOCOL_CMD_COMMIT_ASYNC = 'D', /* <id> */ + DICT_PROTOCOL_CMD_ROLLBACK = 'R', /* <id> */ + + DICT_PROTOCOL_CMD_SET = 'S', /* <id> <key> <value> */ + DICT_PROTOCOL_CMD_UNSET = 'U', /* <id> <key> */ + DICT_PROTOCOL_CMD_ATOMIC_INC = 'A', /* <id> <key> <diff> */ + DICT_PROTOCOL_CMD_TIMESTAMP = 'T', /* <id> <secs> <nsecs> */ +}; + +enum dict_protocol_reply { + DICT_PROTOCOL_REPLY_ERROR = -1, + + DICT_PROTOCOL_REPLY_OK = 'O', /* <value> */ + DICT_PROTOCOL_REPLY_MULTI_OK = 'M', /* protocol v2.2+ */ + DICT_PROTOCOL_REPLY_NOTFOUND = 'N', + DICT_PROTOCOL_REPLY_FAIL = 'F', + DICT_PROTOCOL_REPLY_WRITE_UNCERTAIN = 'W', + DICT_PROTOCOL_REPLY_ASYNC_COMMIT = 'A', + DICT_PROTOCOL_REPLY_ITER_FINISHED = '\0', + DICT_PROTOCOL_REPLY_ASYNC_ID = '*', + DICT_PROTOCOL_REPLY_ASYNC_REPLY = '+', +}; + +#endif diff --git a/src/lib-dict/dict-fail.c b/src/lib-dict/dict-fail.c new file mode 100644 index 0000000..101750b --- /dev/null +++ b/src/lib-dict/dict-fail.c @@ -0,0 +1,134 @@ +/* Copyright (c) 2017-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "dict.h" +#include "dict-private.h" + +struct dict_iterate_context dict_iter_unsupported = +{ + .dict = &dict_driver_fail, +}; + +struct dict_transaction_context dict_transaction_unsupported = +{ + .dict = &dict_driver_fail, +}; + +static int dict_fail_init(struct dict *dict_driver ATTR_UNUSED, + const char *uri ATTR_UNUSED, + const struct dict_settings *set ATTR_UNUSED, + struct dict **dict_r ATTR_UNUSED, const char **error_r) +{ + *error_r = "Unsupported operation (dict does not support this feature)"; + return -1; +} + +static void dict_fail_deinit(struct dict *dict ATTR_UNUSED) +{ +} + +static void dict_fail_wait(struct dict *dict ATTR_UNUSED) +{ +} + +static int dict_fail_lookup(struct dict *dict ATTR_UNUSED, + const struct dict_op_settings *set ATTR_UNUSED, + pool_t pool ATTR_UNUSED, + const char *key ATTR_UNUSED, const char **value_r ATTR_UNUSED, + const char **error_r) +{ + *error_r = "Unsupported operation (dict does not support this feature)"; + return -1; +} + +static struct dict_iterate_context * +dict_fail_iterate_init(struct dict *dict ATTR_UNUSED, + const struct dict_op_settings *set ATTR_UNUSED, + const char *path ATTR_UNUSED, + enum dict_iterate_flags flags ATTR_UNUSED) +{ + return &dict_iter_unsupported; +} + +static bool dict_fail_iterate(struct dict_iterate_context *ctx ATTR_UNUSED, + const char **key_r ATTR_UNUSED, + const char *const **values_r ATTR_UNUSED) +{ + return FALSE; +} + +static int dict_fail_iterate_deinit(struct dict_iterate_context *ctx ATTR_UNUSED, + const char **error_r) +{ + *error_r = "Unsupported operation (dict does not support this feature)"; + return -1; +} + +static struct dict_transaction_context *dict_fail_transaction_init(struct dict *dict ATTR_UNUSED) +{ + return &dict_transaction_unsupported; +} + +static void dict_fail_transaction_commit(struct dict_transaction_context *ctx ATTR_UNUSED, + bool async ATTR_UNUSED, + dict_transaction_commit_callback_t *callback, + void *context) +{ + struct dict_commit_result res = { + .ret = DICT_COMMIT_RET_FAILED, + .error = "Unsupported operation (dict does not support this feature)" + }; + if (callback != NULL) + callback(&res, context); +} + +static void dict_fail_transaction_rollback(struct dict_transaction_context *ctx ATTR_UNUSED) +{ +} + +static void dict_fail_set(struct dict_transaction_context *ctx ATTR_UNUSED, + const char *key ATTR_UNUSED, const char *value ATTR_UNUSED) +{ +} + +static void dict_fail_unset(struct dict_transaction_context *ctx ATTR_UNUSED, + const char *key ATTR_UNUSED) +{ +} + +static void dict_fail_atomic_inc(struct dict_transaction_context *ctx ATTR_UNUSED, + const char *key ATTR_UNUSED, long long diff ATTR_UNUSED) +{ +} + +static bool dict_fail_switch_ioloop(struct dict *dict ATTR_UNUSED) +{ + return TRUE; +} + +static void dict_fail_set_timestamp(struct dict_transaction_context *ctx ATTR_UNUSED, + const struct timespec *ts ATTR_UNUSED) +{ +} + +struct dict dict_driver_fail = { + .name = "fail", + .v = { + .init = dict_fail_init, + .deinit = dict_fail_deinit, + .wait = dict_fail_wait, + .lookup = dict_fail_lookup, + .iterate_init = dict_fail_iterate_init, + .iterate = dict_fail_iterate, + .iterate_deinit = dict_fail_iterate_deinit, + .transaction_init = dict_fail_transaction_init, + .transaction_commit = dict_fail_transaction_commit, + .transaction_rollback = dict_fail_transaction_rollback, + .set = dict_fail_set, + .unset = dict_fail_unset, + .atomic_inc = dict_fail_atomic_inc, + .lookup_async = NULL, + .switch_ioloop = dict_fail_switch_ioloop, + .set_timestamp = dict_fail_set_timestamp + }, +}; diff --git a/src/lib-dict/dict-file.c b/src/lib-dict/dict-file.c new file mode 100644 index 0000000..c9228a8 --- /dev/null +++ b/src/lib-dict/dict-file.c @@ -0,0 +1,709 @@ +/* Copyright (c) 2008-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "hash.h" +#include "str.h" +#include "strescape.h" +#include "home-expand.h" +#include "mkdir-parents.h" +#include "eacces-error.h" +#include "file-lock.h" +#include "file-dotlock.h" +#include "nfs-workarounds.h" +#include "istream.h" +#include "ostream.h" +#include "dict-transaction-memory.h" +#include "dict-private.h" + +#include <stdio.h> +#include <unistd.h> +#include <fcntl.h> +#include <sys/stat.h> + +struct file_dict { + struct dict dict; + pool_t hash_pool; + enum file_lock_method lock_method; + + char *path; + char *home_dir; + bool dict_path_checked; + HASH_TABLE(char *, char *) hash; + int fd; + + bool refreshed; +}; + +struct file_dict_iterate_context { + struct dict_iterate_context ctx; + pool_t pool; + + struct hash_iterate_context *iter; + const char *path; + size_t path_len; + + enum dict_iterate_flags flags; + const char *values[2]; + const char *error; +}; + +static struct dotlock_settings file_dict_dotlock_settings = { + .timeout = 60*2, + .stale_timeout = 60, + .use_io_notify = TRUE +}; + +static int +file_dict_ensure_path_home_dir(struct file_dict *dict, const char *home_dir, + const char **error_r) +{ + if (null_strcmp(dict->home_dir, home_dir) == 0) + return 0; + + if (dict->dict_path_checked) { + *error_r = t_strdup_printf("home_dir changed from %s to %s " + "(requested dict was: %s)", dict->home_dir, + home_dir, dict->path); + return -1; + } + + char *_p = dict->path; + dict->path = i_strdup(home_expand_tilde(dict->path, home_dir)); + dict->home_dir = i_strdup(home_dir); + i_free(_p); + dict->dict_path_checked = TRUE; + return 0; +} + +static int +file_dict_init(struct dict *driver, const char *uri, + const struct dict_settings *set ATTR_UNUSED, + struct dict **dict_r, const char **error_r) +{ + struct file_dict *dict; + const char *p, *path; + + dict = i_new(struct file_dict, 1); + dict->lock_method = FILE_LOCK_METHOD_DOTLOCK; + + p = strchr(uri, ':'); + if (p == NULL) { + /* no parameters */ + path = uri; + } else { + path = t_strdup_until(uri, p++); + if (strcmp(p, "lock=fcntl") == 0) + dict->lock_method = FILE_LOCK_METHOD_FCNTL; + else if (strcmp(p, "lock=flock") == 0) + dict->lock_method = FILE_LOCK_METHOD_FLOCK; + else { + *error_r = t_strdup_printf("Invalid parameter: %s", p+1); + i_free(dict); + return -1; + } + } + + /* keep the path for now, later in dict operations check if home_dir + should be prepended. */ + dict->path = i_strdup(path); + + dict->dict = *driver; + dict->hash_pool = pool_alloconly_create("file dict", 1024); + hash_table_create(&dict->hash, dict->hash_pool, 0, str_hash, strcmp); + dict->fd = -1; + *dict_r = &dict->dict; + return 0; +} + +static void file_dict_deinit(struct dict *_dict) +{ + struct file_dict *dict = (struct file_dict *)_dict; + + i_close_fd_path(&dict->fd, dict->path); + hash_table_destroy(&dict->hash); + pool_unref(&dict->hash_pool); + i_free(dict->path); + i_free(dict->home_dir); + i_free(dict); +} + +static bool file_dict_need_refresh(struct file_dict *dict) +{ + struct stat st1, st2; + + if (dict->dict.iter_count > 0) { + /* Change nothing while there are iterators or they can crash + because the hash table content recreated. */ + return FALSE; + } + + if (dict->fd == -1) + return TRUE; + + /* Disable NFS flushing for now since it can cause unnecessary + problems and there's no easy way for us to know here if + mail_nfs_storage=yes. In any case it's pretty much an unsupported + setting nowadays. */ + /*nfs_flush_file_handle_cache(dict->path);*/ + if (nfs_safe_stat(dict->path, &st1) < 0) { + e_error(dict->dict.event, "stat(%s) failed: %m", dict->path); + return FALSE; + } + + if (fstat(dict->fd, &st2) < 0) { + if (errno != ESTALE) + e_error(dict->dict.event, "fstat(%s) failed: %m", dict->path); + return TRUE; + } + if (st1.st_ino != st2.st_ino || + !CMP_DEV_T(st1.st_dev, st2.st_dev)) { + /* file changed */ + return TRUE; + } + return FALSE; +} + +static int file_dict_open_latest(struct file_dict *dict, const char **error_r) +{ + int open_type; + + if (!file_dict_need_refresh(dict)) + return 0; + + i_close_fd_path(&dict->fd, dict->path); + + open_type = dict->lock_method == FILE_LOCK_METHOD_DOTLOCK ? + O_RDONLY : O_RDWR; + dict->fd = open(dict->path, open_type); + if (dict->fd == -1) { + if (errno == ENOENT) + return 0; + if (errno == EACCES) + *error_r = eacces_error_get("open", dict->path); + else + *error_r = t_strdup_printf("open(%s) failed: %m", dict->path); + return -1; + } + dict->refreshed = FALSE; + return 1; +} + +static int file_dict_refresh(struct file_dict *dict, const char **error_r) +{ + struct istream *input; + char *key, *value; + + if (file_dict_open_latest(dict, error_r) < 0) + return -1; + if (dict->refreshed || dict->dict.iter_count > 0) + return 0; + + hash_table_clear(dict->hash, TRUE); + p_clear(dict->hash_pool); + + if (dict->fd != -1) { + input = i_stream_create_fd(dict->fd, SIZE_MAX); + + while ((key = i_stream_read_next_line(input)) != NULL) { + /* strdup() before the second read */ + key = str_tabunescape(p_strdup(dict->hash_pool, key)); + + if ((value = i_stream_read_next_line(input)) == NULL) + break; + + value = str_tabunescape(p_strdup(dict->hash_pool, value)); + hash_table_update(dict->hash, key, value); + } + i_stream_destroy(&input); + } + dict->refreshed = TRUE; + return 0; +} + +static int file_dict_lookup(struct dict *_dict, + const struct dict_op_settings *set, + pool_t pool, const char *key, + const char **value_r, const char **error_r) +{ + struct file_dict *dict = (struct file_dict *)_dict; + + if (file_dict_ensure_path_home_dir(dict, set->home_dir, error_r) < 0) + return -1; + + if (file_dict_refresh(dict, error_r) < 0) + return -1; + + *value_r = p_strdup(pool, hash_table_lookup(dict->hash, key)); + return *value_r == NULL ? 0 : 1; +} + +static struct dict_iterate_context * +file_dict_iterate_init(struct dict *_dict, + const struct dict_op_settings *set ATTR_UNUSED, + const char *path, enum dict_iterate_flags flags) +{ + struct file_dict_iterate_context *ctx; + struct file_dict *dict = (struct file_dict *)_dict; + const char *error; + pool_t pool; + + pool = pool_alloconly_create("file dict iterate", 256); + ctx = p_new(pool, struct file_dict_iterate_context, 1); + ctx->ctx.dict = _dict; + ctx->pool = pool; + + ctx->path = p_strdup(pool, path); + ctx->path_len = strlen(path); + ctx->flags = flags; + + if (file_dict_ensure_path_home_dir(dict, set->home_dir, &error) < 0 || + file_dict_refresh(dict, &error) < 0) + ctx->error = p_strdup(pool, error); + + ctx->iter = hash_table_iterate_init(dict->hash); + return &ctx->ctx; +} + +static bool +file_dict_iterate_key_matches(struct file_dict_iterate_context *ctx, + const char *key) +{ + if (strncmp(ctx->path, key, ctx->path_len) == 0) + return TRUE; + return FALSE; +} + + +static bool file_dict_iterate(struct dict_iterate_context *_ctx, + const char **key_r, const char *const **values_r) +{ + struct file_dict_iterate_context *ctx = + (struct file_dict_iterate_context *)_ctx; + char *key, *value; + + while (hash_table_iterate(ctx->iter, + ((struct file_dict *)_ctx->dict)->hash, + &key, &value)) { + if (!file_dict_iterate_key_matches(ctx, key)) + continue; + + if ((ctx->flags & DICT_ITERATE_FLAG_RECURSE) != 0) { + /* match everything */ + } else if ((ctx->flags & DICT_ITERATE_FLAG_EXACT_KEY) != 0) { + if (key[ctx->path_len] != '\0') + continue; + } else { + if (strchr(key + ctx->path_len, '/') != NULL) + continue; + } + + *key_r = key; + ctx->values[0] = value; + *values_r = ctx->values; + return TRUE; + } + return FALSE; +} + +static int file_dict_iterate_deinit(struct dict_iterate_context *_ctx, + const char **error_r) +{ + struct file_dict_iterate_context *ctx = + (struct file_dict_iterate_context *)_ctx; + int ret = ctx->error != NULL ? -1 : 0; + + *error_r = t_strdup(ctx->error); + hash_table_iterate_deinit(&ctx->iter); + pool_unref(&ctx->pool); + return ret; +} + +static struct dict_transaction_context * +file_dict_transaction_init(struct dict *_dict) +{ + struct dict_transaction_memory_context *ctx; + pool_t pool; + + pool = pool_alloconly_create("file dict transaction", 2048); + ctx = p_new(pool, struct dict_transaction_memory_context, 1); + dict_transaction_memory_init(ctx, _dict, pool); + return &ctx->ctx; +} + +static void file_dict_apply_changes(struct dict_transaction_memory_context *ctx, + bool *atomic_inc_not_found_r) +{ + struct file_dict *dict = (struct file_dict *)ctx->ctx.dict; + const char *tmp; + char *key, *value, *old_value; + char *orig_key, *orig_value; + const struct dict_transaction_memory_change *change; + size_t new_len; + long long diff; + + array_foreach(&ctx->changes, change) { + if (hash_table_lookup_full(dict->hash, change->key, + &orig_key, &orig_value)) { + key = orig_key; + old_value = orig_value; + } else { + key = NULL; + old_value = NULL; + } + value = NULL; + + switch (change->type) { + case DICT_CHANGE_TYPE_INC: + if (old_value == NULL) { + *atomic_inc_not_found_r = TRUE; + break; + } + if (str_to_llong(old_value, &diff) < 0) + i_unreached(); + diff += change->value.diff; + tmp = t_strdup_printf("%lld", diff); + new_len = strlen(tmp); + if (old_value == NULL || new_len > strlen(old_value)) + value = p_strdup(dict->hash_pool, tmp); + else { + memcpy(old_value, tmp, new_len + 1); + value = old_value; + } + /* fall through */ + case DICT_CHANGE_TYPE_SET: + if (key == NULL) + key = p_strdup(dict->hash_pool, change->key); + if (value == NULL) { + value = p_strdup(dict->hash_pool, + change->value.str); + } + hash_table_update(dict->hash, key, value); + break; + case DICT_CHANGE_TYPE_UNSET: + if (old_value != NULL) + hash_table_remove(dict->hash, key); + break; + } + } +} + +static int +fd_copy_stat_permissions(const struct stat *src_st, + int dest_fd, const char *dest_path, + const char **error_r) +{ + struct stat dest_st; + + if (fstat(dest_fd, &dest_st) < 0) { + *error_r = t_strdup_printf("fstat(%s) failed: %m", dest_path); + return -1; + } + + if (src_st->st_gid != dest_st.st_gid && + ((src_st->st_mode & 0070) >> 3 != (src_st->st_mode & 0007))) { + /* group has different permissions from world. + preserve the group. */ + if (fchown(dest_fd, (uid_t)-1, src_st->st_gid) < 0) { + *error_r = t_strdup_printf("fchown(%s, -1, %s) failed: %m", + dest_path, dec2str(src_st->st_gid)); + return -1; + } + } + + if ((src_st->st_mode & 07777) != (dest_st.st_mode & 07777)) { + if (fchmod(dest_fd, src_st->st_mode & 07777) < 0) { + *error_r = t_strdup_printf("fchmod(%s, %o) failed: %m", + dest_path, (int)(src_st->st_mode & 0777)); + return -1; + } + } + return 0; +} + +static int fd_copy_permissions(int src_fd, const char *src_path, + int dest_fd, const char *dest_path, + const char **error_r) +{ + struct stat src_st; + + if (fstat(src_fd, &src_st) < 0) { + *error_r = t_strdup_printf("fstat(%s) failed: %m", src_path); + return -1; + } + return fd_copy_stat_permissions(&src_st, dest_fd, dest_path, error_r); +} + +static int +fd_copy_parent_dir_permissions(const char *src_path, int dest_fd, + const char *dest_path, const char **error_r) +{ + struct stat src_st; + const char *src_dir, *p; + + p = strrchr(src_path, '/'); + if (p == NULL) + src_dir = "."; + else + src_dir = t_strdup_until(src_path, p); + if (stat(src_dir, &src_st) < 0) { + *error_r = t_strdup_printf("stat(%s) failed: %m", src_dir); + return -1; + } + src_st.st_mode &= 0666; + return fd_copy_stat_permissions(&src_st, dest_fd, dest_path, error_r); +} + +static int file_dict_mkdir(struct file_dict *dict, const char **error_r) +{ + const char *path, *p, *root; + struct stat st; + mode_t mode = 0700; + + p = strrchr(dict->path, '/'); + if (p == NULL) + return 0; + path = t_strdup_until(dict->path, p); + + if (stat_first_parent(path, &root, &st) < 0) { + if (errno == EACCES) + *error_r = eacces_error_get("stat", root); + else + *error_r = t_strdup_printf("stat(%s) failed: %m", root); + return -1; + } + if ((st.st_mode & S_ISGID) != 0) { + /* preserve parent's permissions when it has setgid bit */ + mode = st.st_mode; + } + + if (mkdir_parents(path, mode) < 0 && errno != EEXIST) { + if (errno == EACCES) + *error_r = eacces_error_get("mkdir_parents", path); + else + *error_r = t_strdup_printf("mkdir_parents(%s) failed: %m", path); + return -1; + } + return 0; +} + +static int +file_dict_lock(struct file_dict *dict, struct file_lock **lock_r, + const char **error_r) +{ + int ret; + const char *error; + + if (file_dict_open_latest(dict, error_r) < 0) + return -1; + + if (dict->fd == -1) { + /* quota file doesn't exist yet, we need to create it */ + dict->fd = open(dict->path, O_CREAT | O_RDWR, 0600); + if (dict->fd == -1 && errno == ENOENT) { + if (file_dict_mkdir(dict, error_r) < 0) + return -1; + dict->fd = open(dict->path, O_CREAT | O_RDWR, 0600); + } + if (dict->fd == -1) { + if (errno == EACCES) + *error_r = eacces_error_get("creat", dict->path); + else { + *error_r = t_strdup_printf( + "creat(%s) failed: %m", dict->path); + } + return -1; + } + if (fd_copy_parent_dir_permissions(dict->path, dict->fd, + dict->path, &error) < 0) + e_error(dict->dict.event, "%s", error); + } + + *lock_r = NULL; + struct file_lock_settings lock_set = { + .lock_method = dict->lock_method, + }; + do { + file_lock_free(lock_r); + if (file_wait_lock(dict->fd, dict->path, F_WRLCK, &lock_set, + file_dict_dotlock_settings.timeout, + lock_r, &error) <= 0) { + *error_r = t_strdup_printf( + "file_wait_lock(%s) failed: %s", + dict->path, error); + return -1; + } + /* check again if we need to reopen the file because it was + just replaced */ + } while ((ret = file_dict_open_latest(dict, error_r)) > 0); + + return ret < 0 ? -1 : 0; +} + +static int +file_dict_write_changes(struct dict_transaction_memory_context *ctx, + bool *atomic_inc_not_found_r, const char **error_r) +{ + struct file_dict *dict = (struct file_dict *)ctx->ctx.dict; + struct dotlock *dotlock = NULL; + struct file_lock *lock = NULL; + const char *temp_path = NULL; + const char *error; + struct hash_iterate_context *iter; + struct ostream *output; + char *key, *value; + string_t *str; + int fd = -1; + + *atomic_inc_not_found_r = FALSE; + + if (file_dict_ensure_path_home_dir(dict, ctx->ctx.set.home_dir, error_r) < 0) + return -1; + + switch (dict->lock_method) { + case FILE_LOCK_METHOD_FCNTL: + case FILE_LOCK_METHOD_FLOCK: + if (file_dict_lock(dict, &lock, error_r) < 0) + return -1; + temp_path = t_strdup_printf("%s.tmp", dict->path); + fd = creat(temp_path, 0600); + if (fd == -1) { + *error_r = t_strdup_printf( + "dict-file: creat(%s) failed: %m", temp_path); + file_unlock(&lock); + return -1; + } + break; + case FILE_LOCK_METHOD_DOTLOCK: + fd = file_dotlock_open(&file_dict_dotlock_settings, dict->path, 0, + &dotlock); + if (fd == -1 && errno == ENOENT) { + if (file_dict_mkdir(dict, error_r) < 0) + return -1; + fd = file_dotlock_open(&file_dict_dotlock_settings, + dict->path, 0, &dotlock); + } + if (fd == -1) { + *error_r = t_strdup_printf( + "dict-file: file_dotlock_open(%s) failed: %m", + dict->path); + return -1; + } + temp_path = file_dotlock_get_lock_path(dotlock); + break; + } + + /* refresh once more now that we're locked */ + if (file_dict_refresh(dict, error_r) < 0) { + if (dotlock != NULL) + file_dotlock_delete(&dotlock); + else { + i_close_fd(&fd); + file_unlock(&lock); + } + return -1; + } + if (dict->fd != -1) { + /* preserve the permissions */ + if (fd_copy_permissions(dict->fd, dict->path, fd, temp_path, &error) < 0) + e_error(ctx->ctx.event, "%s", error); + } else { + /* get initial permissions from parent directory */ + if (fd_copy_parent_dir_permissions(dict->path, fd, temp_path, &error) < 0) + e_error(ctx->ctx.event, "%s", error); + } + file_dict_apply_changes(ctx, atomic_inc_not_found_r); + + output = o_stream_create_fd(fd, 0); + o_stream_cork(output); + iter = hash_table_iterate_init(dict->hash); + str = t_str_new(256); + while (hash_table_iterate(iter, dict->hash, &key, &value)) { + str_truncate(str, 0); + str_append_tabescaped(str, key); + str_append_c(str, '\n'); + str_append_tabescaped(str, value); + str_append_c(str, '\n'); + o_stream_nsend(output, str_data(str), str_len(str)); + } + hash_table_iterate_deinit(&iter); + + if (o_stream_finish(output) <= 0) { + *error_r = t_strdup_printf("write(%s) failed: %s", temp_path, + o_stream_get_error(output)); + o_stream_destroy(&output); + if (dotlock != NULL) + file_dotlock_delete(&dotlock); + else { + i_close_fd(&fd); + file_unlock(&lock); + } + return -1; + } + o_stream_destroy(&output); + + if (dotlock != NULL) { + if (file_dotlock_replace(&dotlock, + DOTLOCK_REPLACE_FLAG_DONT_CLOSE_FD) < 0) { + *error_r = t_strdup_printf("file_dotlock_replace() failed: %m"); + i_close_fd(&fd); + return -1; + } + } else { + if (rename(temp_path, dict->path) < 0) { + *error_r = t_strdup_printf("rename(%s, %s) failed: %m", + temp_path, dict->path); + file_unlock(&lock); + i_close_fd(&fd); + return -1; + } + /* dict->fd is locked, not the new fd. We're closing dict->fd + so we can just free the lock struct. */ + file_lock_free(&lock); + } + + i_close_fd(&dict->fd); + dict->fd = fd; + return 0; +} + +static void +file_dict_transaction_commit(struct dict_transaction_context *_ctx, + bool async ATTR_UNUSED, + dict_transaction_commit_callback_t *callback, + void *context) +{ + struct dict_transaction_memory_context *ctx = + (struct dict_transaction_memory_context *)_ctx; + struct dict_commit_result result; + bool atomic_inc_not_found; + + i_zero(&result); + if (file_dict_write_changes(ctx, &atomic_inc_not_found, &result.error) < 0) + result.ret = DICT_COMMIT_RET_FAILED; + else if (atomic_inc_not_found) + result.ret = DICT_COMMIT_RET_NOTFOUND; + else + result.ret = DICT_COMMIT_RET_OK; + pool_unref(&ctx->pool); + + callback(&result, context); +} + +struct dict dict_driver_file = { + .name = "file", + { + .init = file_dict_init, + .deinit = file_dict_deinit, + .lookup = file_dict_lookup, + .iterate_init = file_dict_iterate_init, + .iterate = file_dict_iterate, + .iterate_deinit = file_dict_iterate_deinit, + .transaction_init = file_dict_transaction_init, + .transaction_commit = file_dict_transaction_commit, + .transaction_rollback = dict_transaction_memory_rollback, + .set = dict_transaction_memory_set, + .unset = dict_transaction_memory_unset, + .atomic_inc = dict_transaction_memory_atomic_inc, + } +}; diff --git a/src/lib-dict/dict-iter-lua.c b/src/lib-dict/dict-iter-lua.c new file mode 100644 index 0000000..780de03 --- /dev/null +++ b/src/lib-dict/dict-iter-lua.c @@ -0,0 +1,193 @@ +/* Copyright (c) 2021 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "dict.h" +#include "dlua-script-private.h" +#include "dict-lua-private.h" +#include "dlua-wrapper.h" + +struct lua_dict_iter { + pool_t pool; + struct dict_iterate_context *iter; + ARRAY(int) refs; + int error_ref; + + lua_State *L; + bool yielded:1; +}; + +static void lua_dict_iter_unref(struct lua_dict_iter *iter) +{ + const char *error; + + /* deinit iteration if it hasn't been done yet */ + if (dict_iterate_deinit(&iter->iter, &error) < 0) { + e_error(dlua_script_from_state(iter->L)->event, + "Dict iteration failed: %s", error); + } + + pool_unref(&iter->pool); +} + +DLUA_WRAP_C_DATA(dict_iter, struct lua_dict_iter, lua_dict_iter_unref, NULL); + +static int lua_dict_iterate_step(lua_State *L); + +/* resume after a yield */ +static int lua_dict_iterate_step_continue(lua_State *L, + int status ATTR_UNUSED, + lua_KContext ctx ATTR_UNUSED) +{ + return lua_dict_iterate_step(L); +} + +static void lua_dict_iterate_more(struct lua_dict_iter *iter); + +/* + * Iteration step function + * + * Takes two args (a userdata state, and previous value) and returns the + * next value. + */ +static int lua_dict_iterate_step(lua_State *L) +{ + struct lua_dict_iter *iter; + const int *refs; + unsigned nrefs; + + DLUA_REQUIRE_ARGS(L, 2); + + iter = xlua_dict_iter_getptr(L, 1, NULL); + iter->yielded = FALSE; + + lua_dict_iterate_more(iter); + + if (iter->iter != NULL) { + /* iteration didn't end yet - yield */ + return lua_dict_iterate_step_continue(L, + lua_yieldk(L, 0, 0, lua_dict_iterate_step_continue), 0); + } + + /* dict iteration ended - return first key-value pair */ + refs = array_get(&iter->refs, &nrefs); + i_assert(nrefs % 2 == 0); + + if (nrefs == 0) { + if (iter->error_ref != 0) { + /* dict iteration generated an error - raise it now */ + lua_rawgeti(L, LUA_REGISTRYINDEX, iter->error_ref); + luaL_unref(L, LUA_REGISTRYINDEX, iter->error_ref); + return lua_error(L); + } + + return 0; /* return nil */ + } + + /* get the key & value from the registry */ + lua_rawgeti(L, LUA_REGISTRYINDEX, refs[0]); + lua_rawgeti(L, LUA_REGISTRYINDEX, refs[1]); + luaL_unref(L, LUA_REGISTRYINDEX, refs[0]); + luaL_unref(L, LUA_REGISTRYINDEX, refs[1]); + + array_delete(&iter->refs, 0, 2); + + return 2; +} + +static void lua_dict_iterate_more(struct lua_dict_iter *iter) +{ + const char *key, *const *values; + lua_State *L = iter->L; + const char *error; + + if (iter->iter == NULL) + return; /* done iterating the dict */ + + while (dict_iterate_values(iter->iter, &key, &values)) { + int ref; + + /* stash key */ + lua_pushstring(L, key); + ref = luaL_ref(L, LUA_REGISTRYINDEX); + array_push_back(&iter->refs, &ref); + + /* stash values */ + lua_newtable(L); + for (unsigned int i = 0; values[i] != NULL; i++) { + lua_pushstring(L, values[i]); + lua_seti(L, -2, i + 1); + } + ref = luaL_ref(L, LUA_REGISTRYINDEX); + array_push_back(&iter->refs, &ref); + } + + if (dict_iterate_has_more(iter->iter)) + return; + + if (dict_iterate_deinit(&iter->iter, &error) < 0) { + lua_pushstring(L, error); + iter->error_ref = luaL_ref(L, LUA_REGISTRYINDEX); + } +} + +/* dict iter callback */ +static void lua_dict_iterate_callback(struct lua_dict_iter *iter) +{ + if (iter->yielded) + return; + iter->yielded = TRUE; + dlua_pcall_yieldable_resume(iter->L, 1); +} + +/* + * Iterate a dict at key [-(3|4),+2,e] + * + * Args: + * 1) userdata: sturct dict *dict + * 2) string: key + * 3) integer: flags + * 4*) string: username + * + * Returns: + * Returns a iteration step function and dict iter userdata. + * Username will be NULL if not provided in args. + */ +int lua_dict_iterate(lua_State *L) +{ + enum dict_iterate_flags flags; + struct lua_dict_iter *iter; + struct dict *dict; + const char *path, *username = NULL; + pool_t pool; + + DLUA_REQUIRE_ARGS_IN(L, 3, 4); + + dict = dlua_check_dict(L, 1); + path = luaL_checkstring(L, 2); + flags = luaL_checkinteger(L, 3); + if (lua_gettop(L) >= 4) + username = luaL_checkstring(L, 4); + lua_dict_check_key_prefix(L, path, username); + + struct dict_op_settings set = { + .username = username, + }; + + /* set up iteration */ + pool = pool_alloconly_create("lua dict iter", 128); + iter = p_new(pool, struct lua_dict_iter, 1); + iter->pool = pool; + iter->iter = dict_iterate_init(dict, &set, path, flags | + DICT_ITERATE_FLAG_ASYNC); + p_array_init(&iter->refs, iter->pool, 32); + iter->L = L; + + dict_iterate_set_async_callback(iter->iter, + lua_dict_iterate_callback, iter); + + /* push return values: func, state */ + lua_pushcfunction(L, lua_dict_iterate_step); + xlua_pushdict_iter(L, iter, FALSE); + return 2; +} diff --git a/src/lib-dict/dict-lua-private.h b/src/lib-dict/dict-lua-private.h new file mode 100644 index 0000000..f9f9943 --- /dev/null +++ b/src/lib-dict/dict-lua-private.h @@ -0,0 +1,9 @@ +#ifndef DICT_LUA_PRIVATE_H +#define DICT_LUA_PRIVATE_H + +#include "dict-lua.h" + +int lua_dict_iterate(lua_State *l); +int lua_dict_transaction_begin(lua_State *l); + +#endif diff --git a/src/lib-dict/dict-lua.c b/src/lib-dict/dict-lua.c new file mode 100644 index 0000000..d5de534 --- /dev/null +++ b/src/lib-dict/dict-lua.c @@ -0,0 +1,117 @@ +/* Copyright (c) 2021 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "dict.h" +#include "dlua-script-private.h" +#include "dict-lua-private.h" +#include "dlua-wrapper.h" + +static int lua_dict_lookup(lua_State *); + +static luaL_Reg lua_dict_methods[] = { + { "lookup", lua_dict_lookup }, + { "iterate", lua_dict_iterate }, + { "transaction_begin", lua_dict_transaction_begin }, + { NULL, NULL }, +}; + +/* no actual ref counting */ +static void lua_dict_unref(struct dict *dict ATTR_UNUSED) +{ +} + +DLUA_WRAP_C_DATA(dict, struct dict, lua_dict_unref, lua_dict_methods); + +static int lua_dict_async_continue(lua_State *L, + int status ATTR_UNUSED, + lua_KContext ctx ATTR_UNUSED) +{ + /* + * lua_dict_*_callback() already pushed the result table/nil or error + * string. We simply need to return/error out. + */ + + if (lua_istable(L, -1) || lua_isnil(L, -1)) + return 1; + else + return lua_error(L); +} + +static void lua_dict_lookup_callback(const struct dict_lookup_result *result, + lua_State *L) +{ + if (result->ret < 0) { + lua_pushstring(L, result->error); + } else if (result->ret == 0) { + lua_pushnil(L); + } else { + unsigned int i; + + lua_newtable(L); + + for (i = 0; i < str_array_length(result->values); i++) { + lua_pushstring(L, result->values[i]); + lua_seti(L, -2, i + 1); + } + } + + dlua_pcall_yieldable_resume(L, 1); +} + +void lua_dict_check_key_prefix(lua_State *L, const char *key, + const char *username) +{ + if (str_begins(key, DICT_PATH_SHARED)) + ; + else if (str_begins(key, DICT_PATH_PRIVATE)) { + if (username == NULL || username[0] == '\0') + luaL_error(L, DICT_PATH_PRIVATE" dict key prefix requires username"); + } else { + luaL_error(L, "Invalid dict key prefix"); + } +} + +/* + * Lookup a key in dict [-(2|3),+1,e] + * + * Args: + * 1) userdata: struct dict *dict + * 2) string: key + * 3*) string: username + * + * Returns: + * If key is found, returns a table with values. If key is not found, + * returns nil. + * Username will be NULL if not provided in args. + */ +static int lua_dict_lookup(lua_State *L) +{ + struct dict *dict; + const char *key, *username = NULL; + + DLUA_REQUIRE_ARGS_IN(L, 2, 3); + + dict = xlua_dict_getptr(L, 1, NULL); + key = luaL_checkstring(L, 2); + if (lua_gettop(L) >= 3) + username = luaL_checkstring(L, 3); + lua_dict_check_key_prefix(L, key, username); + + struct dict_op_settings set = { + .username = username, + }; + dict_lookup_async(dict, &set, key, lua_dict_lookup_callback, L); + + return lua_dict_async_continue(L, + lua_yieldk(L, 0, 0, lua_dict_async_continue), 0); +} + +void dlua_push_dict(lua_State *L, struct dict *dict) +{ + xlua_pushdict(L, dict, FALSE); +} + +struct dict *dlua_check_dict(lua_State *L, int idx) +{ + return xlua_dict_getptr(L, idx, NULL); +} diff --git a/src/lib-dict/dict-lua.h b/src/lib-dict/dict-lua.h new file mode 100644 index 0000000..bf4255c --- /dev/null +++ b/src/lib-dict/dict-lua.h @@ -0,0 +1,18 @@ +#ifndef DICT_LUA_H +#define DICT_LUA_H + +#ifdef DLUA_WITH_YIELDS +/* + * Internally, the dict methods yield via lua_yieldk() as implemented in Lua + * 5.3 and newer. + */ + +void lua_dict_check_key_prefix(lua_State *L, const char *key, + const char *username); + +void dlua_push_dict(lua_State *L, struct dict *dict); +struct dict *dlua_check_dict(lua_State *L, int idx); + +#endif + +#endif diff --git a/src/lib-dict/dict-memcached-ascii.c b/src/lib-dict/dict-memcached-ascii.c new file mode 100644 index 0000000..6ae5443 --- /dev/null +++ b/src/lib-dict/dict-memcached-ascii.c @@ -0,0 +1,685 @@ +/* Copyright (c) 2008-2018 Dovecot authors, see the included COPYING memcached_ascii */ + +#include "lib.h" +#include "array.h" +#include "str.h" +#include "istream.h" +#include "ostream.h" +#include "connection.h" +#include "dict-transaction-memory.h" +#include "dict-private.h" + +#define MEMCACHED_DEFAULT_PORT 11211 +#define MEMCACHED_DEFAULT_LOOKUP_TIMEOUT_MSECS (1000*30) +#define DICT_USERNAME_SEPARATOR '/' + +enum memcached_ascii_input_state { + /* GET: expecting VALUE or END */ + MEMCACHED_INPUT_STATE_GET, + /* SET: expecting STORED / NOT_STORED */ + MEMCACHED_INPUT_STATE_STORED, + /* DELETE: expecting DELETED */ + MEMCACHED_INPUT_STATE_DELETED, + /* (INCR+ADD)/DECR: expecting number / NOT_FOUND / STORED / NOT_STORED */ + MEMCACHED_INPUT_STATE_INCRDECR +}; + +struct memcached_ascii_connection { + struct connection conn; + struct memcached_ascii_dict *dict; + + string_t *reply_str; + unsigned int reply_bytes_left; + bool value_received; + bool value_waiting_end; +}; + +struct memcached_ascii_dict_reply { + unsigned int reply_count; + dict_transaction_commit_callback_t *callback; + void *context; +}; + +struct dict_memcached_ascii_commit_ctx { + struct memcached_ascii_dict *dict; + struct dict_transaction_memory_context *memctx; + string_t *str; + + dict_transaction_commit_callback_t *callback; + void *context; +}; + +struct memcached_ascii_dict { + struct dict dict; + struct ip_addr ip; + char *key_prefix; + in_port_t port; + unsigned int timeout_msecs; + + struct timeout *to; + struct memcached_ascii_connection conn; + + ARRAY(enum memcached_ascii_input_state) input_states; + ARRAY(struct memcached_ascii_dict_reply) replies; +}; + +static struct connection_list *memcached_ascii_connections; + +static void +memcached_ascii_callback(struct memcached_ascii_dict *dict, + const struct memcached_ascii_dict_reply *reply, + const struct dict_commit_result *result) +{ + if (reply->callback != NULL) { + if (dict->dict.prev_ioloop != NULL) { + /* Don't let callback see that we've created our + internal ioloop in case it wants to add some ios + or timeouts. */ + current_ioloop = dict->dict.prev_ioloop; + } + reply->callback(result, reply->context); + if (dict->dict.prev_ioloop != NULL) + current_ioloop = dict->dict.ioloop; + } +} + +static void +memcached_ascii_disconnected(struct memcached_ascii_connection *conn, + const char *reason) +{ + const struct dict_commit_result result = { + DICT_COMMIT_RET_FAILED, reason + }; + const struct memcached_ascii_dict_reply *reply; + + connection_disconnect(&conn->conn); + if (conn->dict->dict.ioloop != NULL) + io_loop_stop(conn->dict->dict.ioloop); + + array_foreach(&conn->dict->replies, reply) + memcached_ascii_callback(conn->dict, reply, &result); + array_clear(&conn->dict->replies); + array_clear(&conn->dict->input_states); + conn->reply_bytes_left = 0; +} + +static void memcached_ascii_conn_destroy(struct connection *_conn) +{ + struct memcached_ascii_connection *conn = + (struct memcached_ascii_connection *)_conn; + + memcached_ascii_disconnected(conn, connection_disconnect_reason(_conn)); +} + +static bool memcached_ascii_input_value(struct memcached_ascii_connection *conn) +{ + const unsigned char *data; + size_t size; + + data = i_stream_get_data(conn->conn.input, &size); + if (size > conn->reply_bytes_left) + size = conn->reply_bytes_left; + conn->reply_bytes_left -= size; + + str_append_data(conn->reply_str, data, size); + i_stream_skip(conn->conn.input, size); + if (conn->reply_bytes_left > 0) + return FALSE; + + /* finished. drop the trailing CRLF */ + str_truncate(conn->reply_str, str_len(conn->reply_str)-2); + conn->value_received = TRUE; + return TRUE; +} + +static int memcached_ascii_input_reply_read(struct memcached_ascii_dict *dict, + const char **error_r) +{ + struct memcached_ascii_connection *conn = &dict->conn; + const enum memcached_ascii_input_state *states; + const char *line, *p; + unsigned int count; + long long num; + + if (conn->reply_bytes_left > 0) { + /* continue reading bulk reply */ + if (!memcached_ascii_input_value(conn)) + return 0; + conn->value_waiting_end = TRUE; + } else if (conn->value_waiting_end) { + conn->value_waiting_end = FALSE; + } else { + str_truncate(conn->reply_str, 0); + conn->value_received = FALSE; + } + + line = i_stream_next_line(conn->conn.input); + if (line == NULL) + return 0; + + states = array_get(&dict->input_states, &count); + if (count == 0) { + *error_r = t_strdup_printf( + "memcached_ascii: Unexpected input (expected nothing): %s", line); + return -1; + } + switch (states[0]) { + case MEMCACHED_INPUT_STATE_GET: + /* VALUE <key> <flags> <bytes> + END */ + if (str_begins(line, "VALUE ")) { + p = strrchr(line, ' '); + if (str_to_uint(p+1, &conn->reply_bytes_left) < 0) + break; + conn->reply_bytes_left += 2; /* CRLF */ + return memcached_ascii_input_reply_read(dict, error_r); + } else if (strcmp(line, "END") == 0) + return 1; + break; + case MEMCACHED_INPUT_STATE_STORED: + if (strcmp(line, "STORED") != 0 && + strcmp(line, "NOT_STORED") != 0) + break; + return 1; + case MEMCACHED_INPUT_STATE_DELETED: + if (strcmp(line, "DELETED") != 0) + break; + return 1; + case MEMCACHED_INPUT_STATE_INCRDECR: + if (strcmp(line, "NOT_FOUND") != 0 && + strcmp(line, "STORED") != 0 && + strcmp(line, "NOT_STORED") != 0 && + str_to_llong(line, &num) < 0) + break; + return 1; + } + *error_r = t_strdup_printf( + "memcached_ascii: Unexpected input (state=%d): %s", + states[0], line); + return -1; +} + +static int memcached_ascii_input_reply(struct memcached_ascii_dict *dict, + const char **error_r) +{ + const struct dict_commit_result result = { + DICT_COMMIT_RET_OK, NULL + }; + struct memcached_ascii_dict_reply *replies; + unsigned int count; + int ret; + + if ((ret = memcached_ascii_input_reply_read(dict, error_r)) <= 0) + return ret; + /* finished a reply */ + array_pop_front(&dict->input_states); + + replies = array_get_modifiable(&dict->replies, &count); + i_assert(count > 0); + i_assert(replies[0].reply_count > 0); + if (--replies[0].reply_count == 0) { + memcached_ascii_callback(dict, &replies[0], &result); + array_pop_front(&dict->replies); + } + return 1; +} + +static void memcached_ascii_conn_input(struct connection *_conn) +{ + struct memcached_ascii_connection *conn = + (struct memcached_ascii_connection *)_conn; + const char *error; + int ret; + + switch (i_stream_read(_conn->input)) { + case 0: + return; + case -1: + memcached_ascii_disconnected(conn, + i_stream_get_disconnect_reason(_conn->input)); + return; + default: + break; + } + + while ((ret = memcached_ascii_input_reply(conn->dict, &error)) > 0) ; + if (ret < 0) + memcached_ascii_disconnected(conn, error); + io_loop_stop(conn->dict->dict.ioloop); +} + +static int memcached_ascii_input_wait(struct memcached_ascii_dict *dict, + const char **error_r) +{ + i_assert(io_loop_is_empty(dict->dict.ioloop)); + dict->dict.prev_ioloop = current_ioloop; + io_loop_set_current(dict->dict.ioloop); + if (dict->to != NULL) + dict->to = io_loop_move_timeout(&dict->to); + connection_switch_ioloop(&dict->conn.conn); + io_loop_run(dict->dict.ioloop); + + io_loop_set_current(dict->dict.prev_ioloop); + dict->dict.prev_ioloop = NULL; + + if (dict->to != NULL) + dict->to = io_loop_move_timeout(&dict->to); + connection_switch_ioloop(&dict->conn.conn); + i_assert(io_loop_is_empty(dict->dict.ioloop)); + + if (dict->conn.conn.fd_in == -1) { + *error_r = "memcached_ascii: Communication failure"; + return -1; + } + return 0; +} + +static void memcached_ascii_input_timeout(struct memcached_ascii_dict *dict) +{ + const char *reason = t_strdup_printf( + "memcached_ascii: Request timed out in %u.%03u secs", + dict->timeout_msecs/1000, dict->timeout_msecs%1000); + memcached_ascii_disconnected(&dict->conn, reason); +} + +static int memcached_ascii_wait_replies(struct memcached_ascii_dict *dict, + const char **error_r) +{ + int ret = 0; + + dict->to = timeout_add(dict->timeout_msecs, + memcached_ascii_input_timeout, dict); + while (array_count(&dict->input_states) > 0) { + i_assert(array_count(&dict->replies) > 0); + + if ((ret = memcached_ascii_input_reply(dict, error_r)) != 0) { + if (ret < 0) + memcached_ascii_disconnected(&dict->conn, *error_r); + break; + } + ret = memcached_ascii_input_wait(dict, error_r); + if (ret != 0) + break; + } + + timeout_remove(&dict->to); + return ret < 0 ? -1 : 0; +} + +static int memcached_ascii_wait(struct memcached_ascii_dict *dict, + const char **error_r) +{ + int ret; + + i_assert(dict->conn.conn.fd_in != -1); + + if (dict->conn.conn.input == NULL) { + /* waiting for connection to finish */ + dict->to = timeout_add(dict->timeout_msecs, + memcached_ascii_input_timeout, dict); + ret = memcached_ascii_input_wait(dict, error_r); + timeout_remove(&dict->to); + if (ret < 0) + return -1; + } + if (memcached_ascii_wait_replies(dict, error_r) < 0) + return -1; + i_assert(array_count(&dict->input_states) == 0); + i_assert(array_count(&dict->replies) == 0); + return 0; +} + +static void +memcached_ascii_conn_connected(struct connection *_conn, bool success) +{ + struct memcached_ascii_connection *conn = (struct memcached_ascii_connection *)_conn; + + if (!success) { + e_error(conn->conn.event, "connect() failed: %m"); + } + if (conn->dict->dict.ioloop != NULL) + io_loop_stop(conn->dict->dict.ioloop); +} + +static const struct connection_settings memcached_ascii_conn_set = { + .input_max_size = SIZE_MAX, + .output_max_size = SIZE_MAX, + .client = TRUE +}; + +static const struct connection_vfuncs memcached_ascii_conn_vfuncs = { + .destroy = memcached_ascii_conn_destroy, + .input = memcached_ascii_conn_input, + .client_connected = memcached_ascii_conn_connected +}; + +static const char *memcached_ascii_escape_username(const char *username) +{ + const char *p; + string_t *str = t_str_new(64); + + for (p = username; *p != '\0'; p++) { + switch (*p) { + case DICT_USERNAME_SEPARATOR: + str_append(str, "\\-"); + break; + case '\\': + str_append(str, "\\\\"); + break; + default: + str_append_c(str, *p); + } + } + return str_c(str); +} + +static int +memcached_ascii_dict_init(struct dict *driver, const char *uri, + const struct dict_settings *set, + struct dict **dict_r, const char **error_r) +{ + struct memcached_ascii_dict *dict; + const char *const *args; + struct ioloop *old_ioloop = current_ioloop; + int ret = 0; + + if (memcached_ascii_connections == NULL) { + memcached_ascii_connections = + connection_list_init(&memcached_ascii_conn_set, + &memcached_ascii_conn_vfuncs); + } + + dict = i_new(struct memcached_ascii_dict, 1); + if (net_addr2ip("127.0.0.1", &dict->ip) < 0) + i_unreached(); + dict->port = MEMCACHED_DEFAULT_PORT; + dict->timeout_msecs = MEMCACHED_DEFAULT_LOOKUP_TIMEOUT_MSECS; + dict->key_prefix = i_strdup(""); + + args = t_strsplit(uri, ":"); + for (; *args != NULL; args++) { + if (str_begins(*args, "host=")) { + if (net_addr2ip(*args+5, &dict->ip) < 0) { + *error_r = t_strdup_printf("Invalid IP: %s", + *args+5); + ret = -1; + } + } else if (str_begins(*args, "port=")) { + if (net_str2port(*args+5, &dict->port) < 0) { + *error_r = t_strdup_printf("Invalid port: %s", + *args+5); + ret = -1; + } + } else if (str_begins(*args, "prefix=")) { + i_free(dict->key_prefix); + dict->key_prefix = i_strdup(*args + 7); + } else if (str_begins(*args, "timeout_msecs=")) { + if (str_to_uint(*args+14, &dict->timeout_msecs) < 0) { + *error_r = t_strdup_printf( + "Invalid timeout_msecs: %s", *args+14); + ret = -1; + } + } else { + *error_r = t_strdup_printf("Unknown parameter: %s", + *args); + ret = -1; + } + } + if (ret < 0) { + i_free(dict->key_prefix); + i_free(dict); + return -1; + } + + dict->conn.conn.event_parent = set->event_parent; + connection_init_client_ip(memcached_ascii_connections, &dict->conn.conn, + NULL, &dict->ip, dict->port); + event_set_append_log_prefix(dict->conn.conn.event, "memcached: "); + dict->dict = *driver; + dict->conn.reply_str = str_new(default_pool, 256); + dict->conn.dict = dict; + + i_array_init(&dict->input_states, 4); + i_array_init(&dict->replies, 4); + + dict->dict.ioloop = io_loop_create(); + io_loop_set_current(old_ioloop); + *dict_r = &dict->dict; + return 0; +} + +static void memcached_ascii_dict_deinit(struct dict *_dict) +{ + struct memcached_ascii_dict *dict = + (struct memcached_ascii_dict *)_dict; + struct ioloop *old_ioloop = current_ioloop; + const char *error; + + if (array_count(&dict->input_states) > 0) { + if (memcached_ascii_wait(dict, &error) < 0) + i_error("%s", error); + } + connection_deinit(&dict->conn.conn); + + io_loop_set_current(dict->dict.ioloop); + io_loop_destroy(&dict->dict.ioloop); + io_loop_set_current(old_ioloop); + + str_free(&dict->conn.reply_str); + array_free(&dict->replies); + array_free(&dict->input_states); + i_free(dict->key_prefix); + i_free(dict); + + if (memcached_ascii_connections->connections == NULL) + connection_list_deinit(&memcached_ascii_connections); +} + +static int memcached_ascii_connect(struct memcached_ascii_dict *dict, + const char **error_r) +{ + if (dict->conn.conn.input != NULL) + return 0; + + if (dict->conn.conn.fd_in == -1) { + if (connection_client_connect(&dict->conn.conn) < 0) { + *error_r = t_strdup_printf( + "memcached_ascii: Couldn't connect to %s:%u", + net_ip2addr(&dict->ip), dict->port); + return -1; + } + } + return memcached_ascii_wait(dict, error_r); +} + +static const char * +memcached_ascii_dict_get_full_key(struct memcached_ascii_dict *dict, + const char *username, const char *key) +{ + if (str_begins(key, DICT_PATH_SHARED)) + key += strlen(DICT_PATH_SHARED); + else if (str_begins(key, DICT_PATH_PRIVATE)) { + if (strchr(username, DICT_USERNAME_SEPARATOR) == NULL) { + key = t_strdup_printf("%s%c%s", username, + DICT_USERNAME_SEPARATOR, + key + strlen(DICT_PATH_PRIVATE)); + } else { + /* escape the username */ + key = t_strdup_printf("%s%c%s", memcached_ascii_escape_username(username), + DICT_USERNAME_SEPARATOR, + key + strlen(DICT_PATH_PRIVATE)); + } + } else { + i_unreached(); + } + if (*dict->key_prefix != '\0') + key = t_strconcat(dict->key_prefix, key, NULL); + return key; +} + +static int +memcached_ascii_dict_lookup(struct dict *_dict, + const struct dict_op_settings *set, + pool_t pool, const char *key, const char **value_r, + const char **error_r) +{ + struct memcached_ascii_dict *dict = (struct memcached_ascii_dict *)_dict; + struct memcached_ascii_dict_reply *reply; + enum memcached_ascii_input_state state = MEMCACHED_INPUT_STATE_GET; + + if (memcached_ascii_connect(dict, error_r) < 0) + return -1; + + key = memcached_ascii_dict_get_full_key(dict, set->username, key); + o_stream_nsend_str(dict->conn.conn.output, + t_strdup_printf("get %s\r\n", key)); + array_push_back(&dict->input_states, &state); + + reply = array_append_space(&dict->replies); + reply->reply_count = 1; + + if (memcached_ascii_wait(dict, error_r) < 0) + return -1; + + *value_r = p_strdup(pool, str_c(dict->conn.reply_str)); + return dict->conn.value_received ? 1 : 0; +} + +static struct dict_transaction_context * +memcached_ascii_transaction_init(struct dict *_dict) +{ + struct dict_transaction_memory_context *ctx; + pool_t pool; + + pool = pool_alloconly_create("file dict transaction", 2048); + ctx = p_new(pool, struct dict_transaction_memory_context, 1); + dict_transaction_memory_init(ctx, _dict, pool); + return &ctx->ctx; +} + +static void +memcached_send_change(struct dict_memcached_ascii_commit_ctx *ctx, + const struct dict_op_settings_private *set, + const struct dict_transaction_memory_change *change) +{ + enum memcached_ascii_input_state state; + const char *key, *value; + + key = memcached_ascii_dict_get_full_key(ctx->dict, set->username, + change->key); + + str_truncate(ctx->str, 0); + switch (change->type) { + case DICT_CHANGE_TYPE_SET: + state = MEMCACHED_INPUT_STATE_STORED; + str_printfa(ctx->str, "set %s 0 0 %zu\r\n%s\r\n", + key, strlen(change->value.str), change->value.str); + break; + case DICT_CHANGE_TYPE_UNSET: + state = MEMCACHED_INPUT_STATE_DELETED; + str_printfa(ctx->str, "delete %s\r\n", key); + break; + case DICT_CHANGE_TYPE_INC: + state = MEMCACHED_INPUT_STATE_INCRDECR; + if (change->value.diff > 0) { + str_printfa(ctx->str, "incr %s %lld\r\n", + key, change->value.diff); + array_push_back(&ctx->dict->input_states, &state); + /* same kludge as with append */ + value = t_strdup_printf("%lld", change->value.diff); + str_printfa(ctx->str, "add %s 0 0 %u\r\n%s\r\n", + key, (unsigned int)strlen(value), value); + } else { + str_printfa(ctx->str, "decr %s %lld\r\n", + key, -change->value.diff); + } + break; + } + array_push_back(&ctx->dict->input_states, &state); + o_stream_nsend(ctx->dict->conn.conn.output, + str_data(ctx->str), str_len(ctx->str)); +} + +static int +memcached_ascii_transaction_send(struct dict_memcached_ascii_commit_ctx *ctx, + const struct dict_op_settings_private *set, + const char **error_r) +{ + struct memcached_ascii_dict *dict = ctx->dict; + struct memcached_ascii_dict_reply *reply; + const struct dict_transaction_memory_change *changes; + unsigned int i, count, old_state_count; + + if (memcached_ascii_connect(dict, error_r) < 0) + return -1; + + old_state_count = array_count(&dict->input_states); + changes = array_get(&ctx->memctx->changes, &count); + i_assert(count > 0); + + o_stream_cork(dict->conn.conn.output); + for (i = 0; i < count; i++) T_BEGIN { + memcached_send_change(ctx, set, &changes[i]); + } T_END; + o_stream_uncork(dict->conn.conn.output); + + reply = array_append_space(&dict->replies); + reply->callback = ctx->callback; + reply->context = ctx->context; + reply->reply_count = array_count(&dict->input_states) - old_state_count; + return 0; +} + +static void +memcached_ascii_transaction_commit(struct dict_transaction_context *_ctx, + bool async, + dict_transaction_commit_callback_t *callback, + void *context) +{ + struct dict_transaction_memory_context *ctx = + (struct dict_transaction_memory_context *)_ctx; + struct memcached_ascii_dict *dict = + (struct memcached_ascii_dict *)_ctx->dict; + struct dict_memcached_ascii_commit_ctx commit_ctx; + struct dict_commit_result result = { DICT_COMMIT_RET_OK, NULL }; + const struct dict_op_settings_private *set = &_ctx->set; + + if (_ctx->changed) { + i_zero(&commit_ctx); + commit_ctx.dict = dict; + commit_ctx.memctx = ctx; + commit_ctx.callback = callback; + commit_ctx.context = context; + commit_ctx.str = str_new(default_pool, 128); + + result.ret = memcached_ascii_transaction_send(&commit_ctx, set, &result.error); + str_free(&commit_ctx.str); + + if (async && result.ret == 0) { + pool_unref(&ctx->pool); + return; + } + + if (result.ret == 0) { + if (memcached_ascii_wait(dict, &result.error) < 0) + result.ret = -1; + } + } + callback(&result, context); + pool_unref(&ctx->pool); +} + +struct dict dict_driver_memcached_ascii = { + .name = "memcached_ascii", + { + .init = memcached_ascii_dict_init, + .deinit = memcached_ascii_dict_deinit, + .lookup = memcached_ascii_dict_lookup, + .transaction_init = memcached_ascii_transaction_init, + .transaction_commit = memcached_ascii_transaction_commit, + .transaction_rollback = dict_transaction_memory_rollback, + .set = dict_transaction_memory_set, + .unset = dict_transaction_memory_unset, + .atomic_inc = dict_transaction_memory_atomic_inc, + } +}; diff --git a/src/lib-dict/dict-memcached.c b/src/lib-dict/dict-memcached.c new file mode 100644 index 0000000..c5b7ce6 --- /dev/null +++ b/src/lib-dict/dict-memcached.c @@ -0,0 +1,373 @@ +/* Copyright (c) 2013-2018 Dovecot authors, see the included COPYING memcached */ + +#include "lib.h" +#include "array.h" +#include "str.h" +#include "istream.h" +#include "ostream.h" +#include "connection.h" +#include "dict-private.h" + +#define MEMCACHED_DEFAULT_PORT 11211 +#define MEMCACHED_DEFAULT_LOOKUP_TIMEOUT_MSECS (1000*30) + +/* we need only very limited memcached functionality, so just define the binary + protocol ourself instead requiring protocol_binary.h */ +#define MEMCACHED_REQUEST_HDR_MAGIC 0x80 +#define MEMCACHED_REPLY_HDR_MAGIC 0x81 + +#define MEMCACHED_REQUEST_HDR_LENGTH 24 +#define MEMCACHED_REPLY_HDR_LENGTH 24 + +#define MEMCACHED_CMD_GET 0x00 + +#define MEMCACHED_DATA_TYPE_RAW 0x00 + +enum memcached_response { + MEMCACHED_RESPONSE_OK = 0x0000, + MEMCACHED_RESPONSE_NOTFOUND = 0x0001, + MEMCACHED_RESPONSE_INTERNALERROR= 0x0084, + MEMCACHED_RESPONSE_BUSY = 0x0085, + MEMCACHED_RESPONSE_TEMPFAILURE = 0x0086, +}; + +struct memcached_connection { + struct connection conn; + struct memcached_dict *dict; + + buffer_t *cmd; + struct { + const unsigned char *value; + size_t value_len; + uint16_t status; /* enum memcached_response */ + bool reply_received; + } reply; +}; + +struct memcached_dict { + struct dict dict; + struct ip_addr ip; + char *key_prefix; + in_port_t port; + unsigned int timeout_msecs; + + struct memcached_connection conn; + + bool connected; +}; + +static struct connection_list *memcached_connections; + +static void memcached_conn_destroy(struct connection *_conn) +{ + struct memcached_connection *conn = (struct memcached_connection *)_conn; + + conn->dict->connected = FALSE; + connection_disconnect(_conn); + + if (conn->dict->dict.ioloop != NULL) + io_loop_stop(conn->dict->dict.ioloop); +} + +static int memcached_input_get(struct memcached_connection *conn) +{ + const unsigned char *data; + size_t size; + uint32_t body_len, value_pos; + uint16_t key_len, key_pos, status; + uint8_t extras_len, data_type; + + data = i_stream_get_data(conn->conn.input, &size); + if (size < MEMCACHED_REPLY_HDR_LENGTH) + return 0; + + if (data[0] != MEMCACHED_REPLY_HDR_MAGIC) { + e_error(conn->conn.event, "Invalid reply magic: %u != %u", + data[0], MEMCACHED_REPLY_HDR_MAGIC); + return -1; + } + memcpy(&body_len, data+8, 4); body_len = ntohl(body_len); + body_len += MEMCACHED_REPLY_HDR_LENGTH; + if (size < body_len) { + /* we haven't read the whole response yet */ + return 0; + } + + memcpy(&key_len, data+2, 2); key_len = ntohs(key_len); + extras_len = data[4]; + data_type = data[5]; + memcpy(&status, data+6, 2); status = ntohs(status); + if (data_type != MEMCACHED_DATA_TYPE_RAW) { + e_error(conn->conn.event, "Unsupported data type: %u != %u", + data[0], MEMCACHED_DATA_TYPE_RAW); + return -1; + } + + key_pos = MEMCACHED_REPLY_HDR_LENGTH + extras_len; + value_pos = key_pos + key_len; + if (value_pos > body_len) { + e_error(conn->conn.event, "Invalid key/extras lengths"); + return -1; + } + conn->reply.value = data + value_pos; + conn->reply.value_len = body_len - value_pos; + conn->reply.status = status; + + i_stream_skip(conn->conn.input, body_len); + conn->reply.reply_received = TRUE; + + if (conn->dict->dict.ioloop != NULL) + io_loop_stop(conn->dict->dict.ioloop); + return 1; +} + +static void memcached_conn_input(struct connection *_conn) +{ + struct memcached_connection *conn = (struct memcached_connection *)_conn; + + switch (i_stream_read(_conn->input)) { + case 0: + return; + case -1: + memcached_conn_destroy(_conn); + return; + default: + break; + } + + if (memcached_input_get(conn) < 0) + memcached_conn_destroy(_conn); +} + +static void memcached_conn_connected(struct connection *_conn, bool success) +{ + struct memcached_connection *conn = + (struct memcached_connection *)_conn; + + if (!success) { + e_error(conn->conn.event, "connect() failed: %m"); + } else { + conn->dict->connected = TRUE; + } + if (conn->dict->dict.ioloop != NULL) + io_loop_stop(conn->dict->dict.ioloop); +} + +static const struct connection_settings memcached_conn_set = { + .input_max_size = SIZE_MAX, + .output_max_size = SIZE_MAX, + .client = TRUE +}; + +static const struct connection_vfuncs memcached_conn_vfuncs = { + .destroy = memcached_conn_destroy, + .input = memcached_conn_input, + .client_connected = memcached_conn_connected +}; + +static int +memcached_dict_init(struct dict *driver, const char *uri, + const struct dict_settings *set, + struct dict **dict_r, const char **error_r) +{ + struct memcached_dict *dict; + const char *const *args; + int ret = 0; + + if (memcached_connections == NULL) { + memcached_connections = + connection_list_init(&memcached_conn_set, + &memcached_conn_vfuncs); + } + + dict = i_new(struct memcached_dict, 1); + if (net_addr2ip("127.0.0.1", &dict->ip) < 0) + i_unreached(); + dict->port = MEMCACHED_DEFAULT_PORT; + dict->timeout_msecs = MEMCACHED_DEFAULT_LOOKUP_TIMEOUT_MSECS; + dict->key_prefix = i_strdup(""); + + args = t_strsplit(uri, ":"); + for (; *args != NULL; args++) { + if (str_begins(*args, "host=")) { + if (net_addr2ip(*args+5, &dict->ip) < 0) { + *error_r = t_strdup_printf("Invalid IP: %s", + *args+5); + ret = -1; + } + } else if (str_begins(*args, "port=")) { + if (net_str2port(*args+5, &dict->port) < 0) { + *error_r = t_strdup_printf("Invalid port: %s", + *args+5); + ret = -1; + } + } else if (str_begins(*args, "prefix=")) { + i_free(dict->key_prefix); + dict->key_prefix = i_strdup(*args + 7); + } else if (str_begins(*args, "timeout_msecs=")) { + if (str_to_uint(*args+14, &dict->timeout_msecs) < 0) { + *error_r = t_strdup_printf( + "Invalid timeout_msecs: %s", *args+14); + ret = -1; + } + } else { + *error_r = t_strdup_printf("Unknown parameter: %s", + *args); + ret = -1; + } + } + if (ret < 0) { + i_free(dict->key_prefix); + i_free(dict); + return -1; + } + + dict->conn.conn.event_parent = set->event_parent; + + connection_init_client_ip(memcached_connections, &dict->conn.conn, + NULL, &dict->ip, dict->port); + event_set_append_log_prefix(dict->conn.conn.event, "memcached: "); + dict->dict = *driver; + dict->conn.cmd = buffer_create_dynamic(default_pool, 256); + dict->conn.dict = dict; + *dict_r = &dict->dict; + return 0; +} + +static void memcached_dict_deinit(struct dict *_dict) +{ + struct memcached_dict *dict = (struct memcached_dict *)_dict; + + connection_deinit(&dict->conn.conn); + buffer_free(&dict->conn.cmd); + i_free(dict->key_prefix); + i_free(dict); + + if (memcached_connections->connections == NULL) + connection_list_deinit(&memcached_connections); +} + +static void memcached_dict_lookup_timeout(struct memcached_dict *dict) +{ + e_error(dict->dict.event, "Lookup timed out in %u.%03u secs", + dict->timeout_msecs/1000, dict->timeout_msecs%1000); + io_loop_stop(dict->dict.ioloop); +} + +static void memcached_add_header(buffer_t *buf, unsigned int key_len) +{ + uint32_t body_len = htonl(key_len); + + i_assert(key_len <= 0xffff); + + buffer_append_c(buf, MEMCACHED_REQUEST_HDR_MAGIC); + buffer_append_c(buf, MEMCACHED_CMD_GET); + buffer_append_c(buf, (key_len >> 8) & 0xff); + buffer_append_c(buf, key_len & 0xff); + buffer_append_c(buf, 0); /* extras length */ + buffer_append_c(buf, MEMCACHED_DATA_TYPE_RAW); + buffer_append_zero(buf, 2); /* vbucket id - we probably don't care? */ + buffer_append(buf, &body_len, sizeof(body_len)); + buffer_append_zero(buf, 4+8); /* opaque + cas */ + i_assert(buf->used == MEMCACHED_REQUEST_HDR_LENGTH); +} + +static int +memcached_dict_lookup(struct dict *_dict, const struct dict_op_settings *set ATTR_UNUSED, + pool_t pool, const char *key, const char **value_r, + const char **error_r) +{ + struct memcached_dict *dict = (struct memcached_dict *)_dict; + struct ioloop *prev_ioloop = current_ioloop; + struct timeout *to; + size_t key_len; + + if (str_begins(key, DICT_PATH_SHARED)) + key += strlen(DICT_PATH_SHARED); + else { + *error_r = t_strdup_printf("memcached: Only shared keys supported currently"); + return -1; + } + if (*dict->key_prefix != '\0') + key = t_strconcat(dict->key_prefix, key, NULL); + key_len = strlen(key); + if (key_len > 0xffff) { + *error_r = t_strdup_printf( + "memcached: Key is too long (%zu bytes): %s", key_len, key); + return -1; + } + + i_assert(dict->dict.ioloop == NULL); + + dict->dict.ioloop = io_loop_create(); + connection_switch_ioloop(&dict->conn.conn); + + if (dict->conn.conn.fd_in == -1 && + connection_client_connect(&dict->conn.conn) < 0) { + e_error(dict->conn.conn.event, "Couldn't connect"); + } else { + to = timeout_add(dict->timeout_msecs, + memcached_dict_lookup_timeout, dict); + if (!dict->connected) { + /* wait for connection */ + io_loop_run(dict->dict.ioloop); + } + + if (dict->connected) { + buffer_set_used_size(dict->conn.cmd, 0); + memcached_add_header(dict->conn.cmd, key_len); + buffer_append(dict->conn.cmd, key, key_len); + + o_stream_nsend(dict->conn.conn.output, + dict->conn.cmd->data, + dict->conn.cmd->used); + + i_zero(&dict->conn.reply); + io_loop_run(dict->dict.ioloop); + } + timeout_remove(&to); + } + + io_loop_set_current(prev_ioloop); + connection_switch_ioloop(&dict->conn.conn); + io_loop_set_current(dict->dict.ioloop); + io_loop_destroy(&dict->dict.ioloop); + + if (!dict->conn.reply.reply_received) { + /* we failed in some way. make sure we disconnect since the + connection state isn't known anymore */ + memcached_conn_destroy(&dict->conn.conn); + *error_r = "Communication failure"; + return -1; + } + switch (dict->conn.reply.status) { + case MEMCACHED_RESPONSE_OK: + *value_r = p_strndup(pool, dict->conn.reply.value, + dict->conn.reply.value_len); + return 1; + case MEMCACHED_RESPONSE_NOTFOUND: + return 0; + case MEMCACHED_RESPONSE_INTERNALERROR: + *error_r = "Lookup failed: Internal error"; + return -1; + case MEMCACHED_RESPONSE_BUSY: + *error_r = "Lookup failed: Busy"; + return -1; + case MEMCACHED_RESPONSE_TEMPFAILURE: + *error_r = "Lookup failed: Temporary failure"; + return -1; + } + + *error_r = t_strdup_printf("Lookup failed: Error code=%u", + dict->conn.reply.status); + return -1; +} + +struct dict dict_driver_memcached = { + .name = "memcached", + { + .init = memcached_dict_init, + .deinit = memcached_dict_deinit, + .lookup = memcached_dict_lookup, + } +}; diff --git a/src/lib-dict/dict-private.h b/src/lib-dict/dict-private.h new file mode 100644 index 0000000..e5ec5e3 --- /dev/null +++ b/src/lib-dict/dict-private.h @@ -0,0 +1,123 @@ +#ifndef DICT_PRIVATE_H +#define DICT_PRIVATE_H + +#include <time.h> +#include "dict.h" + +struct ioloop; + +struct dict_vfuncs { + int (*init)(struct dict *dict_driver, const char *uri, + const struct dict_settings *set, + struct dict **dict_r, const char **error_r); + void (*deinit)(struct dict *dict); + void (*wait)(struct dict *dict); + + int (*lookup)(struct dict *dict, const struct dict_op_settings *set, + pool_t pool, const char *key, const char **value_r, + const char **error_r); + + struct dict_iterate_context * + (*iterate_init)(struct dict *dict, + const struct dict_op_settings *set, + const char *path, + enum dict_iterate_flags flags); + bool (*iterate)(struct dict_iterate_context *ctx, + const char **key_r, const char *const **values_r); + int (*iterate_deinit)(struct dict_iterate_context *ctx, + const char **error_r); + + struct dict_transaction_context *(*transaction_init)(struct dict *dict); + /* call the callback before returning if non-async commits */ + void (*transaction_commit)(struct dict_transaction_context *ctx, + bool async, + dict_transaction_commit_callback_t *callback, + void *context); + void (*transaction_rollback)(struct dict_transaction_context *ctx); + + void (*set)(struct dict_transaction_context *ctx, + const char *key, const char *value); + void (*unset)(struct dict_transaction_context *ctx, + const char *key); + void (*atomic_inc)(struct dict_transaction_context *ctx, + const char *key, long long diff); + + void (*lookup_async)(struct dict *dict, const struct dict_op_settings *set, + const char *key, dict_lookup_callback_t *callback, + void *context); + bool (*switch_ioloop)(struct dict *dict); + void (*set_timestamp)(struct dict_transaction_context *ctx, + const struct timespec *ts); +}; + +struct dict_commit_callback_ctx; + +struct dict_op_settings_private { + char *username; + char *home_dir; +}; + +struct dict { + const char *name; + + struct dict_vfuncs v; + unsigned int iter_count; + unsigned int transaction_count; + struct dict_transaction_context *transactions; + int refcount; + struct event *event; + struct ioloop *ioloop, *prev_ioloop; + struct dict_commit_callback_ctx *commits; +}; + +struct dict_iterate_context { + struct dict *dict; + struct event *event; + struct dict_op_settings_private set; + enum dict_iterate_flags flags; + + dict_iterate_callback_t *async_callback; + void *async_context; + + uint64_t row_count, max_rows; + + bool has_more:1; +}; + +struct dict_transaction_context { + struct dict *dict; + struct dict_op_settings_private set; + struct dict_transaction_context *prev, *next; + + struct event *event; + struct timespec timestamp; + + bool changed:1; + bool no_slowness_warning:1; +}; + +void dict_transaction_commit_async_noop_callback( + const struct dict_commit_result *result, void *context); + +extern struct dict dict_driver_client; +extern struct dict dict_driver_file; +extern struct dict dict_driver_fs; +extern struct dict dict_driver_memcached; +extern struct dict dict_driver_memcached_ascii; +extern struct dict dict_driver_redis; +extern struct dict dict_driver_cdb; +extern struct dict dict_driver_fail; + +extern struct dict_iterate_context dict_iter_unsupported; +extern struct dict_transaction_context dict_transaction_unsupported; + +void dict_pre_api_callback(struct dict *dict); +void dict_post_api_callback(struct dict *dict); + +/* Duplicate an object of type dict_op_settings. Used for initializing/freeing + iterator and transaction contexts. */ +void dict_op_settings_dup(const struct dict_op_settings *source, + struct dict_op_settings_private *dest_r); +void dict_op_settings_private_free(struct dict_op_settings_private *set); + +#endif diff --git a/src/lib-dict/dict-redis.c b/src/lib-dict/dict-redis.c new file mode 100644 index 0000000..01ec7b0 --- /dev/null +++ b/src/lib-dict/dict-redis.c @@ -0,0 +1,831 @@ +/* Copyright (c) 2008-2018 Dovecot authors, see the included COPYING redis */ + +#include "lib.h" +#include "array.h" +#include "str.h" +#include "istream.h" +#include "ostream.h" +#include "connection.h" +#include "dict-private.h" + +#define REDIS_DEFAULT_PORT 6379 +#define REDIS_DEFAULT_LOOKUP_TIMEOUT_MSECS (1000*30) +#define DICT_USERNAME_SEPARATOR '/' + +enum redis_input_state { + /* expecting +OK reply for AUTH */ + REDIS_INPUT_STATE_AUTH, + /* expecting +OK reply for SELECT */ + REDIS_INPUT_STATE_SELECT, + /* expecting $-1 / $<size> followed by GET reply */ + REDIS_INPUT_STATE_GET, + /* expecting +QUEUED */ + REDIS_INPUT_STATE_MULTI, + /* expecting +OK reply for DISCARD */ + REDIS_INPUT_STATE_DISCARD, + /* expecting *<nreplies> */ + REDIS_INPUT_STATE_EXEC, + /* expecting EXEC reply */ + REDIS_INPUT_STATE_EXEC_REPLY +}; + +struct redis_connection { + struct connection conn; + struct redis_dict *dict; + + string_t *last_reply; + unsigned int bytes_left; + bool value_not_found; + bool value_received; +}; + +struct redis_dict_reply { + unsigned int reply_count; + dict_transaction_commit_callback_t *callback; + void *context; +}; + +struct redis_dict { + struct dict dict; + char *password, *key_prefix, *expire_value; + unsigned int timeout_msecs, db_id; + + struct redis_connection conn; + + ARRAY(enum redis_input_state) input_states; + ARRAY(struct redis_dict_reply) replies; + + bool connected; + bool transaction_open; + bool db_id_set; +}; + +struct redis_dict_transaction_context { + struct dict_transaction_context ctx; + unsigned int cmd_count; + char *error; +}; + +static struct connection_list *redis_connections; + +static void +redis_input_state_add(struct redis_dict *dict, enum redis_input_state state) +{ + array_push_back(&dict->input_states, &state); +} + +static void redis_input_state_remove(struct redis_dict *dict) +{ + array_pop_front(&dict->input_states); +} + +static void redis_reply_callback(struct redis_connection *conn, + const struct redis_dict_reply *reply, + const struct dict_commit_result *result) +{ + if (conn->dict->dict.prev_ioloop != NULL) + io_loop_set_current(conn->dict->dict.prev_ioloop); + reply->callback(result, reply->context); + if (conn->dict->dict.prev_ioloop != NULL) + io_loop_set_current(conn->dict->dict.ioloop); +} + +static void +redis_disconnected(struct redis_connection *conn, const char *reason) +{ + const struct dict_commit_result result = { + DICT_COMMIT_RET_FAILED, reason + }; + const struct redis_dict_reply *reply; + + conn->dict->db_id_set = FALSE; + conn->dict->connected = FALSE; + connection_disconnect(&conn->conn); + + array_foreach(&conn->dict->replies, reply) + redis_reply_callback(conn, reply, &result); + array_clear(&conn->dict->replies); + array_clear(&conn->dict->input_states); + + if (conn->dict->dict.ioloop != NULL) + io_loop_stop(conn->dict->dict.ioloop); +} + +static void redis_conn_destroy(struct connection *_conn) +{ + struct redis_connection *conn = (struct redis_connection *)_conn; + + redis_disconnected(conn, connection_disconnect_reason(_conn)); +} + +static void redis_dict_wait_timeout(struct redis_dict *dict) +{ + const char *reason = t_strdup_printf( + "redis: Commit timed out in %u.%03u secs", + dict->timeout_msecs/1000, dict->timeout_msecs%1000); + redis_disconnected(&dict->conn, reason); +} + +static void redis_wait(struct redis_dict *dict) +{ + struct timeout *to; + + i_assert(dict->dict.ioloop == NULL); + + dict->dict.prev_ioloop = current_ioloop; + dict->dict.ioloop = io_loop_create(); + to = timeout_add(dict->timeout_msecs, redis_dict_wait_timeout, dict); + connection_switch_ioloop(&dict->conn.conn); + + do { + io_loop_run(dict->dict.ioloop); + } while (array_count(&dict->input_states) > 0); + + timeout_remove(&to); + io_loop_set_current(dict->dict.prev_ioloop); + connection_switch_ioloop(&dict->conn.conn); + io_loop_set_current(dict->dict.ioloop); + io_loop_destroy(&dict->dict.ioloop); + dict->dict.prev_ioloop = NULL; +} + +static int redis_input_get(struct redis_connection *conn, const char **error_r) +{ + const unsigned char *data; + size_t size; + const char *line; + + if (conn->bytes_left == 0) { + /* read the size first */ + line = i_stream_next_line(conn->conn.input); + if (line == NULL) + return 0; + if (strcmp(line, "$-1") == 0) { + conn->value_received = TRUE; + conn->value_not_found = TRUE; + if (conn->dict->dict.ioloop != NULL) + io_loop_stop(conn->dict->dict.ioloop); + redis_input_state_remove(conn->dict); + return 1; + } + if (line[0] != '$' || str_to_uint(line+1, &conn->bytes_left) < 0) { + *error_r = t_strdup_printf( + "redis: Unexpected input (wanted $size): %s", line); + return -1; + } + conn->bytes_left += 2; /* include trailing CRLF */ + } + + data = i_stream_get_data(conn->conn.input, &size); + if (size > conn->bytes_left) + size = conn->bytes_left; + str_append_data(conn->last_reply, data, size); + + conn->bytes_left -= size; + i_stream_skip(conn->conn.input, size); + + if (conn->bytes_left > 0) + return 0; + + /* reply fully read - drop trailing CRLF */ + conn->value_received = TRUE; + str_truncate(conn->last_reply, str_len(conn->last_reply)-2); + + if (conn->dict->dict.ioloop != NULL) + io_loop_stop(conn->dict->dict.ioloop); + redis_input_state_remove(conn->dict); + return 1; +} + +static int +redis_conn_input_more(struct redis_connection *conn, const char **error_r) +{ + struct redis_dict *dict = conn->dict; + struct redis_dict_reply *reply; + const enum redis_input_state *states; + enum redis_input_state state; + unsigned int count, num_replies; + const char *line; + + states = array_get(&dict->input_states, &count); + if (count == 0) { + line = i_stream_next_line(conn->conn.input); + if (line == NULL) + return 0; + *error_r = t_strdup_printf( + "redis: Unexpected input (expected nothing): %s", line); + return -1; + } + state = states[0]; + if (state == REDIS_INPUT_STATE_GET) + return redis_input_get(conn, error_r); + + line = i_stream_next_line(conn->conn.input); + if (line == NULL) + return 0; + + redis_input_state_remove(dict); + switch (state) { + case REDIS_INPUT_STATE_GET: + i_unreached(); + case REDIS_INPUT_STATE_AUTH: + case REDIS_INPUT_STATE_SELECT: + case REDIS_INPUT_STATE_MULTI: + case REDIS_INPUT_STATE_DISCARD: + if (line[0] != '+') + break; + return 1; + case REDIS_INPUT_STATE_EXEC: + if (line[0] != '*' || str_to_uint(line+1, &num_replies) < 0) + break; + + reply = array_front_modifiable(&dict->replies); + i_assert(reply->reply_count > 0); + if (reply->reply_count != num_replies) { + *error_r = t_strdup_printf( + "redis: EXEC expected %u replies, not %u", + reply->reply_count, num_replies); + return -1; + } + return 1; + case REDIS_INPUT_STATE_EXEC_REPLY: + if (*line != '+' && *line != ':') + break; + /* success, just ignore the actual reply */ + reply = array_front_modifiable(&dict->replies); + i_assert(reply->reply_count > 0); + if (--reply->reply_count == 0) { + const struct dict_commit_result result = { + DICT_COMMIT_RET_OK, NULL + }; + redis_reply_callback(conn, reply, &result); + array_pop_front(&dict->replies); + /* if we're running in a dict-ioloop, we're handling a + synchronous commit and need to stop now */ + if (array_count(&dict->replies) == 0 && + conn->dict->dict.ioloop != NULL) + io_loop_stop(conn->dict->dict.ioloop); + } + return 1; + } + str_truncate(dict->conn.last_reply, 0); + str_append(dict->conn.last_reply, line); + *error_r = t_strdup_printf("redis: Unexpected input (state=%d): %s", state, line); + return -1; +} + +static void redis_conn_input(struct connection *_conn) +{ + struct redis_connection *conn = (struct redis_connection *)_conn; + const char *error = NULL; + int ret; + + switch (i_stream_read(_conn->input)) { + case 0: + return; + case -1: + redis_disconnected(conn, i_stream_get_error(_conn->input)); + return; + default: + break; + } + + while ((ret = redis_conn_input_more(conn, &error)) > 0) ; + if (ret < 0) { + i_assert(error != NULL); + redis_disconnected(conn, error); + } +} + +static void redis_conn_connected(struct connection *_conn, bool success) +{ + struct redis_connection *conn = (struct redis_connection *)_conn; + + if (!success) { + e_error(conn->conn.event, "connect() failed: %m"); + } else { + conn->dict->connected = TRUE; + } + if (conn->dict->dict.ioloop != NULL) + io_loop_stop(conn->dict->dict.ioloop); +} + +static const struct connection_settings redis_conn_set = { + .input_max_size = SIZE_MAX, + .output_max_size = SIZE_MAX, + .client = TRUE +}; + +static const struct connection_vfuncs redis_conn_vfuncs = { + .destroy = redis_conn_destroy, + .input = redis_conn_input, + .client_connected = redis_conn_connected +}; + +static const char *redis_escape_username(const char *username) +{ + const char *p; + string_t *str = t_str_new(64); + + for (p = username; *p != '\0'; p++) { + switch (*p) { + case DICT_USERNAME_SEPARATOR: + str_append(str, "\\-"); + break; + case '\\': + str_append(str, "\\\\"); + break; + default: + str_append_c(str, *p); + } + } + return str_c(str); +} + +static int +redis_dict_init(struct dict *driver, const char *uri, + const struct dict_settings *set, + struct dict **dict_r, const char **error_r) +{ + struct redis_dict *dict; + struct ip_addr ip; + unsigned int secs; + in_port_t port = REDIS_DEFAULT_PORT; + const char *const *args, *unix_path = NULL; + int ret = 0; + + if (redis_connections == NULL) { + redis_connections = + connection_list_init(&redis_conn_set, + &redis_conn_vfuncs); + } + + dict = i_new(struct redis_dict, 1); + if (net_addr2ip("127.0.0.1", &ip) < 0) + i_unreached(); + dict->timeout_msecs = REDIS_DEFAULT_LOOKUP_TIMEOUT_MSECS; + dict->key_prefix = i_strdup(""); + dict->password = i_strdup(""); + + args = t_strsplit(uri, ":"); + for (; *args != NULL; args++) { + if (str_begins(*args, "path=")) { + unix_path = *args + 5; + } else if (str_begins(*args, "host=")) { + if (net_addr2ip(*args+5, &ip) < 0) { + *error_r = t_strdup_printf("Invalid IP: %s", + *args+5); + ret = -1; + } + } else if (str_begins(*args, "port=")) { + if (net_str2port(*args+5, &port) < 0) { + *error_r = t_strdup_printf("Invalid port: %s", + *args+5); + ret = -1; + } + } else if (str_begins(*args, "prefix=")) { + i_free(dict->key_prefix); + dict->key_prefix = i_strdup(*args + 7); + } else if (str_begins(*args, "db=")) { + if (str_to_uint(*args+3, &dict->db_id) < 0) { + *error_r = t_strdup_printf( + "Invalid db number: %s", *args+3); + ret = -1; + } + } else if (str_begins(*args, "expire_secs=")) { + const char *value = *args + 12; + + if (str_to_uint(value, &secs) < 0 || secs == 0) { + *error_r = t_strdup_printf( + "Invalid expire_secs: %s", value); + ret = -1; + } + i_free(dict->expire_value); + dict->expire_value = i_strdup(value); + } else if (str_begins(*args, "timeout_msecs=")) { + if (str_to_uint(*args+14, &dict->timeout_msecs) < 0) { + *error_r = t_strdup_printf( + "Invalid timeout_msecs: %s", *args+14); + ret = -1; + } + } else if (str_begins(*args, "password=")) { + i_free(dict->password); + dict->password = i_strdup(*args + 9); + } else { + *error_r = t_strdup_printf("Unknown parameter: %s", + *args); + ret = -1; + } + } + if (ret < 0) { + i_free(dict->password); + i_free(dict->key_prefix); + i_free(dict); + return -1; + } + + dict->conn.conn.event_parent = set->event_parent; + + if (unix_path != NULL) { + connection_init_client_unix(redis_connections, &dict->conn.conn, + unix_path); + } else { + connection_init_client_ip(redis_connections, &dict->conn.conn, + NULL, &ip, port); + } + event_set_append_log_prefix(dict->conn.conn.event, "redis: "); + dict->dict = *driver; + dict->conn.last_reply = str_new(default_pool, 256); + dict->conn.dict = dict; + + i_array_init(&dict->input_states, 4); + i_array_init(&dict->replies, 4); + + *dict_r = &dict->dict; + return 0; +} + +static void redis_dict_deinit(struct dict *_dict) +{ + struct redis_dict *dict = (struct redis_dict *)_dict; + + if (array_count(&dict->input_states) > 0) { + i_assert(dict->connected); + redis_wait(dict); + } + connection_deinit(&dict->conn.conn); + str_free(&dict->conn.last_reply); + array_free(&dict->replies); + array_free(&dict->input_states); + i_free(dict->expire_value); + i_free(dict->key_prefix); + i_free(dict->password); + i_free(dict); + + if (redis_connections->connections == NULL) + connection_list_deinit(&redis_connections); +} + +static void redis_dict_wait(struct dict *_dict) +{ + struct redis_dict *dict = (struct redis_dict *)_dict; + + if (array_count(&dict->input_states) > 0) + redis_wait(dict); +} + +static void redis_dict_lookup_timeout(struct redis_dict *dict) +{ + const char *reason = t_strdup_printf( + "redis: Lookup timed out in %u.%03u secs", + dict->timeout_msecs/1000, dict->timeout_msecs%1000); + redis_disconnected(&dict->conn, reason); +} + +static const char * +redis_dict_get_full_key(struct redis_dict *dict, const char *username, + const char *key) +{ + const char *username_sp = strchr(username, DICT_USERNAME_SEPARATOR); + + if (str_begins(key, DICT_PATH_SHARED)) + key += strlen(DICT_PATH_SHARED); + else if (str_begins(key, DICT_PATH_PRIVATE)) { + key = t_strdup_printf("%s%c%s", + username_sp == NULL ? username : + redis_escape_username(username), + DICT_USERNAME_SEPARATOR, + key + strlen(DICT_PATH_PRIVATE)); + } else { + i_unreached(); + } + if (*dict->key_prefix != '\0') + key = t_strconcat(dict->key_prefix, key, NULL); + return key; +} + +static void redis_dict_auth(struct redis_dict *dict) +{ + const char *cmd; + + if (*dict->password == '\0') + return; + + cmd = t_strdup_printf("*2\r\n$4\r\nAUTH\r\n$%d\r\n%s\r\n", + (int)strlen(dict->password), dict->password); + o_stream_nsend_str(dict->conn.conn.output, cmd); + redis_input_state_add(dict, REDIS_INPUT_STATE_AUTH); +} + +static void redis_dict_select_db(struct redis_dict *dict) +{ + const char *cmd, *db_str; + + if (dict->db_id_set) + return; + dict->db_id_set = TRUE; + if (dict->db_id == 0) { + /* 0 is the default */ + return; + } + db_str = dec2str(dict->db_id); + cmd = t_strdup_printf("*2\r\n$6\r\nSELECT\r\n$%d\r\n%s\r\n", + (int)strlen(db_str), db_str); + o_stream_nsend_str(dict->conn.conn.output, cmd); + redis_input_state_add(dict, REDIS_INPUT_STATE_SELECT); +} + +static int redis_dict_lookup(struct dict *_dict, + const struct dict_op_settings *set, + pool_t pool, const char *key, + const char **value_r, const char **error_r) +{ + struct redis_dict *dict = (struct redis_dict *)_dict; + struct timeout *to; + const char *cmd; + + key = redis_dict_get_full_key(dict, set->username, key); + + dict->conn.value_received = FALSE; + dict->conn.value_not_found = FALSE; + + i_assert(dict->dict.ioloop == NULL); + + dict->dict.prev_ioloop = current_ioloop; + dict->dict.ioloop = io_loop_create(); + connection_switch_ioloop(&dict->conn.conn); + + if (dict->conn.conn.fd_in == -1 && + connection_client_connect(&dict->conn.conn) < 0) { + e_error(dict->conn.conn.event, "Couldn't connect"); + } else { + to = timeout_add(dict->timeout_msecs, + redis_dict_lookup_timeout, dict); + if (!dict->connected) { + /* wait for connection */ + io_loop_run(dict->dict.ioloop); + if (dict->connected) + redis_dict_auth(dict); + } + + if (dict->connected) { + redis_dict_select_db(dict); + cmd = t_strdup_printf("*2\r\n$3\r\nGET\r\n$%d\r\n%s\r\n", + (int)strlen(key), key); + o_stream_nsend_str(dict->conn.conn.output, cmd); + + str_truncate(dict->conn.last_reply, 0); + redis_input_state_add(dict, REDIS_INPUT_STATE_GET); + do { + io_loop_run(dict->dict.ioloop); + } while (array_count(&dict->input_states) > 0); + } + timeout_remove(&to); + } + + io_loop_set_current(dict->dict.prev_ioloop); + connection_switch_ioloop(&dict->conn.conn); + io_loop_set_current(dict->dict.ioloop); + io_loop_destroy(&dict->dict.ioloop); + dict->dict.prev_ioloop = NULL; + + if (!dict->conn.value_received) { + /* we failed in some way. make sure we disconnect since the + connection state isn't known anymore */ + *error_r = t_strdup_printf("redis: Communication failure (last reply: %s)", + str_c(dict->conn.last_reply)); + redis_disconnected(&dict->conn, *error_r); + return -1; + } + if (dict->conn.value_not_found) + return 0; + + *value_r = p_strdup(pool, str_c(dict->conn.last_reply)); + return 1; +} + +static struct dict_transaction_context * +redis_transaction_init(struct dict *_dict) +{ + struct redis_dict *dict = (struct redis_dict *)_dict; + struct redis_dict_transaction_context *ctx; + + i_assert(!dict->transaction_open); + dict->transaction_open = TRUE; + + ctx = i_new(struct redis_dict_transaction_context, 1); + ctx->ctx.dict = _dict; + + if (dict->conn.conn.fd_in == -1 && + connection_client_connect(&dict->conn.conn) < 0) { + e_error(dict->conn.conn.event, "Couldn't connect"); + } else if (!dict->connected) { + /* wait for connection */ + redis_wait(dict); + if (dict->connected) + redis_dict_auth(dict); + } + if (dict->connected) + redis_dict_select_db(dict); + return &ctx->ctx; +} + +static void +redis_transaction_commit(struct dict_transaction_context *_ctx, bool async, + dict_transaction_commit_callback_t *callback, + void *context) +{ + struct redis_dict_transaction_context *ctx = + (struct redis_dict_transaction_context *)_ctx; + struct redis_dict *dict = (struct redis_dict *)_ctx->dict; + struct redis_dict_reply *reply; + unsigned int i; + struct dict_commit_result result = { .ret = DICT_COMMIT_RET_OK }; + + i_assert(dict->transaction_open); + dict->transaction_open = FALSE; + + if (ctx->error != NULL) { + /* make sure we're disconnected */ + redis_disconnected(&dict->conn, ctx->error); + result.ret = -1; + result.error = ctx->error; + } else if (_ctx->changed) { + i_assert(ctx->cmd_count > 0); + + o_stream_nsend_str(dict->conn.conn.output, + "*1\r\n$4\r\nEXEC\r\n"); + reply = array_append_space(&dict->replies); + reply->callback = callback; + reply->context = context; + reply->reply_count = ctx->cmd_count; + redis_input_state_add(dict, REDIS_INPUT_STATE_EXEC); + for (i = 0; i < ctx->cmd_count; i++) + redis_input_state_add(dict, REDIS_INPUT_STATE_EXEC_REPLY); + if (async) { + i_free(ctx); + return; + } + redis_wait(dict); + } + callback(&result, context); + i_free(ctx->error); + i_free(ctx); +} + +static void redis_transaction_rollback(struct dict_transaction_context *_ctx) +{ + struct redis_dict_transaction_context *ctx = + (struct redis_dict_transaction_context *)_ctx; + struct redis_dict *dict = (struct redis_dict *)_ctx->dict; + struct redis_dict_reply *reply; + + i_assert(dict->transaction_open); + dict->transaction_open = FALSE; + + if (ctx->error != NULL) { + /* make sure we're disconnected */ + redis_disconnected(&dict->conn, ctx->error); + } else if (_ctx->changed) { + o_stream_nsend_str(dict->conn.conn.output, + "*1\r\n$7\r\nDISCARD\r\n"); + reply = array_append_space(&dict->replies); + reply->reply_count = 1; + redis_input_state_add(dict, REDIS_INPUT_STATE_DISCARD); + } + i_free(ctx->error); + i_free(ctx); +} + +static int redis_check_transaction(struct redis_dict_transaction_context *ctx) +{ + struct redis_dict *dict = (struct redis_dict *)ctx->ctx.dict; + + if (ctx->error != NULL) + return -1; + if (!dict->connected) { + ctx->error = i_strdup("Disconnected during transaction"); + return -1; + } + if (ctx->ctx.changed) + return 0; + + redis_input_state_add(dict, REDIS_INPUT_STATE_MULTI); + if (o_stream_send_str(dict->conn.conn.output, + "*1\r\n$5\r\nMULTI\r\n") < 0) { + ctx->error = i_strdup_printf("write() failed: %s", + o_stream_get_error(dict->conn.conn.output)); + return -1; + } + return 0; +} + +static void +redis_append_expire(struct redis_dict_transaction_context *ctx, + string_t *cmd, const char *key) +{ + struct redis_dict *dict = (struct redis_dict *)ctx->ctx.dict; + + if (dict->expire_value == NULL) + return; + + str_printfa(cmd, "*3\r\n$6\r\nEXPIRE\r\n$%u\r\n%s\r\n$%u\r\n%s\r\n", + (unsigned int)strlen(key), key, + (unsigned int)strlen(dict->expire_value), + dict->expire_value); + redis_input_state_add(dict, REDIS_INPUT_STATE_MULTI); + ctx->cmd_count++; +} + +static void redis_set(struct dict_transaction_context *_ctx, + const char *key, const char *value) +{ + struct redis_dict_transaction_context *ctx = + (struct redis_dict_transaction_context *)_ctx; + struct redis_dict *dict = (struct redis_dict *)_ctx->dict; + const struct dict_op_settings_private *set = &_ctx->set; + string_t *cmd; + + if (redis_check_transaction(ctx) < 0) + return; + + key = redis_dict_get_full_key(dict, set->username, key); + cmd = t_str_new(128); + str_printfa(cmd, "*3\r\n$3\r\nSET\r\n$%u\r\n%s\r\n$%u\r\n%s\r\n", + (unsigned int)strlen(key), key, + (unsigned int)strlen(value), value); + redis_input_state_add(dict, REDIS_INPUT_STATE_MULTI); + ctx->cmd_count++; + redis_append_expire(ctx, cmd, key); + if (o_stream_send(dict->conn.conn.output, str_data(cmd), str_len(cmd)) < 0) { + ctx->error = i_strdup_printf("write() failed: %s", + o_stream_get_error(dict->conn.conn.output)); + } +} + +static void redis_unset(struct dict_transaction_context *_ctx, + const char *key) +{ + struct redis_dict_transaction_context *ctx = + (struct redis_dict_transaction_context *)_ctx; + struct redis_dict *dict = (struct redis_dict *)_ctx->dict; + const struct dict_op_settings_private *set = &_ctx->set; + const char *cmd; + + if (redis_check_transaction(ctx) < 0) + return; + + key = redis_dict_get_full_key(dict, set->username, key); + cmd = t_strdup_printf("*2\r\n$3\r\nDEL\r\n$%u\r\n%s\r\n", + (unsigned int)strlen(key), key); + if (o_stream_send_str(dict->conn.conn.output, cmd) < 0) { + ctx->error = i_strdup_printf("write() failed: %s", + o_stream_get_error(dict->conn.conn.output)); + } + redis_input_state_add(dict, REDIS_INPUT_STATE_MULTI); + ctx->cmd_count++; +} + +static void redis_atomic_inc(struct dict_transaction_context *_ctx, + const char *key, long long diff) +{ + struct redis_dict_transaction_context *ctx = + (struct redis_dict_transaction_context *)_ctx; + struct redis_dict *dict = (struct redis_dict *)_ctx->dict; + const struct dict_op_settings_private *set = &_ctx->set; + const char *diffstr; + string_t *cmd; + + if (redis_check_transaction(ctx) < 0) + return; + + key = redis_dict_get_full_key(dict, set->username, key); + diffstr = t_strdup_printf("%lld", diff); + cmd = t_str_new(128); + str_printfa(cmd, "*3\r\n$6\r\nINCRBY\r\n$%u\r\n%s\r\n$%u\r\n%s\r\n", + (unsigned int)strlen(key), key, + (unsigned int)strlen(diffstr), diffstr); + redis_input_state_add(dict, REDIS_INPUT_STATE_MULTI); + ctx->cmd_count++; + redis_append_expire(ctx, cmd, key); + if (o_stream_send(dict->conn.conn.output, str_data(cmd), str_len(cmd)) < 0) { + ctx->error = i_strdup_printf("write() failed: %s", + o_stream_get_error(dict->conn.conn.output)); + } +} + +struct dict dict_driver_redis = { + .name = "redis", + { + .init = redis_dict_init, + .deinit = redis_dict_deinit, + .wait = redis_dict_wait, + .lookup = redis_dict_lookup, + .transaction_init = redis_transaction_init, + .transaction_commit = redis_transaction_commit, + .transaction_rollback = redis_transaction_rollback, + .set = redis_set, + .unset = redis_unset, + .atomic_inc = redis_atomic_inc, + } +}; diff --git a/src/lib-dict/dict-transaction-memory.c b/src/lib-dict/dict-transaction-memory.c new file mode 100644 index 0000000..4793ad3 --- /dev/null +++ b/src/lib-dict/dict-transaction-memory.c @@ -0,0 +1,59 @@ +/* Copyright (c) 2005-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "dict-transaction-memory.h" + +void dict_transaction_memory_init(struct dict_transaction_memory_context *ctx, + struct dict *dict, pool_t pool) +{ + ctx->ctx.dict = dict; + ctx->pool = pool; + p_array_init(&ctx->changes, pool, 32); +} + +void dict_transaction_memory_rollback(struct dict_transaction_context *_ctx) +{ + struct dict_transaction_memory_context *ctx = + (struct dict_transaction_memory_context *)_ctx; + + pool_unref(&ctx->pool); +} + +void dict_transaction_memory_set(struct dict_transaction_context *_ctx, + const char *key, const char *value) +{ + struct dict_transaction_memory_context *ctx = + (struct dict_transaction_memory_context *)_ctx; + struct dict_transaction_memory_change *change; + + change = array_append_space(&ctx->changes); + change->type = DICT_CHANGE_TYPE_SET; + change->key = p_strdup(ctx->pool, key); + change->value.str = p_strdup(ctx->pool, value); +} + +void dict_transaction_memory_unset(struct dict_transaction_context *_ctx, + const char *key) +{ + struct dict_transaction_memory_context *ctx = + (struct dict_transaction_memory_context *)_ctx; + struct dict_transaction_memory_change *change; + + change = array_append_space(&ctx->changes); + change->type = DICT_CHANGE_TYPE_UNSET; + change->key = p_strdup(ctx->pool, key); +} + +void dict_transaction_memory_atomic_inc(struct dict_transaction_context *_ctx, + const char *key, long long diff) +{ + struct dict_transaction_memory_context *ctx = + (struct dict_transaction_memory_context *)_ctx; + struct dict_transaction_memory_change *change; + + change = array_append_space(&ctx->changes); + change->type = DICT_CHANGE_TYPE_INC; + change->key = p_strdup(ctx->pool, key); + change->value.diff = diff; +} diff --git a/src/lib-dict/dict-transaction-memory.h b/src/lib-dict/dict-transaction-memory.h new file mode 100644 index 0000000..2164c1e --- /dev/null +++ b/src/lib-dict/dict-transaction-memory.h @@ -0,0 +1,38 @@ +#ifndef DICT_TRANSACTION_MEMORY_H +#define DICT_TRANSACTION_MEMORY_H + +#include "dict-private.h" + +enum dict_change_type { + DICT_CHANGE_TYPE_SET, + DICT_CHANGE_TYPE_UNSET, + DICT_CHANGE_TYPE_INC +}; + +struct dict_transaction_memory_change { + enum dict_change_type type; + const char *key; + union { + const char *str; + long long diff; + } value; +}; + +struct dict_transaction_memory_context { + struct dict_transaction_context ctx; + pool_t pool; + ARRAY(struct dict_transaction_memory_change) changes; +}; + +void dict_transaction_memory_init(struct dict_transaction_memory_context *ctx, + struct dict *dict, pool_t pool); +void dict_transaction_memory_rollback(struct dict_transaction_context *ctx); + +void dict_transaction_memory_set(struct dict_transaction_context *ctx, + const char *key, const char *value); +void dict_transaction_memory_unset(struct dict_transaction_context *ctx, + const char *key); +void dict_transaction_memory_atomic_inc(struct dict_transaction_context *ctx, + const char *key, long long diff); + +#endif diff --git a/src/lib-dict/dict-txn-lua.c b/src/lib-dict/dict-txn-lua.c new file mode 100644 index 0000000..34a475d --- /dev/null +++ b/src/lib-dict/dict-txn-lua.c @@ -0,0 +1,262 @@ +/* Copyright (c) 2021 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "dict.h" +#include "dlua-script-private.h" +#include "dict-lua-private.h" +#include "dlua-wrapper.h" + +struct lua_dict_txn { + pool_t pool; + struct dict_transaction_context *txn; + enum { + STATE_OPEN, + STATE_COMMITTED, + STATE_ABORTED, + } state; + + lua_State *L; + const char *username; +}; + +static int lua_dict_transaction_rollback(lua_State *L); +static int lua_dict_transaction_commit(lua_State *L); +static int lua_dict_set(lua_State *L); +static int lua_dict_unset(lua_State *L); +static int lua_dict_set_timestamp(lua_State *L); + +static luaL_Reg lua_dict_txn_methods[] = { + { "rollback", lua_dict_transaction_rollback }, + { "commit", lua_dict_transaction_commit }, + { "set", lua_dict_set }, + { "unset", lua_dict_unset }, + { "set_timestamp", lua_dict_set_timestamp }, + { NULL, NULL }, +}; + +static void sanity_check_txn(lua_State *L, struct lua_dict_txn *txn) +{ + switch (txn->state) { + case STATE_OPEN: + return; + case STATE_COMMITTED: + luaL_error(L, "dict transaction already committed"); + return; + case STATE_ABORTED: + luaL_error(L, "dict transaction already aborted"); + return; + } + + i_unreached(); +} + +/* no actual ref counting, but we use it for clean up */ +static void lua_dict_txn_unref(struct lua_dict_txn *txn) +{ + /* rollback any transactions that were forgotten about */ + dict_transaction_rollback(&txn->txn); + + pool_unref(&txn->pool); +} + +DLUA_WRAP_C_DATA(dict_txn, struct lua_dict_txn, lua_dict_txn_unref, + lua_dict_txn_methods); + +/* + * Abort a transaction [-1,+0,e] + * + * Args: + * 1) userdata: struct lua_dict_txn * + */ +static int lua_dict_transaction_rollback(lua_State *L) +{ + struct lua_dict_txn *txn; + + DLUA_REQUIRE_ARGS(L, 1); + + txn = xlua_dict_txn_getptr(L, 1, NULL); + sanity_check_txn(L, txn); + + txn->state = STATE_ABORTED; + dict_transaction_rollback(&txn->txn); + + return 0; +} + +static int lua_dict_transaction_commit_continue(lua_State *L, + int status ATTR_UNUSED, + lua_KContext ctx ATTR_UNUSED) +{ + if (!lua_isnil(L, -1)) + lua_error(L); /* commit failed */ + + lua_pop(L, 1); /* pop the nil indicating the lack of error */ + + return 0; +} + +static void +lua_dict_transaction_commit_callback(const struct dict_commit_result *result, + struct lua_dict_txn *txn) +{ + + switch (result->ret) { + case DICT_COMMIT_RET_OK: + /* push a nil to indicate everything is ok */ + lua_pushnil(txn->L); + break; + case DICT_COMMIT_RET_NOTFOUND: + /* we don't expose dict_atomic_inc(), so this should never happen */ + i_unreached(); + case DICT_COMMIT_RET_FAILED: + case DICT_COMMIT_RET_WRITE_UNCERTAIN: + /* push the error we'll raise when we resume */ + i_assert(result->error != NULL); + lua_pushfstring(txn->L, "dict transaction commit failed: %s", + result->error); + break; + } + + dlua_pcall_yieldable_resume(txn->L, 1); +} + +/* + * Commit a transaction [-1,+0,e] + * + * Args: + * 1) userdata: struct lua_dict_txn * + */ +static int lua_dict_transaction_commit(lua_State *L) +{ + struct lua_dict_txn *txn; + + DLUA_REQUIRE_ARGS(L, 1); + + txn = xlua_dict_txn_getptr(L, 1, NULL); + sanity_check_txn(L, txn); + + txn->state = STATE_COMMITTED; + dict_transaction_commit_async(&txn->txn, + lua_dict_transaction_commit_callback, txn); + + return lua_dict_transaction_commit_continue(L, + lua_yieldk(L, 0, 0, lua_dict_transaction_commit_continue), 0); +} + +/* + * Set key to value [-3,+0,e] + * + * Args: + * 1) userdata: struct lua_dict_txn * + * 2) string: key + * 3) string: value + */ +static int lua_dict_set(lua_State *L) +{ + struct lua_dict_txn *txn; + const char *key, *value; + + DLUA_REQUIRE_ARGS(L, 3); + + txn = xlua_dict_txn_getptr(L, 1, NULL); + key = luaL_checkstring(L, 2); + value = luaL_checkstring(L, 3); + lua_dict_check_key_prefix(L, key, txn->username); + + dict_set(txn->txn, key, value); + + return 0; +} + +/* + * Unset key [-2,+0,e] + * + * Args: + * 1) userdata: struct lua_dict_txn * + * 2) string: key + */ +static int lua_dict_unset(lua_State *L) +{ + struct lua_dict_txn *txn; + const char *key; + + DLUA_REQUIRE_ARGS(L, 2); + + txn = xlua_dict_txn_getptr(L, 1, NULL); + key = luaL_checkstring(L, 2); + lua_dict_check_key_prefix(L, key, txn->username); + + dict_unset(txn->txn, key); + + return 0; +} + +/* + * Start a dict transaction [-(1|2),+1,e] + * + * Args: + * 1) userdata: struct dict * + * 2*) string: username + * + * Returns: + * Returns a new transaction object. + * Username will be NULL if not provided in args. + */ +int lua_dict_transaction_begin(lua_State *L) +{ + struct lua_dict_txn *txn; + struct dict *dict; + const char *username = NULL; + pool_t pool; + + DLUA_REQUIRE_ARGS_IN(L, 1, 2); + + dict = dlua_check_dict(L, 1); + if (lua_gettop(L) >= 2) + username = luaL_checkstring(L, 2); + + pool = pool_alloconly_create("lua dict txn", 128); + txn = p_new(pool, struct lua_dict_txn, 1); + txn->pool = pool; + + struct dict_op_settings set = { + .username = username, + }; + txn->txn = dict_transaction_begin(dict, &set); + txn->state = STATE_OPEN; + txn->L = L; + txn->username = p_strdup(txn->pool, username); + + xlua_pushdict_txn(L, txn, FALSE); + + return 1; +} + +/* + * Set timestamp to the transaction [-2,+0,e] + * + * Args: + * 1) userdata: struct lua_dict_txn * + * 2) PosixTimespec : { tv_sec, tv_nsec } + */ +static int lua_dict_set_timestamp(lua_State *L) +{ + struct lua_dict_txn *txn; + lua_Number tv_sec, tv_nsec; + + DLUA_REQUIRE_ARGS(L, 2); + + txn = xlua_dict_txn_getptr(L, 1, NULL); + if (dlua_table_get_number_by_str(L, 2, "tv_sec", &tv_sec) <= 0) + luaL_error(L, "tv_sec missing from table"); + if (dlua_table_get_number_by_str(L, 2, "tv_nsec", &tv_nsec) <= 0) + luaL_error(L, "tv_nsec missing from table"); + + struct timespec ts = { + .tv_sec = tv_sec, + .tv_nsec = tv_nsec + }; + dict_transaction_set_timestamp(txn->txn, &ts); + return 0; +} diff --git a/src/lib-dict/dict.c b/src/lib-dict/dict.c new file mode 100644 index 0000000..cdaec09 --- /dev/null +++ b/src/lib-dict/dict.c @@ -0,0 +1,759 @@ +/* Copyright (c) 2005-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "guid.h" +#include "llist.h" +#include "ioloop.h" +#include "str.h" +#include "ioloop.h" +#include "dict-private.h" + +struct dict_commit_callback_ctx { + pool_t pool; + struct dict_commit_callback_ctx *prev, *next; + struct dict *dict; + struct event *event; + dict_transaction_commit_callback_t *callback; + struct dict_op_settings_private set; + struct timeout *to; + void *context; + struct dict_commit_result result; + bool delayed_callback:1; +}; + +struct dict_lookup_callback_ctx { + struct dict *dict; + struct event *event; + dict_lookup_callback_t *callback; + void *context; +}; + +static ARRAY(struct dict *) dict_drivers; + +static void +dict_commit_async_timeout(struct dict_commit_callback_ctx *ctx); + +static struct event_category event_category_dict = { + .name = "dict", +}; + +static struct dict *dict_driver_lookup(const char *name) +{ + struct dict *dict; + + array_foreach_elem(&dict_drivers, dict) { + if (strcmp(dict->name, name) == 0) + return dict; + } + return NULL; +} + +void dict_transaction_commit_async_noop_callback( + const struct dict_commit_result *result ATTR_UNUSED, + void *context ATTR_UNUSED) +{ + /* do nothing */ +} + +void dict_driver_register(struct dict *driver) +{ + if (!array_is_created(&dict_drivers)) + i_array_init(&dict_drivers, 8); + + if (dict_driver_lookup(driver->name) != NULL) { + i_fatal("dict_driver_register(%s): Already registered", + driver->name); + } + array_push_back(&dict_drivers, &driver); +} + +void dict_driver_unregister(struct dict *driver) +{ + struct dict *const *dicts; + unsigned int idx = UINT_MAX; + + array_foreach(&dict_drivers, dicts) { + if (*dicts == driver) { + idx = array_foreach_idx(&dict_drivers, dicts); + break; + } + } + i_assert(idx != UINT_MAX); + array_delete(&dict_drivers, idx, 1); + + if (array_count(&dict_drivers) == 0) + array_free(&dict_drivers); +} + +int dict_init(const char *uri, const struct dict_settings *set, + struct dict **dict_r, const char **error_r) +{ + struct dict_settings set_dup = *set; + struct dict *dict; + const char *p, *name, *error; + + p = strchr(uri, ':'); + if (p == NULL) { + *error_r = t_strdup_printf("Dictionary URI is missing ':': %s", + uri); + return -1; + } + + name = t_strdup_until(uri, p); + dict = dict_driver_lookup(name); + if (dict == NULL) { + *error_r = t_strdup_printf("Unknown dict module: %s", name); + return -1; + } + struct event *event = event_create(set->event_parent); + event_add_category(event, &event_category_dict); + event_add_str(event, "driver", dict->name); + event_set_append_log_prefix(event, t_strdup_printf("dict(%s): ", + dict->name)); + set_dup.event_parent = event; + if (dict->v.init(dict, p+1, &set_dup, dict_r, &error) < 0) { + *error_r = t_strdup_printf("dict %s: %s", name, error); + event_unref(&event); + return -1; + } + i_assert(*dict_r != NULL); + (*dict_r)->refcount++; + (*dict_r)->event = event; + e_debug(event_create_passthrough(event)->set_name("dict_created")->event(), + "dict created (uri=%s, base_dir=%s)", uri, set->base_dir); + + return 0; +} + +static void dict_ref(struct dict *dict) +{ + i_assert(dict->refcount > 0); + + dict->refcount++; +} + +static void dict_unref(struct dict **_dict) +{ + struct dict *dict = *_dict; + *_dict = NULL; + if (dict == NULL) + return; + struct event *event = dict->event; + i_assert(dict->refcount > 0); + if (--dict->refcount == 0) { + dict->v.deinit(dict); + e_debug(event_create_passthrough(event)-> + set_name("dict_destroyed")->event(), "dict destroyed"); + event_unref(&event); + } +} + +void dict_deinit(struct dict **_dict) +{ + struct dict *dict = *_dict; + + *_dict = NULL; + + i_assert(dict->iter_count == 0); + i_assert(dict->transaction_count == 0); + i_assert(dict->transactions == NULL); + i_assert(dict->commits == NULL); + dict_unref(&dict); +} + +void dict_wait(struct dict *dict) +{ + struct dict_commit_callback_ctx *commit, *next; + + e_debug(dict->event, "Waiting for dict to finish pending operations"); + if (dict->v.wait != NULL) + dict->v.wait(dict); + for (commit = dict->commits; commit != NULL; commit = next) { + next = commit->next; + dict_commit_async_timeout(commit); + } +} + +bool dict_switch_ioloop(struct dict *dict) +{ + struct dict_commit_callback_ctx *commit; + bool ret = FALSE; + + for (commit = dict->commits; commit != NULL; commit = commit->next) { + commit->to = io_loop_move_timeout(&commit->to); + ret = TRUE; + } + if (dict->v.switch_ioloop != NULL) { + if (dict->v.switch_ioloop(dict)) + return TRUE; + } + return ret; +} + +static bool dict_key_prefix_is_valid(const char *key, const char *username) +{ + if (str_begins(key, DICT_PATH_SHARED)) + return TRUE; + if (str_begins(key, DICT_PATH_PRIVATE)) { + i_assert(username != NULL && username[0] != '\0'); + return TRUE; + } + return FALSE; + +} + +void dict_pre_api_callback(struct dict *dict) +{ + if (dict->prev_ioloop != NULL) { + /* Don't let callback see that we've created our + internal ioloop in case it wants to add some ios + or timeouts. */ + io_loop_set_current(dict->prev_ioloop); + } +} + +void dict_post_api_callback(struct dict *dict) +{ + if (dict->prev_ioloop != NULL) { + io_loop_set_current(dict->ioloop); + io_loop_stop(dict->ioloop); + } +} + +static void dict_lookup_finished(struct event *event, int ret, const char *error) +{ + i_assert(ret >= 0 || error != NULL); + const char *key = event_find_field_recursive_str(event, "key"); + if (ret < 0) + event_add_str(event, "error", error); + else if (ret == 0) + event_add_str(event, "key_not_found", "yes"); + event_set_name(event, "dict_lookup_finished"); + e_debug(event, "Lookup finished for '%s': %s", key, ret > 0 ? + "found" : + "not found"); +} + +static void dict_transaction_finished(struct event *event, enum dict_commit_ret ret, + bool rollback, const char *error) +{ + i_assert(ret > DICT_COMMIT_RET_FAILED || error != NULL); + if (ret == DICT_COMMIT_RET_FAILED || ret == DICT_COMMIT_RET_WRITE_UNCERTAIN) { + if (ret == DICT_COMMIT_RET_WRITE_UNCERTAIN) + event_add_str(event, "write_uncertain", "yes"); + event_add_str(event, "error", error); + } else if (rollback) { + event_add_str(event, "rollback", "yes"); + } else if (ret == 0) { + event_add_str(event, "key_not_found", "yes"); + } + event_set_name(event, "dict_transaction_finished"); + e_debug(event, "Dict transaction finished"); +} + +static void +dict_lookup_callback(const struct dict_lookup_result *result, + void *context) +{ + struct dict_lookup_callback_ctx *ctx = context; + + dict_pre_api_callback(ctx->dict); + ctx->callback(result, ctx->context); + dict_post_api_callback(ctx->dict); + dict_lookup_finished(ctx->event, result->ret, result->error); + event_unref(&ctx->event); + + dict_unref(&ctx->dict); + i_free(ctx); +} + +static void +dict_commit_async_timeout(struct dict_commit_callback_ctx *ctx) +{ + DLLIST_REMOVE(&ctx->dict->commits, ctx); + timeout_remove(&ctx->to); + dict_pre_api_callback(ctx->dict); + if (ctx->callback != NULL) + ctx->callback(&ctx->result, ctx->context); + else if (ctx->result.ret < 0) + e_error(ctx->event, "Commit failed: %s", ctx->result.error); + dict_post_api_callback(ctx->dict); + + dict_transaction_finished(ctx->event, ctx->result.ret, FALSE, ctx->result.error); + dict_op_settings_private_free(&ctx->set); + event_unref(&ctx->event); + dict_unref(&ctx->dict); + pool_unref(&ctx->pool); +} + +static void dict_commit_callback(const struct dict_commit_result *result, + void *context) +{ + struct dict_commit_callback_ctx *ctx = context; + + i_assert(result->ret >= 0 || result->error != NULL); + ctx->result = *result; + if (ctx->delayed_callback) { + ctx->result.error = p_strdup(ctx->pool, ctx->result.error); + ctx->to = timeout_add_short(0, dict_commit_async_timeout, ctx); + } else { + dict_commit_async_timeout(ctx); + } +} + +static struct event * +dict_event_create(struct dict *dict, const struct dict_op_settings *set) +{ + struct event *event = event_create(dict->event); + if (set->username != NULL) + event_add_str(event, "user", set->username); + return event; +} + +int dict_lookup(struct dict *dict, const struct dict_op_settings *set, + pool_t pool, const char *key, + const char **value_r, const char **error_r) +{ + struct event *event = dict_event_create(dict, set); + int ret; + i_assert(dict_key_prefix_is_valid(key, set->username)); + + e_debug(event, "Looking up '%s'", key); + event_add_str(event, "key", key); + ret = dict->v.lookup(dict, set, pool, key, value_r, error_r); + dict_lookup_finished(event, ret, *error_r); + event_unref(&event); + return ret; +} + +#undef dict_lookup_async +void dict_lookup_async(struct dict *dict, const struct dict_op_settings *set, + const char *key, dict_lookup_callback_t *callback, + void *context) +{ + i_assert(dict_key_prefix_is_valid(key, set->username)); + if (dict->v.lookup_async == NULL) { + struct dict_lookup_result result; + + i_zero(&result); + /* event is going to be sent by dict_lookup */ + result.ret = dict_lookup(dict, set, pool_datastack_create(), + key, &result.value, &result.error); + const char *const values[] = { result.value, NULL }; + result.values = values; + callback(&result, context); + return; + } + struct dict_lookup_callback_ctx *lctx = + i_new(struct dict_lookup_callback_ctx, 1); + lctx->dict = dict; + dict_ref(lctx->dict); + lctx->callback = callback; + lctx->context = context; + lctx->event = dict_event_create(dict, set); + event_add_str(lctx->event, "key", key); + e_debug(lctx->event, "Looking up (async) '%s'", key); + dict->v.lookup_async(dict, set, key, dict_lookup_callback, lctx); +} + +struct dict_iterate_context * +dict_iterate_init(struct dict *dict, const struct dict_op_settings *set, + const char *path, enum dict_iterate_flags flags) +{ + struct dict_iterate_context *ctx; + + i_assert(path != NULL); + i_assert(dict_key_prefix_is_valid(path, set->username)); + + if (dict->v.iterate_init == NULL) { + /* not supported by backend */ + ctx = &dict_iter_unsupported; + } else { + ctx = dict->v.iterate_init(dict, set, path, flags); + } + /* the dict in context can differ from the dict + passed as parameter, e.g. it can be dict-fail when + iteration is not supported. */ + ctx->event = dict_event_create(dict, set); + ctx->flags = flags; + dict_op_settings_dup(set, &ctx->set); + + event_add_str(ctx->event, "key", path); + event_set_name(ctx->event, "dict_iteration_started"); + e_debug(ctx->event, "Iterating prefix %s", path); + ctx->dict->iter_count++; + return ctx; +} + +bool dict_iterate(struct dict_iterate_context *ctx, + const char **key_r, const char **value_r) +{ + const char *const *values; + + if (!dict_iterate_values(ctx, key_r, &values)) + return FALSE; + if ((ctx->flags & DICT_ITERATE_FLAG_NO_VALUE) == 0) + *value_r = values[0]; + return TRUE; +} + +bool dict_iterate_values(struct dict_iterate_context *ctx, + const char **key_r, const char *const **values_r) +{ + + if (ctx->max_rows > 0 && ctx->row_count >= ctx->max_rows) { + e_debug(ctx->event, "Maximum row count (%"PRIu64") reached", + ctx->max_rows); + /* row count was limited */ + ctx->has_more = FALSE; + return FALSE; + } + if (!ctx->dict->v.iterate(ctx, key_r, values_r)) + return FALSE; + if ((ctx->flags & DICT_ITERATE_FLAG_NO_VALUE) != 0) { + /* always return value as NULL to be consistent across + drivers */ + *values_r = NULL; + } else { + i_assert(values_r[0] != NULL); + } + ctx->row_count++; + return TRUE; +} + +#undef dict_iterate_set_async_callback +void dict_iterate_set_async_callback(struct dict_iterate_context *ctx, + dict_iterate_callback_t *callback, + void *context) +{ + ctx->async_callback = callback; + ctx->async_context = context; +} + +void dict_iterate_set_limit(struct dict_iterate_context *ctx, + uint64_t max_rows) +{ + ctx->max_rows = max_rows; +} + +bool dict_iterate_has_more(struct dict_iterate_context *ctx) +{ + return ctx->has_more; +} + +int dict_iterate_deinit(struct dict_iterate_context **_ctx, + const char **error_r) +{ + struct dict_iterate_context *ctx = *_ctx; + + if (ctx == NULL) + return 0; + + struct event *event = ctx->event; + int ret; + uint64_t rows; + + i_assert(ctx->dict->iter_count > 0); + ctx->dict->iter_count--; + + *_ctx = NULL; + rows = ctx->row_count; + struct dict_op_settings_private set_copy = ctx->set; + ret = ctx->dict->v.iterate_deinit(ctx, error_r); + dict_op_settings_private_free(&set_copy); + + event_add_int(event, "rows", rows); + event_set_name(event, "dict_iteration_finished"); + + if (ret < 0) { + event_add_str(event, "error", *error_r); + e_debug(event, "Iteration finished: %s", *error_r); + } else { + if (rows == 0) + event_add_str(event, "key_not_found", "yes"); + e_debug(event, "Iteration finished, got %"PRIu64" rows", rows); + } + + event_unref(&event); + return ret; +} + +struct dict_transaction_context * +dict_transaction_begin(struct dict *dict, const struct dict_op_settings *set) +{ + struct dict_transaction_context *ctx; + guid_128_t guid; + if (dict->v.transaction_init == NULL) + ctx = &dict_transaction_unsupported; + else + ctx = dict->v.transaction_init(dict); + /* the dict in context can differ from the dict + passed as parameter, e.g. it can be dict-fail when + transactions are not supported. */ + ctx->dict->transaction_count++; + DLLIST_PREPEND(&ctx->dict->transactions, ctx); + ctx->event = dict_event_create(dict, set); + dict_op_settings_dup(set, &ctx->set); + guid_128_generate(guid); + event_add_str(ctx->event, "txid", guid_128_to_string(guid)); + event_set_name(ctx->event, "dict_transaction_started"); + e_debug(ctx->event, "Starting transaction"); + return ctx; +} + +void dict_transaction_no_slowness_warning(struct dict_transaction_context *ctx) +{ + ctx->no_slowness_warning = TRUE; +} + +void dict_transaction_set_timestamp(struct dict_transaction_context *ctx, + const struct timespec *ts) +{ + /* These asserts are mainly here to guarantee a possibility in future + to change the API to support multiple timestamps within the same + transaction, so this call would apply only to the following + changes. */ + i_assert(!ctx->changed); + i_assert(ctx->timestamp.tv_sec == 0); + i_assert(ts->tv_sec > 0); + + ctx->timestamp = *ts; + struct event_passthrough *e = event_create_passthrough(ctx->event)-> + set_name("dict_set_timestamp"); + + e_debug(e->event(), "Setting timestamp on transaction to (%"PRIdTIME_T", %ld)", + ts->tv_sec, ts->tv_nsec); + if (ctx->dict->v.set_timestamp != NULL) + ctx->dict->v.set_timestamp(ctx, ts); +} + +struct dict_commit_sync_result { + int ret; + char *error; +}; + +static void +dict_transaction_commit_sync_callback(const struct dict_commit_result *result, + void *context) +{ + struct dict_commit_sync_result *sync_result = context; + + sync_result->ret = result->ret; + sync_result->error = i_strdup(result->error); +} + +int dict_transaction_commit(struct dict_transaction_context **_ctx, + const char **error_r) +{ + pool_t pool = pool_alloconly_create("dict_commit_callback_ctx", 64); + struct dict_commit_callback_ctx *cctx = + p_new(pool, struct dict_commit_callback_ctx, 1); + struct dict_transaction_context *ctx = *_ctx; + struct dict_commit_sync_result result; + + *_ctx = NULL; + cctx->pool = pool; + i_zero(&result); + i_assert(ctx->dict->transaction_count > 0); + ctx->dict->transaction_count--; + DLLIST_REMOVE(&ctx->dict->transactions, ctx); + DLLIST_PREPEND(&ctx->dict->commits, cctx); + cctx->dict = ctx->dict; + dict_ref(cctx->dict); + cctx->callback = dict_transaction_commit_sync_callback; + cctx->context = &result; + cctx->event = ctx->event; + cctx->set = ctx->set; + + ctx->dict->v.transaction_commit(ctx, FALSE, dict_commit_callback, cctx); + *error_r = t_strdup(result.error); + i_free(result.error); + return result.ret; +} + +#undef dict_transaction_commit_async +void dict_transaction_commit_async(struct dict_transaction_context **_ctx, + dict_transaction_commit_callback_t *callback, + void *context) +{ + pool_t pool = pool_alloconly_create("dict_commit_callback_ctx", 64); + struct dict_commit_callback_ctx *cctx = + p_new(pool, struct dict_commit_callback_ctx, 1); + struct dict_transaction_context *ctx = *_ctx; + + *_ctx = NULL; + i_assert(ctx->dict->transaction_count > 0); + ctx->dict->transaction_count--; + DLLIST_REMOVE(&ctx->dict->transactions, ctx); + DLLIST_PREPEND(&ctx->dict->commits, cctx); + if (callback == NULL) + callback = dict_transaction_commit_async_noop_callback; + cctx->pool = pool; + cctx->dict = ctx->dict; + dict_ref(cctx->dict); + cctx->callback = callback; + cctx->context = context; + cctx->event = ctx->event; + cctx->set = ctx->set; + cctx->delayed_callback = TRUE; + ctx->dict->v.transaction_commit(ctx, TRUE, dict_commit_callback, cctx); + cctx->delayed_callback = FALSE; +} + +void dict_transaction_commit_async_nocallback( + struct dict_transaction_context **ctx) +{ + dict_transaction_commit_async(ctx, NULL, NULL); +} + +void dict_transaction_rollback(struct dict_transaction_context **_ctx) +{ + struct dict_transaction_context *ctx = *_ctx; + + if (ctx == NULL) + return; + + struct event *event = ctx->event; + + *_ctx = NULL; + i_assert(ctx->dict->transaction_count > 0); + ctx->dict->transaction_count--; + DLLIST_REMOVE(&ctx->dict->transactions, ctx); + struct dict_op_settings_private set_copy = ctx->set; + ctx->dict->v.transaction_rollback(ctx); + dict_transaction_finished(event, DICT_COMMIT_RET_OK, TRUE, NULL); + dict_op_settings_private_free(&set_copy); + event_unref(&event); +} + +void dict_set(struct dict_transaction_context *ctx, + const char *key, const char *value) +{ + i_assert(dict_key_prefix_is_valid(key, ctx->set.username)); + struct event_passthrough *e = event_create_passthrough(ctx->event)-> + set_name("dict_set_key")-> + add_str("key", key); + + e_debug(e->event(), "Setting '%s' to '%s'", key, value); + + T_BEGIN { + ctx->dict->v.set(ctx, key, value); + } T_END; + ctx->changed = TRUE; +} + +void dict_unset(struct dict_transaction_context *ctx, + const char *key) +{ + i_assert(dict_key_prefix_is_valid(key, ctx->set.username)); + struct event_passthrough *e = event_create_passthrough(ctx->event)-> + set_name("dict_unset_key")-> + add_str("key", key); + + e_debug(e->event(), "Unsetting '%s'", key); + + T_BEGIN { + ctx->dict->v.unset(ctx, key); + } T_END; + ctx->changed = TRUE; +} + +void dict_atomic_inc(struct dict_transaction_context *ctx, + const char *key, long long diff) +{ + i_assert(dict_key_prefix_is_valid(key, ctx->set.username)); + struct event_passthrough *e = event_create_passthrough(ctx->event)-> + set_name("dict_increment_key")-> + add_str("key", key); + + e_debug(e->event(), "Incrementing '%s' with %lld", key, diff); + + if (diff != 0) T_BEGIN { + ctx->dict->v.atomic_inc(ctx, key, diff); + ctx->changed = TRUE; + } T_END; +} + +const char *dict_escape_string(const char *str) +{ + const char *p; + string_t *ret; + + /* see if we need to escape it */ + for (p = str; *p != '\0'; p++) { + if (*p == '/' || *p == '\\') + break; + } + + if (*p == '\0') + return str; + + /* escape */ + ret = t_str_new((size_t) (p - str) + 128); + str_append_data(ret, str, (size_t) (p - str)); + + for (; *p != '\0'; p++) { + switch (*p) { + case '/': + str_append_c(ret, '\\'); + str_append_c(ret, '|'); + break; + case '\\': + str_append_c(ret, '\\'); + str_append_c(ret, '\\'); + break; + default: + str_append_c(ret, *p); + break; + } + } + return str_c(ret); +} + +const char *dict_unescape_string(const char *str) +{ + const char *p; + string_t *ret; + + /* see if we need to unescape it */ + for (p = str; *p != '\0'; p++) { + if (*p == '\\') + break; + } + + if (*p == '\0') + return str; + + /* unescape */ + ret = t_str_new((size_t) (p - str) + strlen(p) + 1); + str_append_data(ret, str, (size_t) (p - str)); + + for (; *p != '\0'; p++) { + if (*p != '\\') + str_append_c(ret, *p); + else { + if (*++p == '|') + str_append_c(ret, '/'); + else if (*p == '\0') + break; + else + str_append_c(ret, *p); + } + } + return str_c(ret); +} + +void dict_op_settings_dup(const struct dict_op_settings *source, + struct dict_op_settings_private *dest_r) +{ + i_zero(dest_r); + dest_r->username = i_strdup(source->username); + dest_r->home_dir = i_strdup(source->home_dir); +} + +void dict_op_settings_private_free(struct dict_op_settings_private *set) +{ + i_free(set->username); + i_free(set->home_dir); +} diff --git a/src/lib-dict/dict.h b/src/lib-dict/dict.h new file mode 100644 index 0000000..8e4b39e --- /dev/null +++ b/src/lib-dict/dict.h @@ -0,0 +1,200 @@ +#ifndef DICT_H +#define DICT_H + +#define DICT_PATH_PRIVATE "priv/" +#define DICT_PATH_SHARED "shared/" + +struct timespec; +struct dict; +struct dict_iterate_context; + +enum dict_iterate_flags { + /* Recurse to all the sub-hierarchies (e.g. iterating "foo/" will + return "foo/a", but should it return "foo/a/b"?) */ + DICT_ITERATE_FLAG_RECURSE = 0x01, + /* Sort returned results by key */ + DICT_ITERATE_FLAG_SORT_BY_KEY = 0x02, + /* Sort returned results by value */ + DICT_ITERATE_FLAG_SORT_BY_VALUE = 0x04, + /* Don't return values, only keys */ + DICT_ITERATE_FLAG_NO_VALUE = 0x08, + /* Don't recurse at all. This is basically the same as dict_lookup(), + but it'll return all the rows instead of only the first one. */ + DICT_ITERATE_FLAG_EXACT_KEY = 0x10, + /* Perform iteration asynchronously. */ + DICT_ITERATE_FLAG_ASYNC = 0x20 +}; + +enum dict_data_type { + DICT_DATA_TYPE_STRING = 0, + DICT_DATA_TYPE_UINT32, + DICT_DATA_TYPE_LAST +}; + +struct dict_settings { + const char *base_dir; + /* set to parent event, if exists */ + struct event *event_parent; +}; + +struct dict_op_settings { + const char *username; + /* home directory for the user, if known */ + const char *home_dir; +}; + +struct dict_lookup_result { + int ret; + + /* First returned value (ret > 0) */ + const char *value; + /* NULL-terminated list of all returned values (ret > 0) */ + const char *const *values; + + /* Error message for a failed lookup (ret < 0) */ + const char *error; +}; + +enum dict_commit_ret { + DICT_COMMIT_RET_OK = 1, + DICT_COMMIT_RET_NOTFOUND = 0, + DICT_COMMIT_RET_FAILED = -1, + /* write may or may not have succeeded (e.g. write timeout or + disconnected from server) */ + DICT_COMMIT_RET_WRITE_UNCERTAIN = -2, +}; + +struct dict_commit_result { + enum dict_commit_ret ret; + const char *error; +}; + +typedef void dict_lookup_callback_t(const struct dict_lookup_result *result, + void *context); +typedef void dict_iterate_callback_t(void *context); +typedef void +dict_transaction_commit_callback_t(const struct dict_commit_result *result, + void *context); + +void dict_driver_register(struct dict *driver); +void dict_driver_unregister(struct dict *driver); + +void dict_drivers_register_builtin(void); +void dict_drivers_unregister_builtin(void); + +void dict_drivers_register_all(void); +void dict_drivers_unregister_all(void); + +/* Open dictionary with given URI (type:data). + Returns 0 if ok, -1 if URI is invalid. */ +int dict_init(const char *uri, const struct dict_settings *set, + struct dict **dict_r, const char **error_r); +/* Close dictionary. */ +void dict_deinit(struct dict **dict); +/* Wait for all pending asynchronous operations to finish. */ +void dict_wait(struct dict *dict); +/* Switch the dict to the current ioloop. This can be used to do dict_wait() + among other IO work. Returns TRUE if there is actually some work that can + be waited on. */ +bool dict_switch_ioloop(struct dict *dict) ATTR_NOWARN_UNUSED_RESULT; + +/* Lookup value for key. Set it to NULL if it's not found. + Returns 1 if found, 0 if not found and -1 if lookup failed. */ +int dict_lookup(struct dict *dict, const struct dict_op_settings *set, pool_t pool, + const char *key, const char **value_r, const char **error_r); +void dict_lookup_async(struct dict *dict, const struct dict_op_settings *set, + const char *key, dict_lookup_callback_t *callback, + void *context); +#define dict_lookup_async(dict, set, key, callback, context) \ + dict_lookup_async(dict, set, key, (dict_lookup_callback_t *)(callback), \ + 1 ? (context) : \ + CALLBACK_TYPECHECK(callback, \ + void (*)(const struct dict_lookup_result *, typeof(context)))) + +/* Iterate through all values in a path. flag indicates how iteration + is carried out */ +struct dict_iterate_context * +dict_iterate_init(struct dict *dict, const struct dict_op_settings *set, + const char *path, enum dict_iterate_flags flags); +/* Set async callback. Note that if dict_iterate_init() already did all the + work, this callback may never be called. So after dict_iterate_init() you + should call dict_iterate() in any case to see if all the results are + already available. */ +void dict_iterate_set_async_callback(struct dict_iterate_context *ctx, + dict_iterate_callback_t *callback, + void *context); +#define dict_iterate_set_async_callback(ctx, callback, context) \ + dict_iterate_set_async_callback(ctx, (dict_iterate_callback_t *)(callback), \ + 1 ? (context) : \ + CALLBACK_TYPECHECK(callback, void (*)(typeof(context)))) +/* Limit how many rows will be returned by the iteration (0 = unlimited). + This allows backends to optimize the query (e.g. use LIMIT 1 with SQL). */ +void dict_iterate_set_limit(struct dict_iterate_context *ctx, + uint64_t max_rows); +/* If dict_iterate() returns FALSE, the iteration may be finished or if this + is an async iteration it may be waiting for more data. If this function + returns TRUE, the dict callback is called again with more data. If dict + supports multiple values, dict_iterate_values() can be used to return all + of them. dict_iterate() returns only the first value and ignores the rest. */ +bool dict_iterate_has_more(struct dict_iterate_context *ctx); +bool dict_iterate(struct dict_iterate_context *ctx, + const char **key_r, const char **value_r); +bool dict_iterate_values(struct dict_iterate_context *ctx, + const char **key_r, const char *const **values_r); +/* Returns 0 = ok, -1 = iteration failed */ +int dict_iterate_deinit(struct dict_iterate_context **ctx, const char **error_r); + +/* Start a new dictionary transaction. */ +struct dict_transaction_context * +dict_transaction_begin(struct dict *dict, const struct dict_op_settings *set); +/* Don't log a warning if the transaction commit took a long time. + This is needed if there are no guarantees that an asynchronous commit will + finish up anytime soon. Mainly useful for transactions which aren't + especially important whether they finish or not. */ +void dict_transaction_no_slowness_warning(struct dict_transaction_context *ctx); +/* Set write timestamp for the entire transaction. This must be set before + any changes are done and can't be changed afterwards. Currently only + dict-sql with Cassandra backend does anything with this. */ +void dict_transaction_set_timestamp(struct dict_transaction_context *ctx, + const struct timespec *ts); +/* Commit the transaction. Returns 1 if ok, 0 if dict_atomic_inc() was used + on a nonexistent key, -1 if failed. */ +int dict_transaction_commit(struct dict_transaction_context **ctx, + const char **error_r); +/* Commit the transaction, but don't wait to see if it finishes successfully. + The callback is called when the transaction is finished. If it's not called + by the time you want to deinitialize dict, call dict_flush() to wait for the + result. */ +void dict_transaction_commit_async(struct dict_transaction_context **ctx, + dict_transaction_commit_callback_t *callback, + void *context) ATTR_NULL(2, 3); +#define dict_transaction_commit_async(ctx, callback, context) \ + dict_transaction_commit_async(ctx, (dict_transaction_commit_callback_t *)(callback), \ + 1 ? (context) : \ + CALLBACK_TYPECHECK(callback, \ + void (*)(const struct dict_commit_result *, typeof(context)))) +/* Same as dict_transaction_commit_async(), but don't call a callback. */ +void dict_transaction_commit_async_nocallback( + struct dict_transaction_context **ctx); +/* Rollback all changes made in transaction. */ +void dict_transaction_rollback(struct dict_transaction_context **ctx); + +/* Set key=value in dictionary. */ +void dict_set(struct dict_transaction_context *ctx, + const char *key, const char *value); +/* Unset a record in dictionary, identified by key*/ +void dict_unset(struct dict_transaction_context *ctx, + const char *key); +/* Increase/decrease a numeric value in dictionary. Note that the value is + changed when transaction is being committed, so you can't know beforehand + what the value will become. The value is updated only if it already exists, + otherwise commit() will return 0. */ +void dict_atomic_inc(struct dict_transaction_context *ctx, + const char *key, long long diff); + +/* Escape/unescape '/' characters in a string, so that it can be safely added + into path components in dict keys. */ +const char *dict_escape_string(const char *str); +const char *dict_unescape_string(const char *str); + +#endif diff --git a/src/lib-dict/test-dict-client.c b/src/lib-dict/test-dict-client.c new file mode 100644 index 0000000..9b3b80b --- /dev/null +++ b/src/lib-dict/test-dict-client.c @@ -0,0 +1,106 @@ +/* Copyright (c) 2016-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "ioloop.h" +#include "dict-private.h" + +#include <stdio.h> + +static int pending = 0; + +static void lookup_callback(const struct dict_lookup_result *result, + void *context ATTR_UNUSED) +{ + if (result->error != NULL) + i_error("%s", result->error); + /*else if (result->ret == 0) + i_info("not found"); + else + i_info("%s", result->value);*/ + pending--; +} + +static void commit_callback(const struct dict_commit_result *result, + void *context ATTR_UNUSED) +{ + if (result->ret < 0) + i_error("commit %d", result->ret); + pending--; +} + +int main(int argc, char *argv[]) +{ + const char *prefix, *uri; + struct dict *dict; + struct dict_settings set; + struct dict_op_settings opset; + struct ioloop *ioloop; + const char *error; + unsigned int i; + char key[1000], value[100]; + + lib_init(); + ioloop = io_loop_create(); + dict_driver_register(&dict_driver_client); + + if (argc < 3) + i_fatal("Usage: <prefix> <uri>"); + prefix = argv[1]; + uri = argv[2]; + + i_zero(&set); + i_zero(&opset); + set.base_dir = "/var/run/dovecot"; + opset.username = "testuser"; + + if (dict_init(uri, &set, &dict, &error) < 0) + i_fatal("dict_init(%s) failed: %s", argv[1], error); + + for (i = 0;; i++) { + i_snprintf(key, sizeof(key), "%s/%02x", prefix, + i_rand_limit(0xff)); + i_snprintf(value, sizeof(value), "%04x", i_rand_limit(0xffff)); + switch (i_rand_limit(4)) { + case 0: + pending++; + dict_lookup_async(dict, NULL, key, lookup_callback, NULL); + break; + case 1: { + struct dict_transaction_context *trans; + + pending++; + trans = dict_transaction_begin(dict, &opset); + dict_set(trans, key, value); + dict_transaction_commit_async(&trans, commit_callback, NULL); + break; + } + case 2: { + struct dict_transaction_context *trans; + + pending++; + trans = dict_transaction_begin(dict, &opset); + dict_unset(trans, key); + dict_transaction_commit_async(&trans, commit_callback, NULL); + break; + } + case 3: { + struct dict_iterate_context *iter; + const char *k, *v; + + iter = dict_iterate_init(dict, &opset, prefix, DICT_ITERATE_FLAG_EXACT_KEY); + while (dict_iterate(iter, &k, &v)) ; + if (dict_iterate_deinit(&iter, &error) < 0) + i_error("iter failed: %s", error); + break; + } + } + while (pending > 100) { + dict_wait(dict); + printf("%d\n", pending); fflush(stdout); + } + } + dict_deinit(&dict); + + io_loop_destroy(&ioloop); + lib_deinit(); +} diff --git a/src/lib-dict/test-dict.c b/src/lib-dict/test-dict.c new file mode 100644 index 0000000..04864ff --- /dev/null +++ b/src/lib-dict/test-dict.c @@ -0,0 +1,46 @@ +/* Copyright (c) 2010-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "dict-private.h" +#include "test-common.h" + +struct dict dict_driver_client; +struct dict dict_driver_file; +struct dict dict_driver_memcached; +struct dict dict_driver_memcached_ascii; +struct dict dict_driver_redis; + +static void test_dict_escape(void) +{ + static const char *input[] = { + "", "", + "foo", "foo", + "foo\\", "foo\\\\", + "foo\\bar", "foo\\\\bar", + "\\bar", "\\\\bar", + "foo/", "foo\\|", + "foo/bar", "foo\\|bar", + "/bar", "\\|bar", + "////", "\\|\\|\\|\\|", + "/", "\\|" + }; + unsigned int i; + + test_begin("dict escape"); + for (i = 0; i < N_ELEMENTS(input); i += 2) { + test_assert(strcmp(dict_escape_string(input[i]), input[i+1]) == 0); + test_assert(strcmp(dict_unescape_string(input[i+1]), input[i]) == 0); + } + test_assert(strcmp(dict_unescape_string("x\\"), "x") == 0); + test_assert(strcmp(dict_unescape_string("\\"), "") == 0); + test_end(); +} + +int main(void) +{ + static void (*const test_functions[])(void) = { + test_dict_escape, + NULL + }; + return test_run(test_functions); +} |