diff options
Diffstat (limited to 'src/lib-dict')
-rw-r--r-- | src/lib-dict/Makefile.am | 73 | ||||
-rw-r--r-- | src/lib-dict/Makefile.in | 970 | ||||
-rw-r--r-- | src/lib-dict/dict-client.c | 1508 | ||||
-rw-r--r-- | src/lib-dict/dict-client.h | 48 | ||||
-rw-r--r-- | src/lib-dict/dict-fail.c | 134 | ||||
-rw-r--r-- | src/lib-dict/dict-file.c | 709 | ||||
-rw-r--r-- | src/lib-dict/dict-iter-lua.c | 193 | ||||
-rw-r--r-- | src/lib-dict/dict-lua-private.h | 9 | ||||
-rw-r--r-- | src/lib-dict/dict-lua.c | 117 | ||||
-rw-r--r-- | src/lib-dict/dict-lua.h | 18 | ||||
-rw-r--r-- | src/lib-dict/dict-memcached-ascii.c | 685 | ||||
-rw-r--r-- | src/lib-dict/dict-memcached.c | 373 | ||||
-rw-r--r-- | src/lib-dict/dict-private.h | 127 | ||||
-rw-r--r-- | src/lib-dict/dict-redis.c | 831 | ||||
-rw-r--r-- | src/lib-dict/dict-transaction-memory.c | 59 | ||||
-rw-r--r-- | src/lib-dict/dict-transaction-memory.h | 38 | ||||
-rw-r--r-- | src/lib-dict/dict-txn-lua.c | 262 | ||||
-rw-r--r-- | src/lib-dict/dict.c | 769 | ||||
-rw-r--r-- | src/lib-dict/dict.h | 208 | ||||
-rw-r--r-- | src/lib-dict/test-dict-client.c | 117 | ||||
-rw-r--r-- | src/lib-dict/test-dict.c | 46 |
21 files changed, 7294 insertions, 0 deletions
diff --git a/src/lib-dict/Makefile.am b/src/lib-dict/Makefile.am new file mode 100644 index 0000000..97e5eb6 --- /dev/null +++ b/src/lib-dict/Makefile.am @@ -0,0 +1,73 @@ +noinst_LTLIBRARIES = \ + libdict.la + +AM_CPPFLAGS = \ + -I$(top_srcdir)/src/lib \ + -I$(top_srcdir)/src/lib-test \ + -I$(top_srcdir)/src/lib-settings + +base_sources = \ + dict.c \ + dict-client.c \ + dict-file.c \ + dict-memcached.c \ + dict-memcached-ascii.c \ + dict-redis.c \ + dict-fail.c \ + dict-transaction-memory.c + +libdict_la_SOURCES = \ + $(base_sources) + +headers = \ + dict.h \ + dict-client.h \ + dict-private.h \ + dict-transaction-memory.h + +# Internally, the dict methods yield via lua_yieldk() as implemented in Lua +# 5.3 and newer. +if DLUA_WITH_YIELDS +noinst_LTLIBRARIES += libdict_lua.la + +libdict_lua_la_SOURCES = \ + dict-lua.c \ + dict-iter-lua.c \ + dict-txn-lua.c +libdict_lua_la_CPPFLAGS = \ + $(AM_CPPFLAGS) \ + $(LUA_CFLAGS) \ + -I$(top_srcdir)/src/lib-lua +libdict_lua_la_LIBADD = +libdict_lua_la_DEPENDENCIES = \ + libdict.la + +headers += \ + dict-lua.h \ + dict-lua-private.h +endif + +pkginc_libdir=$(pkgincludedir) +pkginc_lib_HEADERS = $(headers) + +test_programs = \ + test-dict + +noinst_PROGRAMS = $(test_programs) test-dict-client + +test_libs = \ + ../lib-test/libtest.la \ + ../lib/liblib.la + +test_dict_SOURCES = test-dict.c +test_dict_LDADD = libdict.la $(test_libs) +test_dict_DEPENDENCIES = $(noinst_LTLIBRARIES) $(test_libs) + +test_dict_client_SOURCES = test-dict-client.c +test_dict_client_LDADD = $(noinst_LTLIBRARIES) ../lib/liblib.la +test_dict_client_DEPENDENCIES = $(noinst_LTLIBRARIES) $(test_libs) + +check-local: + for bin in $(test_programs) $(check_PROGRAMS); do \ + if ! $(RUN_TEST) ./$$bin; then exit 1; fi; \ + done diff --git a/src/lib-dict/Makefile.in b/src/lib-dict/Makefile.in new file mode 100644 index 0000000..3c55975 --- /dev/null +++ b/src/lib-dict/Makefile.in @@ -0,0 +1,970 @@ +# Makefile.in generated by automake 1.16.3 from Makefile.am. +# @configure_input@ + +# Copyright (C) 1994-2020 Free Software Foundation, Inc. + +# This Makefile.in is free software; the Free Software Foundation +# gives unlimited permission to copy and/or distribute it, +# with or without modifications, as long as this notice is preserved. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY, to the extent permitted by law; without +# even the implied warranty of MERCHANTABILITY or FITNESS FOR A +# PARTICULAR PURPOSE. + +@SET_MAKE@ + + + +VPATH = @srcdir@ +am__is_gnu_make = { \ + if test -z '$(MAKELEVEL)'; then \ + false; \ + elif test -n '$(MAKE_HOST)'; then \ + true; \ + elif test -n '$(MAKE_VERSION)' && test -n '$(CURDIR)'; then \ + true; \ + else \ + false; \ + fi; \ +} +am__make_running_with_option = \ + case $${target_option-} in \ + ?) ;; \ + *) echo "am__make_running_with_option: internal error: invalid" \ + "target option '$${target_option-}' specified" >&2; \ + exit 1;; \ + esac; \ + has_opt=no; \ + sane_makeflags=$$MAKEFLAGS; \ + if $(am__is_gnu_make); then \ + sane_makeflags=$$MFLAGS; \ + else \ + case $$MAKEFLAGS in \ + *\\[\ \ ]*) \ + bs=\\; \ + sane_makeflags=`printf '%s\n' "$$MAKEFLAGS" \ + | sed "s/$$bs$$bs[$$bs $$bs ]*//g"`;; \ + esac; \ + fi; \ + skip_next=no; \ + strip_trailopt () \ + { \ + flg=`printf '%s\n' "$$flg" | sed "s/$$1.*$$//"`; \ + }; \ + for flg in $$sane_makeflags; do \ + test $$skip_next = yes && { skip_next=no; continue; }; \ + case $$flg in \ + *=*|--*) continue;; \ + -*I) strip_trailopt 'I'; skip_next=yes;; \ + -*I?*) strip_trailopt 'I';; \ + -*O) strip_trailopt 'O'; skip_next=yes;; \ + -*O?*) strip_trailopt 'O';; \ + -*l) strip_trailopt 'l'; skip_next=yes;; \ + -*l?*) strip_trailopt 'l';; \ + -[dEDm]) skip_next=yes;; \ + -[JT]) skip_next=yes;; \ + esac; \ + case $$flg in \ + *$$target_option*) has_opt=yes; break;; \ + esac; \ + done; \ + test $$has_opt = yes +am__make_dryrun = (target_option=n; $(am__make_running_with_option)) +am__make_keepgoing = (target_option=k; $(am__make_running_with_option)) +pkgdatadir = $(datadir)/@PACKAGE@ +pkgincludedir = $(includedir)/@PACKAGE@ +pkglibdir = $(libdir)/@PACKAGE@ +pkglibexecdir = $(libexecdir)/@PACKAGE@ +am__cd = CDPATH="$${ZSH_VERSION+.}$(PATH_SEPARATOR)" && cd +install_sh_DATA = $(install_sh) -c -m 644 +install_sh_PROGRAM = $(install_sh) -c +install_sh_SCRIPT = $(install_sh) -c +INSTALL_HEADER = $(INSTALL_DATA) +transform = $(program_transform_name) +NORMAL_INSTALL = : +PRE_INSTALL = : +POST_INSTALL = : +NORMAL_UNINSTALL = : +PRE_UNINSTALL = : +POST_UNINSTALL = : +build_triplet = @build@ +host_triplet = @host@ + +# Internally, the dict methods yield via lua_yieldk() as implemented in Lua +# 5.3 and newer. +@DLUA_WITH_YIELDS_TRUE@am__append_1 = libdict_lua.la +@DLUA_WITH_YIELDS_FALSE@libdict_lua_la_DEPENDENCIES = +@DLUA_WITH_YIELDS_TRUE@am__append_2 = \ +@DLUA_WITH_YIELDS_TRUE@ dict-lua.h \ +@DLUA_WITH_YIELDS_TRUE@ dict-lua-private.h + +noinst_PROGRAMS = $(am__EXEEXT_1) test-dict-client$(EXEEXT) +subdir = src/lib-dict +ACLOCAL_M4 = $(top_srcdir)/aclocal.m4 +am__aclocal_m4_deps = $(top_srcdir)/m4/ac_checktype2.m4 \ + $(top_srcdir)/m4/ac_typeof.m4 $(top_srcdir)/m4/arc4random.m4 \ + $(top_srcdir)/m4/blockdev.m4 $(top_srcdir)/m4/c99_vsnprintf.m4 \ + $(top_srcdir)/m4/clock_gettime.m4 $(top_srcdir)/m4/crypt.m4 \ + $(top_srcdir)/m4/crypt_xpg6.m4 $(top_srcdir)/m4/dbqlk.m4 \ + $(top_srcdir)/m4/dirent_dtype.m4 $(top_srcdir)/m4/dovecot.m4 \ + $(top_srcdir)/m4/fd_passing.m4 $(top_srcdir)/m4/fdatasync.m4 \ + $(top_srcdir)/m4/flexible_array_member.m4 \ + $(top_srcdir)/m4/glibc.m4 $(top_srcdir)/m4/gmtime_max.m4 \ + $(top_srcdir)/m4/gmtime_tm_gmtoff.m4 \ + $(top_srcdir)/m4/ioloop.m4 $(top_srcdir)/m4/iovec.m4 \ + $(top_srcdir)/m4/ipv6.m4 $(top_srcdir)/m4/libcap.m4 \ + $(top_srcdir)/m4/libtool.m4 $(top_srcdir)/m4/libwrap.m4 \ + $(top_srcdir)/m4/linux_mremap.m4 $(top_srcdir)/m4/ltoptions.m4 \ + $(top_srcdir)/m4/ltsugar.m4 $(top_srcdir)/m4/ltversion.m4 \ + $(top_srcdir)/m4/lt~obsolete.m4 $(top_srcdir)/m4/mmap_write.m4 \ + $(top_srcdir)/m4/mntctl.m4 $(top_srcdir)/m4/modules.m4 \ + $(top_srcdir)/m4/notify.m4 $(top_srcdir)/m4/nsl.m4 \ + $(top_srcdir)/m4/off_t_max.m4 $(top_srcdir)/m4/pkg.m4 \ + $(top_srcdir)/m4/pr_set_dumpable.m4 \ + $(top_srcdir)/m4/q_quotactl.m4 $(top_srcdir)/m4/quota.m4 \ + $(top_srcdir)/m4/random.m4 $(top_srcdir)/m4/rlimit.m4 \ + $(top_srcdir)/m4/sendfile.m4 $(top_srcdir)/m4/size_t_signed.m4 \ + $(top_srcdir)/m4/sockpeercred.m4 $(top_srcdir)/m4/sql.m4 \ + $(top_srcdir)/m4/ssl.m4 $(top_srcdir)/m4/st_tim.m4 \ + $(top_srcdir)/m4/static_array.m4 $(top_srcdir)/m4/test_with.m4 \ + $(top_srcdir)/m4/time_t.m4 $(top_srcdir)/m4/typeof.m4 \ + $(top_srcdir)/m4/typeof_dev_t.m4 \ + $(top_srcdir)/m4/uoff_t_max.m4 $(top_srcdir)/m4/vararg.m4 \ + $(top_srcdir)/m4/want_apparmor.m4 \ + $(top_srcdir)/m4/want_bsdauth.m4 \ + $(top_srcdir)/m4/want_bzlib.m4 \ + $(top_srcdir)/m4/want_cassandra.m4 \ + $(top_srcdir)/m4/want_cdb.m4 \ + $(top_srcdir)/m4/want_checkpassword.m4 \ + $(top_srcdir)/m4/want_clucene.m4 $(top_srcdir)/m4/want_db.m4 \ + $(top_srcdir)/m4/want_gssapi.m4 $(top_srcdir)/m4/want_icu.m4 \ + $(top_srcdir)/m4/want_ldap.m4 $(top_srcdir)/m4/want_lua.m4 \ + $(top_srcdir)/m4/want_lz4.m4 $(top_srcdir)/m4/want_lzma.m4 \ + $(top_srcdir)/m4/want_mysql.m4 $(top_srcdir)/m4/want_pam.m4 \ + $(top_srcdir)/m4/want_passwd.m4 $(top_srcdir)/m4/want_pgsql.m4 \ + $(top_srcdir)/m4/want_prefetch.m4 \ + $(top_srcdir)/m4/want_shadow.m4 \ + $(top_srcdir)/m4/want_sodium.m4 $(top_srcdir)/m4/want_solr.m4 \ + $(top_srcdir)/m4/want_sqlite.m4 \ + $(top_srcdir)/m4/want_stemmer.m4 \ + $(top_srcdir)/m4/want_systemd.m4 \ + $(top_srcdir)/m4/want_textcat.m4 \ + $(top_srcdir)/m4/want_unwind.m4 $(top_srcdir)/m4/want_zlib.m4 \ + $(top_srcdir)/m4/want_zstd.m4 $(top_srcdir)/configure.ac +am__configure_deps = $(am__aclocal_m4_deps) $(CONFIGURE_DEPENDENCIES) \ + $(ACLOCAL_M4) +DIST_COMMON = $(srcdir)/Makefile.am $(am__pkginc_lib_HEADERS_DIST) \ + $(am__DIST_COMMON) +mkinstalldirs = $(install_sh) -d +CONFIG_HEADER = $(top_builddir)/config.h +CONFIG_CLEAN_FILES = +CONFIG_CLEAN_VPATH_FILES = +am__EXEEXT_1 = test-dict$(EXEEXT) +PROGRAMS = $(noinst_PROGRAMS) +LTLIBRARIES = $(noinst_LTLIBRARIES) +libdict_la_LIBADD = +am__objects_1 = dict.lo dict-client.lo dict-file.lo dict-memcached.lo \ + dict-memcached-ascii.lo dict-redis.lo dict-fail.lo \ + dict-transaction-memory.lo +am_libdict_la_OBJECTS = $(am__objects_1) +libdict_la_OBJECTS = $(am_libdict_la_OBJECTS) +AM_V_lt = $(am__v_lt_@AM_V@) +am__v_lt_ = $(am__v_lt_@AM_DEFAULT_V@) +am__v_lt_0 = --silent +am__v_lt_1 = +am__libdict_lua_la_SOURCES_DIST = dict-lua.c dict-iter-lua.c \ + dict-txn-lua.c +@DLUA_WITH_YIELDS_TRUE@am_libdict_lua_la_OBJECTS = \ +@DLUA_WITH_YIELDS_TRUE@ libdict_lua_la-dict-lua.lo \ +@DLUA_WITH_YIELDS_TRUE@ libdict_lua_la-dict-iter-lua.lo \ +@DLUA_WITH_YIELDS_TRUE@ libdict_lua_la-dict-txn-lua.lo +libdict_lua_la_OBJECTS = $(am_libdict_lua_la_OBJECTS) +@DLUA_WITH_YIELDS_TRUE@am_libdict_lua_la_rpath = +am_test_dict_OBJECTS = test-dict.$(OBJEXT) +test_dict_OBJECTS = $(am_test_dict_OBJECTS) +am_test_dict_client_OBJECTS = test-dict-client.$(OBJEXT) +test_dict_client_OBJECTS = $(am_test_dict_client_OBJECTS) +AM_V_P = $(am__v_P_@AM_V@) +am__v_P_ = $(am__v_P_@AM_DEFAULT_V@) +am__v_P_0 = false +am__v_P_1 = : +AM_V_GEN = $(am__v_GEN_@AM_V@) +am__v_GEN_ = $(am__v_GEN_@AM_DEFAULT_V@) +am__v_GEN_0 = @echo " GEN " $@; +am__v_GEN_1 = +AM_V_at = $(am__v_at_@AM_V@) +am__v_at_ = $(am__v_at_@AM_DEFAULT_V@) +am__v_at_0 = @ +am__v_at_1 = +DEFAULT_INCLUDES = -I.@am__isrc@ -I$(top_builddir) +depcomp = $(SHELL) $(top_srcdir)/depcomp +am__maybe_remake_depfiles = depfiles +am__depfiles_remade = ./$(DEPDIR)/dict-client.Plo \ + ./$(DEPDIR)/dict-fail.Plo ./$(DEPDIR)/dict-file.Plo \ + ./$(DEPDIR)/dict-memcached-ascii.Plo \ + ./$(DEPDIR)/dict-memcached.Plo ./$(DEPDIR)/dict-redis.Plo \ + ./$(DEPDIR)/dict-transaction-memory.Plo ./$(DEPDIR)/dict.Plo \ + ./$(DEPDIR)/libdict_lua_la-dict-iter-lua.Plo \ + ./$(DEPDIR)/libdict_lua_la-dict-lua.Plo \ + ./$(DEPDIR)/libdict_lua_la-dict-txn-lua.Plo \ + ./$(DEPDIR)/test-dict-client.Po ./$(DEPDIR)/test-dict.Po +am__mv = mv -f +COMPILE = $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) \ + $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) +LTCOMPILE = $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) \ + $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) \ + $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) \ + $(AM_CFLAGS) $(CFLAGS) +AM_V_CC = $(am__v_CC_@AM_V@) +am__v_CC_ = $(am__v_CC_@AM_DEFAULT_V@) +am__v_CC_0 = @echo " CC " $@; +am__v_CC_1 = +CCLD = $(CC) +LINK = $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) \ + $(LIBTOOLFLAGS) --mode=link $(CCLD) $(AM_CFLAGS) $(CFLAGS) \ + $(AM_LDFLAGS) $(LDFLAGS) -o $@ +AM_V_CCLD = $(am__v_CCLD_@AM_V@) +am__v_CCLD_ = $(am__v_CCLD_@AM_DEFAULT_V@) +am__v_CCLD_0 = @echo " CCLD " $@; +am__v_CCLD_1 = +SOURCES = $(libdict_la_SOURCES) $(libdict_lua_la_SOURCES) \ + $(test_dict_SOURCES) $(test_dict_client_SOURCES) +DIST_SOURCES = $(libdict_la_SOURCES) \ + $(am__libdict_lua_la_SOURCES_DIST) $(test_dict_SOURCES) \ + $(test_dict_client_SOURCES) +am__can_run_installinfo = \ + case $$AM_UPDATE_INFO_DIR in \ + n|no|NO) false;; \ + *) (install-info --version) >/dev/null 2>&1;; \ + esac +am__pkginc_lib_HEADERS_DIST = dict.h dict-client.h dict-private.h \ + dict-transaction-memory.h dict-lua.h dict-lua-private.h +am__vpath_adj_setup = srcdirstrip=`echo "$(srcdir)" | sed 's|.|.|g'`; +am__vpath_adj = case $$p in \ + $(srcdir)/*) f=`echo "$$p" | sed "s|^$$srcdirstrip/||"`;; \ + *) f=$$p;; \ + esac; +am__strip_dir = f=`echo $$p | sed -e 's|^.*/||'`; +am__install_max = 40 +am__nobase_strip_setup = \ + srcdirstrip=`echo "$(srcdir)" | sed 's/[].[^$$\\*|]/\\\\&/g'` +am__nobase_strip = \ + for p in $$list; do echo "$$p"; done | sed -e "s|$$srcdirstrip/||" +am__nobase_list = $(am__nobase_strip_setup); \ + for p in $$list; do echo "$$p $$p"; done | \ + sed "s| $$srcdirstrip/| |;"' / .*\//!s/ .*/ ./; s,\( .*\)/[^/]*$$,\1,' | \ + $(AWK) 'BEGIN { files["."] = "" } { files[$$2] = files[$$2] " " $$1; \ + if (++n[$$2] == $(am__install_max)) \ + { print $$2, files[$$2]; n[$$2] = 0; files[$$2] = "" } } \ + END { for (dir in files) print dir, files[dir] }' +am__base_list = \ + sed '$$!N;$$!N;$$!N;$$!N;$$!N;$$!N;$$!N;s/\n/ /g' | \ + sed '$$!N;$$!N;$$!N;$$!N;s/\n/ /g' +am__uninstall_files_from_dir = { \ + test -z "$$files" \ + || { test ! -d "$$dir" && test ! -f "$$dir" && test ! -r "$$dir"; } \ + || { echo " ( cd '$$dir' && rm -f" $$files ")"; \ + $(am__cd) "$$dir" && rm -f $$files; }; \ + } +am__installdirs = "$(DESTDIR)$(pkginc_libdir)" +HEADERS = $(pkginc_lib_HEADERS) +am__tagged_files = $(HEADERS) $(SOURCES) $(TAGS_FILES) $(LISP) +# Read a list of newline-separated strings from the standard input, +# and print each of them once, without duplicates. Input order is +# *not* preserved. +am__uniquify_input = $(AWK) '\ + BEGIN { nonempty = 0; } \ + { items[$$0] = 1; nonempty = 1; } \ + END { if (nonempty) { for (i in items) print i; }; } \ +' +# Make sure the list of sources is unique. This is necessary because, +# e.g., the same source file might be shared among _SOURCES variables +# for different programs/libraries. +am__define_uniq_tagged_files = \ + list='$(am__tagged_files)'; \ + unique=`for i in $$list; do \ + if test -f "$$i"; then echo $$i; else echo $(srcdir)/$$i; fi; \ + done | $(am__uniquify_input)` +ETAGS = etags +CTAGS = ctags +am__DIST_COMMON = $(srcdir)/Makefile.in $(top_srcdir)/depcomp +DISTFILES = $(DIST_COMMON) $(DIST_SOURCES) $(TEXINFOS) $(EXTRA_DIST) +ACLOCAL = @ACLOCAL@ +ACLOCAL_AMFLAGS = @ACLOCAL_AMFLAGS@ +AMTAR = @AMTAR@ +AM_DEFAULT_VERBOSITY = @AM_DEFAULT_VERBOSITY@ +APPARMOR_LIBS = @APPARMOR_LIBS@ +AR = @AR@ +AUTH_CFLAGS = @AUTH_CFLAGS@ +AUTH_LIBS = @AUTH_LIBS@ +AUTOCONF = @AUTOCONF@ +AUTOHEADER = @AUTOHEADER@ +AUTOMAKE = @AUTOMAKE@ +AWK = @AWK@ +BINARY_CFLAGS = @BINARY_CFLAGS@ +BINARY_LDFLAGS = @BINARY_LDFLAGS@ +BISON = @BISON@ +CASSANDRA_CFLAGS = @CASSANDRA_CFLAGS@ +CASSANDRA_LIBS = @CASSANDRA_LIBS@ +CC = @CC@ +CCDEPMODE = @CCDEPMODE@ +CDB_LIBS = @CDB_LIBS@ +CFLAGS = @CFLAGS@ +CLUCENE_CFLAGS = @CLUCENE_CFLAGS@ +CLUCENE_LIBS = @CLUCENE_LIBS@ +COMPRESS_LIBS = @COMPRESS_LIBS@ +CPP = @CPP@ +CPPFLAGS = @CPPFLAGS@ +CRYPT_LIBS = @CRYPT_LIBS@ +CXX = @CXX@ +CXXCPP = @CXXCPP@ +CXXDEPMODE = @CXXDEPMODE@ +CXXFLAGS = @CXXFLAGS@ +CYGPATH_W = @CYGPATH_W@ +DEFS = @DEFS@ +DEPDIR = @DEPDIR@ +DICT_LIBS = @DICT_LIBS@ +DLLIB = @DLLIB@ +DLLTOOL = @DLLTOOL@ +DSYMUTIL = @DSYMUTIL@ +DUMPBIN = @DUMPBIN@ +ECHO_C = @ECHO_C@ +ECHO_N = @ECHO_N@ +ECHO_T = @ECHO_T@ +EGREP = @EGREP@ +EXEEXT = @EXEEXT@ +FGREP = @FGREP@ +FLEX = @FLEX@ +FUZZER_CPPFLAGS = @FUZZER_CPPFLAGS@ +FUZZER_LDFLAGS = @FUZZER_LDFLAGS@ +GREP = @GREP@ +INSTALL = @INSTALL@ +INSTALL_DATA = @INSTALL_DATA@ +INSTALL_PROGRAM = @INSTALL_PROGRAM@ +INSTALL_SCRIPT = @INSTALL_SCRIPT@ +INSTALL_STRIP_PROGRAM = @INSTALL_STRIP_PROGRAM@ +KRB5CONFIG = @KRB5CONFIG@ +KRB5_CFLAGS = @KRB5_CFLAGS@ +KRB5_LIBS = @KRB5_LIBS@ +LD = @LD@ +LDAP_LIBS = @LDAP_LIBS@ +LDFLAGS = @LDFLAGS@ +LD_NO_WHOLE_ARCHIVE = @LD_NO_WHOLE_ARCHIVE@ +LD_WHOLE_ARCHIVE = @LD_WHOLE_ARCHIVE@ +LIBCAP = @LIBCAP@ +LIBDOVECOT = @LIBDOVECOT@ +LIBDOVECOT_COMPRESS = @LIBDOVECOT_COMPRESS@ +LIBDOVECOT_DEPS = @LIBDOVECOT_DEPS@ +LIBDOVECOT_DSYNC = @LIBDOVECOT_DSYNC@ +LIBDOVECOT_LA_LIBS = @LIBDOVECOT_LA_LIBS@ +LIBDOVECOT_LDA = @LIBDOVECOT_LDA@ +LIBDOVECOT_LDAP = @LIBDOVECOT_LDAP@ +LIBDOVECOT_LIBFTS = @LIBDOVECOT_LIBFTS@ +LIBDOVECOT_LIBFTS_DEPS = @LIBDOVECOT_LIBFTS_DEPS@ +LIBDOVECOT_LOGIN = @LIBDOVECOT_LOGIN@ +LIBDOVECOT_LUA = @LIBDOVECOT_LUA@ +LIBDOVECOT_LUA_DEPS = @LIBDOVECOT_LUA_DEPS@ +LIBDOVECOT_SQL = @LIBDOVECOT_SQL@ +LIBDOVECOT_STORAGE = @LIBDOVECOT_STORAGE@ +LIBDOVECOT_STORAGE_DEPS = @LIBDOVECOT_STORAGE_DEPS@ +LIBEXTTEXTCAT_CFLAGS = @LIBEXTTEXTCAT_CFLAGS@ +LIBEXTTEXTCAT_LIBS = @LIBEXTTEXTCAT_LIBS@ +LIBICONV = @LIBICONV@ +LIBICU_CFLAGS = @LIBICU_CFLAGS@ +LIBICU_LIBS = @LIBICU_LIBS@ +LIBOBJS = @LIBOBJS@ +LIBS = @LIBS@ +LIBSODIUM_CFLAGS = @LIBSODIUM_CFLAGS@ +LIBSODIUM_LIBS = @LIBSODIUM_LIBS@ +LIBTIRPC_CFLAGS = @LIBTIRPC_CFLAGS@ +LIBTIRPC_LIBS = @LIBTIRPC_LIBS@ +LIBTOOL = @LIBTOOL@ +LIBUNWIND_CFLAGS = @LIBUNWIND_CFLAGS@ +LIBUNWIND_LIBS = @LIBUNWIND_LIBS@ +LIBWRAP_LIBS = @LIBWRAP_LIBS@ +LINKED_STORAGE_LDADD = @LINKED_STORAGE_LDADD@ +LIPO = @LIPO@ +LN_S = @LN_S@ +LTLIBICONV = @LTLIBICONV@ +LTLIBOBJS = @LTLIBOBJS@ +LT_SYS_LIBRARY_PATH = @LT_SYS_LIBRARY_PATH@ +LUA_CFLAGS = @LUA_CFLAGS@ +LUA_LIBS = @LUA_LIBS@ +MAINT = @MAINT@ +MAKEINFO = @MAKEINFO@ +MANIFEST_TOOL = @MANIFEST_TOOL@ +MKDIR_P = @MKDIR_P@ +MODULE_LIBS = @MODULE_LIBS@ +MODULE_SUFFIX = @MODULE_SUFFIX@ +MYSQL_CFLAGS = @MYSQL_CFLAGS@ +MYSQL_CONFIG = @MYSQL_CONFIG@ +MYSQL_LIBS = @MYSQL_LIBS@ +NM = @NM@ +NMEDIT = @NMEDIT@ +NOPLUGIN_LDFLAGS = @NOPLUGIN_LDFLAGS@ +OBJDUMP = @OBJDUMP@ +OBJEXT = @OBJEXT@ +OTOOL = @OTOOL@ +OTOOL64 = @OTOOL64@ +PACKAGE = @PACKAGE@ +PACKAGE_BUGREPORT = @PACKAGE_BUGREPORT@ +PACKAGE_NAME = @PACKAGE_NAME@ +PACKAGE_STRING = @PACKAGE_STRING@ +PACKAGE_TARNAME = @PACKAGE_TARNAME@ +PACKAGE_URL = @PACKAGE_URL@ +PACKAGE_VERSION = @PACKAGE_VERSION@ +PANDOC = @PANDOC@ +PATH_SEPARATOR = @PATH_SEPARATOR@ +PGSQL_CFLAGS = @PGSQL_CFLAGS@ +PGSQL_LIBS = @PGSQL_LIBS@ +PG_CONFIG = @PG_CONFIG@ +PIE_CFLAGS = @PIE_CFLAGS@ +PIE_LDFLAGS = @PIE_LDFLAGS@ +PKG_CONFIG = @PKG_CONFIG@ +PKG_CONFIG_LIBDIR = @PKG_CONFIG_LIBDIR@ +PKG_CONFIG_PATH = @PKG_CONFIG_PATH@ +QUOTA_LIBS = @QUOTA_LIBS@ +RANLIB = @RANLIB@ +RELRO_LDFLAGS = @RELRO_LDFLAGS@ +RPCGEN = @RPCGEN@ +RUN_TEST = @RUN_TEST@ +SED = @SED@ +SETTING_FILES = @SETTING_FILES@ +SET_MAKE = @SET_MAKE@ +SHELL = @SHELL@ +SQLITE_CFLAGS = @SQLITE_CFLAGS@ +SQLITE_LIBS = @SQLITE_LIBS@ +SQL_CFLAGS = @SQL_CFLAGS@ +SQL_LIBS = @SQL_LIBS@ +SSL_CFLAGS = @SSL_CFLAGS@ +SSL_LIBS = @SSL_LIBS@ +STRIP = @STRIP@ +SYSTEMD_CFLAGS = @SYSTEMD_CFLAGS@ +SYSTEMD_LIBS = @SYSTEMD_LIBS@ +VALGRIND = @VALGRIND@ +VERSION = @VERSION@ +ZSTD_CFLAGS = @ZSTD_CFLAGS@ +ZSTD_LIBS = @ZSTD_LIBS@ +abs_builddir = @abs_builddir@ +abs_srcdir = @abs_srcdir@ +abs_top_builddir = @abs_top_builddir@ +abs_top_srcdir = @abs_top_srcdir@ +ac_ct_AR = @ac_ct_AR@ +ac_ct_CC = @ac_ct_CC@ +ac_ct_CXX = @ac_ct_CXX@ +ac_ct_DUMPBIN = @ac_ct_DUMPBIN@ +am__include = @am__include@ +am__leading_dot = @am__leading_dot@ +am__quote = @am__quote@ +am__tar = @am__tar@ +am__untar = @am__untar@ +bindir = @bindir@ +build = @build@ +build_alias = @build_alias@ +build_cpu = @build_cpu@ +build_os = @build_os@ +build_vendor = @build_vendor@ +builddir = @builddir@ +datadir = @datadir@ +datarootdir = @datarootdir@ +dict_drivers = @dict_drivers@ +docdir = @docdir@ +dvidir = @dvidir@ +exec_prefix = @exec_prefix@ +host = @host@ +host_alias = @host_alias@ +host_cpu = @host_cpu@ +host_os = @host_os@ +host_vendor = @host_vendor@ +htmldir = @htmldir@ +includedir = @includedir@ +infodir = @infodir@ +install_sh = @install_sh@ +libdir = @libdir@ +libexecdir = @libexecdir@ +localedir = @localedir@ +localstatedir = @localstatedir@ +mandir = @mandir@ +mkdir_p = @mkdir_p@ +moduledir = @moduledir@ +oldincludedir = @oldincludedir@ +pdfdir = @pdfdir@ +prefix = @prefix@ +program_transform_name = @program_transform_name@ +psdir = @psdir@ +rundir = @rundir@ +runstatedir = @runstatedir@ +sbindir = @sbindir@ +sharedstatedir = @sharedstatedir@ +sql_drivers = @sql_drivers@ +srcdir = @srcdir@ +ssldir = @ssldir@ +statedir = @statedir@ +sysconfdir = @sysconfdir@ +systemdservicetype = @systemdservicetype@ +systemdsystemunitdir = @systemdsystemunitdir@ +target_alias = @target_alias@ +top_build_prefix = @top_build_prefix@ +top_builddir = @top_builddir@ +top_srcdir = @top_srcdir@ +noinst_LTLIBRARIES = libdict.la $(am__append_1) +AM_CPPFLAGS = \ + -I$(top_srcdir)/src/lib \ + -I$(top_srcdir)/src/lib-test \ + -I$(top_srcdir)/src/lib-settings + +base_sources = \ + dict.c \ + dict-client.c \ + dict-file.c \ + dict-memcached.c \ + dict-memcached-ascii.c \ + dict-redis.c \ + dict-fail.c \ + dict-transaction-memory.c + +libdict_la_SOURCES = \ + $(base_sources) + +headers = dict.h dict-client.h dict-private.h \ + dict-transaction-memory.h $(am__append_2) +@DLUA_WITH_YIELDS_TRUE@libdict_lua_la_SOURCES = \ +@DLUA_WITH_YIELDS_TRUE@ dict-lua.c \ +@DLUA_WITH_YIELDS_TRUE@ dict-iter-lua.c \ +@DLUA_WITH_YIELDS_TRUE@ dict-txn-lua.c + +@DLUA_WITH_YIELDS_TRUE@libdict_lua_la_CPPFLAGS = \ +@DLUA_WITH_YIELDS_TRUE@ $(AM_CPPFLAGS) \ +@DLUA_WITH_YIELDS_TRUE@ $(LUA_CFLAGS) \ +@DLUA_WITH_YIELDS_TRUE@ -I$(top_srcdir)/src/lib-lua + +@DLUA_WITH_YIELDS_TRUE@libdict_lua_la_LIBADD = +@DLUA_WITH_YIELDS_TRUE@libdict_lua_la_DEPENDENCIES = \ +@DLUA_WITH_YIELDS_TRUE@ libdict.la + +pkginc_libdir = $(pkgincludedir) +pkginc_lib_HEADERS = $(headers) +test_programs = \ + test-dict + +test_libs = \ + ../lib-test/libtest.la \ + ../lib/liblib.la + +test_dict_SOURCES = test-dict.c +test_dict_LDADD = libdict.la $(test_libs) +test_dict_DEPENDENCIES = $(noinst_LTLIBRARIES) $(test_libs) +test_dict_client_SOURCES = test-dict-client.c +test_dict_client_LDADD = $(noinst_LTLIBRARIES) ../lib/liblib.la +test_dict_client_DEPENDENCIES = $(noinst_LTLIBRARIES) $(test_libs) +all: all-am + +.SUFFIXES: +.SUFFIXES: .c .lo .o .obj +$(srcdir)/Makefile.in: @MAINTAINER_MODE_TRUE@ $(srcdir)/Makefile.am $(am__configure_deps) + @for dep in $?; do \ + case '$(am__configure_deps)' in \ + *$$dep*) \ + ( cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh ) \ + && { if test -f $@; then exit 0; else break; fi; }; \ + exit 1;; \ + esac; \ + done; \ + echo ' cd $(top_srcdir) && $(AUTOMAKE) --foreign src/lib-dict/Makefile'; \ + $(am__cd) $(top_srcdir) && \ + $(AUTOMAKE) --foreign src/lib-dict/Makefile +Makefile: $(srcdir)/Makefile.in $(top_builddir)/config.status + @case '$?' in \ + *config.status*) \ + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh;; \ + *) \ + echo ' cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__maybe_remake_depfiles)'; \ + cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__maybe_remake_depfiles);; \ + esac; + +$(top_builddir)/config.status: $(top_srcdir)/configure $(CONFIG_STATUS_DEPENDENCIES) + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh + +$(top_srcdir)/configure: @MAINTAINER_MODE_TRUE@ $(am__configure_deps) + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh +$(ACLOCAL_M4): @MAINTAINER_MODE_TRUE@ $(am__aclocal_m4_deps) + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh +$(am__aclocal_m4_deps): + +clean-noinstPROGRAMS: + @list='$(noinst_PROGRAMS)'; test -n "$$list" || exit 0; \ + echo " rm -f" $$list; \ + rm -f $$list || exit $$?; \ + test -n "$(EXEEXT)" || exit 0; \ + list=`for p in $$list; do echo "$$p"; done | sed 's/$(EXEEXT)$$//'`; \ + echo " rm -f" $$list; \ + rm -f $$list + +clean-noinstLTLIBRARIES: + -test -z "$(noinst_LTLIBRARIES)" || rm -f $(noinst_LTLIBRARIES) + @list='$(noinst_LTLIBRARIES)'; \ + locs=`for p in $$list; do echo $$p; done | \ + sed 's|^[^/]*$$|.|; s|/[^/]*$$||; s|$$|/so_locations|' | \ + sort -u`; \ + test -z "$$locs" || { \ + echo rm -f $${locs}; \ + rm -f $${locs}; \ + } + +libdict.la: $(libdict_la_OBJECTS) $(libdict_la_DEPENDENCIES) $(EXTRA_libdict_la_DEPENDENCIES) + $(AM_V_CCLD)$(LINK) $(libdict_la_OBJECTS) $(libdict_la_LIBADD) $(LIBS) + +libdict_lua.la: $(libdict_lua_la_OBJECTS) $(libdict_lua_la_DEPENDENCIES) $(EXTRA_libdict_lua_la_DEPENDENCIES) + $(AM_V_CCLD)$(LINK) $(am_libdict_lua_la_rpath) $(libdict_lua_la_OBJECTS) $(libdict_lua_la_LIBADD) $(LIBS) + +test-dict$(EXEEXT): $(test_dict_OBJECTS) $(test_dict_DEPENDENCIES) $(EXTRA_test_dict_DEPENDENCIES) + @rm -f test-dict$(EXEEXT) + $(AM_V_CCLD)$(LINK) $(test_dict_OBJECTS) $(test_dict_LDADD) $(LIBS) + +test-dict-client$(EXEEXT): $(test_dict_client_OBJECTS) $(test_dict_client_DEPENDENCIES) $(EXTRA_test_dict_client_DEPENDENCIES) + @rm -f test-dict-client$(EXEEXT) + $(AM_V_CCLD)$(LINK) $(test_dict_client_OBJECTS) $(test_dict_client_LDADD) $(LIBS) + +mostlyclean-compile: + -rm -f *.$(OBJEXT) + +distclean-compile: + -rm -f *.tab.c + +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/dict-client.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/dict-fail.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/dict-file.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/dict-memcached-ascii.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/dict-memcached.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/dict-redis.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/dict-transaction-memory.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/dict.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libdict_lua_la-dict-iter-lua.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libdict_lua_la-dict-lua.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libdict_lua_la-dict-txn-lua.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/test-dict-client.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/test-dict.Po@am__quote@ # am--include-marker + +$(am__depfiles_remade): + @$(MKDIR_P) $(@D) + @echo '# dummy' >$@-t && $(am__mv) $@-t $@ + +am--depfiles: $(am__depfiles_remade) + +.c.o: +@am__fastdepCC_TRUE@ $(AM_V_CC)$(COMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ $< +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='$<' object='$@' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(COMPILE) -c -o $@ $< + +.c.obj: +@am__fastdepCC_TRUE@ $(AM_V_CC)$(COMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ `$(CYGPATH_W) '$<'` +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='$<' object='$@' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(COMPILE) -c -o $@ `$(CYGPATH_W) '$<'` + +.c.lo: +@am__fastdepCC_TRUE@ $(AM_V_CC)$(LTCOMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ $< +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Plo +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='$<' object='$@' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(LTCOMPILE) -c -o $@ $< + +libdict_lua_la-dict-lua.lo: dict-lua.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libdict_lua_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT libdict_lua_la-dict-lua.lo -MD -MP -MF $(DEPDIR)/libdict_lua_la-dict-lua.Tpo -c -o libdict_lua_la-dict-lua.lo `test -f 'dict-lua.c' || echo '$(srcdir)/'`dict-lua.c +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/libdict_lua_la-dict-lua.Tpo $(DEPDIR)/libdict_lua_la-dict-lua.Plo +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='dict-lua.c' object='libdict_lua_la-dict-lua.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libdict_lua_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o libdict_lua_la-dict-lua.lo `test -f 'dict-lua.c' || echo '$(srcdir)/'`dict-lua.c + +libdict_lua_la-dict-iter-lua.lo: dict-iter-lua.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libdict_lua_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT libdict_lua_la-dict-iter-lua.lo -MD -MP -MF $(DEPDIR)/libdict_lua_la-dict-iter-lua.Tpo -c -o libdict_lua_la-dict-iter-lua.lo `test -f 'dict-iter-lua.c' || echo '$(srcdir)/'`dict-iter-lua.c +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/libdict_lua_la-dict-iter-lua.Tpo $(DEPDIR)/libdict_lua_la-dict-iter-lua.Plo +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='dict-iter-lua.c' object='libdict_lua_la-dict-iter-lua.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libdict_lua_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o libdict_lua_la-dict-iter-lua.lo `test -f 'dict-iter-lua.c' || echo '$(srcdir)/'`dict-iter-lua.c + +libdict_lua_la-dict-txn-lua.lo: dict-txn-lua.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libdict_lua_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT libdict_lua_la-dict-txn-lua.lo -MD -MP -MF $(DEPDIR)/libdict_lua_la-dict-txn-lua.Tpo -c -o libdict_lua_la-dict-txn-lua.lo `test -f 'dict-txn-lua.c' || echo '$(srcdir)/'`dict-txn-lua.c +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/libdict_lua_la-dict-txn-lua.Tpo $(DEPDIR)/libdict_lua_la-dict-txn-lua.Plo +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='dict-txn-lua.c' object='libdict_lua_la-dict-txn-lua.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libdict_lua_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o libdict_lua_la-dict-txn-lua.lo `test -f 'dict-txn-lua.c' || echo '$(srcdir)/'`dict-txn-lua.c + +mostlyclean-libtool: + -rm -f *.lo + +clean-libtool: + -rm -rf .libs _libs +install-pkginc_libHEADERS: $(pkginc_lib_HEADERS) + @$(NORMAL_INSTALL) + @list='$(pkginc_lib_HEADERS)'; test -n "$(pkginc_libdir)" || list=; \ + if test -n "$$list"; then \ + echo " $(MKDIR_P) '$(DESTDIR)$(pkginc_libdir)'"; \ + $(MKDIR_P) "$(DESTDIR)$(pkginc_libdir)" || exit 1; \ + fi; \ + for p in $$list; do \ + if test -f "$$p"; then d=; else d="$(srcdir)/"; fi; \ + echo "$$d$$p"; \ + done | $(am__base_list) | \ + while read files; do \ + echo " $(INSTALL_HEADER) $$files '$(DESTDIR)$(pkginc_libdir)'"; \ + $(INSTALL_HEADER) $$files "$(DESTDIR)$(pkginc_libdir)" || exit $$?; \ + done + +uninstall-pkginc_libHEADERS: + @$(NORMAL_UNINSTALL) + @list='$(pkginc_lib_HEADERS)'; test -n "$(pkginc_libdir)" || list=; \ + files=`for p in $$list; do echo $$p; done | sed -e 's|^.*/||'`; \ + dir='$(DESTDIR)$(pkginc_libdir)'; $(am__uninstall_files_from_dir) + +ID: $(am__tagged_files) + $(am__define_uniq_tagged_files); mkid -fID $$unique +tags: tags-am +TAGS: tags + +tags-am: $(TAGS_DEPENDENCIES) $(am__tagged_files) + set x; \ + here=`pwd`; \ + $(am__define_uniq_tagged_files); \ + shift; \ + if test -z "$(ETAGS_ARGS)$$*$$unique"; then :; else \ + test -n "$$unique" || unique=$$empty_fix; \ + if test $$# -gt 0; then \ + $(ETAGS) $(ETAGSFLAGS) $(AM_ETAGSFLAGS) $(ETAGS_ARGS) \ + "$$@" $$unique; \ + else \ + $(ETAGS) $(ETAGSFLAGS) $(AM_ETAGSFLAGS) $(ETAGS_ARGS) \ + $$unique; \ + fi; \ + fi +ctags: ctags-am + +CTAGS: ctags +ctags-am: $(TAGS_DEPENDENCIES) $(am__tagged_files) + $(am__define_uniq_tagged_files); \ + test -z "$(CTAGS_ARGS)$$unique" \ + || $(CTAGS) $(CTAGSFLAGS) $(AM_CTAGSFLAGS) $(CTAGS_ARGS) \ + $$unique + +GTAGS: + here=`$(am__cd) $(top_builddir) && pwd` \ + && $(am__cd) $(top_srcdir) \ + && gtags -i $(GTAGS_ARGS) "$$here" +cscopelist: cscopelist-am + +cscopelist-am: $(am__tagged_files) + list='$(am__tagged_files)'; \ + case "$(srcdir)" in \ + [\\/]* | ?:[\\/]*) sdir="$(srcdir)" ;; \ + *) sdir=$(subdir)/$(srcdir) ;; \ + esac; \ + for i in $$list; do \ + if test -f "$$i"; then \ + echo "$(subdir)/$$i"; \ + else \ + echo "$$sdir/$$i"; \ + fi; \ + done >> $(top_builddir)/cscope.files + +distclean-tags: + -rm -f TAGS ID GTAGS GRTAGS GSYMS GPATH tags + +distdir: $(BUILT_SOURCES) + $(MAKE) $(AM_MAKEFLAGS) distdir-am + +distdir-am: $(DISTFILES) + @srcdirstrip=`echo "$(srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \ + topsrcdirstrip=`echo "$(top_srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \ + list='$(DISTFILES)'; \ + dist_files=`for file in $$list; do echo $$file; done | \ + sed -e "s|^$$srcdirstrip/||;t" \ + -e "s|^$$topsrcdirstrip/|$(top_builddir)/|;t"`; \ + case $$dist_files in \ + */*) $(MKDIR_P) `echo "$$dist_files" | \ + sed '/\//!d;s|^|$(distdir)/|;s,/[^/]*$$,,' | \ + sort -u` ;; \ + esac; \ + for file in $$dist_files; do \ + if test -f $$file || test -d $$file; then d=.; else d=$(srcdir); fi; \ + if test -d $$d/$$file; then \ + dir=`echo "/$$file" | sed -e 's,/[^/]*$$,,'`; \ + if test -d "$(distdir)/$$file"; then \ + find "$(distdir)/$$file" -type d ! -perm -700 -exec chmod u+rwx {} \;; \ + fi; \ + if test -d $(srcdir)/$$file && test $$d != $(srcdir); then \ + cp -fpR $(srcdir)/$$file "$(distdir)$$dir" || exit 1; \ + find "$(distdir)/$$file" -type d ! -perm -700 -exec chmod u+rwx {} \;; \ + fi; \ + cp -fpR $$d/$$file "$(distdir)$$dir" || exit 1; \ + else \ + test -f "$(distdir)/$$file" \ + || cp -p $$d/$$file "$(distdir)/$$file" \ + || exit 1; \ + fi; \ + done +check-am: all-am + $(MAKE) $(AM_MAKEFLAGS) check-local +check: check-am +all-am: Makefile $(PROGRAMS) $(LTLIBRARIES) $(HEADERS) +installdirs: + for dir in "$(DESTDIR)$(pkginc_libdir)"; do \ + test -z "$$dir" || $(MKDIR_P) "$$dir"; \ + done +install: install-am +install-exec: install-exec-am +install-data: install-data-am +uninstall: uninstall-am + +install-am: all-am + @$(MAKE) $(AM_MAKEFLAGS) install-exec-am install-data-am + +installcheck: installcheck-am +install-strip: + if test -z '$(STRIP)'; then \ + $(MAKE) $(AM_MAKEFLAGS) INSTALL_PROGRAM="$(INSTALL_STRIP_PROGRAM)" \ + install_sh_PROGRAM="$(INSTALL_STRIP_PROGRAM)" INSTALL_STRIP_FLAG=-s \ + install; \ + else \ + $(MAKE) $(AM_MAKEFLAGS) INSTALL_PROGRAM="$(INSTALL_STRIP_PROGRAM)" \ + install_sh_PROGRAM="$(INSTALL_STRIP_PROGRAM)" INSTALL_STRIP_FLAG=-s \ + "INSTALL_PROGRAM_ENV=STRIPPROG='$(STRIP)'" install; \ + fi +mostlyclean-generic: + +clean-generic: + +distclean-generic: + -test -z "$(CONFIG_CLEAN_FILES)" || rm -f $(CONFIG_CLEAN_FILES) + -test . = "$(srcdir)" || test -z "$(CONFIG_CLEAN_VPATH_FILES)" || rm -f $(CONFIG_CLEAN_VPATH_FILES) + +maintainer-clean-generic: + @echo "This command is intended for maintainers to use" + @echo "it deletes files that may require special tools to rebuild." +clean: clean-am + +clean-am: clean-generic clean-libtool clean-noinstLTLIBRARIES \ + clean-noinstPROGRAMS mostlyclean-am + +distclean: distclean-am + -rm -f ./$(DEPDIR)/dict-client.Plo + -rm -f ./$(DEPDIR)/dict-fail.Plo + -rm -f ./$(DEPDIR)/dict-file.Plo + -rm -f ./$(DEPDIR)/dict-memcached-ascii.Plo + -rm -f ./$(DEPDIR)/dict-memcached.Plo + -rm -f ./$(DEPDIR)/dict-redis.Plo + -rm -f ./$(DEPDIR)/dict-transaction-memory.Plo + -rm -f ./$(DEPDIR)/dict.Plo + -rm -f ./$(DEPDIR)/libdict_lua_la-dict-iter-lua.Plo + -rm -f ./$(DEPDIR)/libdict_lua_la-dict-lua.Plo + -rm -f ./$(DEPDIR)/libdict_lua_la-dict-txn-lua.Plo + -rm -f ./$(DEPDIR)/test-dict-client.Po + -rm -f ./$(DEPDIR)/test-dict.Po + -rm -f Makefile +distclean-am: clean-am distclean-compile distclean-generic \ + distclean-tags + +dvi: dvi-am + +dvi-am: + +html: html-am + +html-am: + +info: info-am + +info-am: + +install-data-am: install-pkginc_libHEADERS + +install-dvi: install-dvi-am + +install-dvi-am: + +install-exec-am: + +install-html: install-html-am + +install-html-am: + +install-info: install-info-am + +install-info-am: + +install-man: + +install-pdf: install-pdf-am + +install-pdf-am: + +install-ps: install-ps-am + +install-ps-am: + +installcheck-am: + +maintainer-clean: maintainer-clean-am + -rm -f ./$(DEPDIR)/dict-client.Plo + -rm -f ./$(DEPDIR)/dict-fail.Plo + -rm -f ./$(DEPDIR)/dict-file.Plo + -rm -f ./$(DEPDIR)/dict-memcached-ascii.Plo + -rm -f ./$(DEPDIR)/dict-memcached.Plo + -rm -f ./$(DEPDIR)/dict-redis.Plo + -rm -f ./$(DEPDIR)/dict-transaction-memory.Plo + -rm -f ./$(DEPDIR)/dict.Plo + -rm -f ./$(DEPDIR)/libdict_lua_la-dict-iter-lua.Plo + -rm -f ./$(DEPDIR)/libdict_lua_la-dict-lua.Plo + -rm -f ./$(DEPDIR)/libdict_lua_la-dict-txn-lua.Plo + -rm -f ./$(DEPDIR)/test-dict-client.Po + -rm -f ./$(DEPDIR)/test-dict.Po + -rm -f Makefile +maintainer-clean-am: distclean-am maintainer-clean-generic + +mostlyclean: mostlyclean-am + +mostlyclean-am: mostlyclean-compile mostlyclean-generic \ + mostlyclean-libtool + +pdf: pdf-am + +pdf-am: + +ps: ps-am + +ps-am: + +uninstall-am: uninstall-pkginc_libHEADERS + +.MAKE: check-am install-am install-strip + +.PHONY: CTAGS GTAGS TAGS all all-am am--depfiles check check-am \ + check-local clean clean-generic clean-libtool \ + clean-noinstLTLIBRARIES clean-noinstPROGRAMS cscopelist-am \ + ctags ctags-am distclean distclean-compile distclean-generic \ + distclean-libtool distclean-tags distdir dvi dvi-am html \ + html-am info info-am install install-am install-data \ + install-data-am install-dvi install-dvi-am install-exec \ + install-exec-am install-html install-html-am install-info \ + install-info-am install-man install-pdf install-pdf-am \ + install-pkginc_libHEADERS install-ps install-ps-am \ + install-strip installcheck installcheck-am installdirs \ + maintainer-clean maintainer-clean-generic mostlyclean \ + mostlyclean-compile mostlyclean-generic mostlyclean-libtool \ + pdf pdf-am ps ps-am tags tags-am uninstall uninstall-am \ + uninstall-pkginc_libHEADERS + +.PRECIOUS: Makefile + + +check-local: + for bin in $(test_programs) $(check_PROGRAMS); do \ + if ! $(RUN_TEST) ./$$bin; then exit 1; fi; \ + done + +# Tell versions [3.59,3.63) of GNU make to not export all variables. +# Otherwise a system limit (for SysV at least) may be exceeded. +.NOEXPORT: diff --git a/src/lib-dict/dict-client.c b/src/lib-dict/dict-client.c new file mode 100644 index 0000000..2c688ef --- /dev/null +++ b/src/lib-dict/dict-client.c @@ -0,0 +1,1508 @@ +/* Copyright (c) 2005-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "llist.h" +#include "str.h" +#include "strescape.h" +#include "file-lock.h" +#include "time-util.h" +#include "connection.h" +#include "ostream.h" +#include "eacces-error.h" +#include "dict-private.h" +#include "dict-client.h" + +#include <unistd.h> +#include <fcntl.h> + +/* Disconnect from dict server after this many milliseconds of idling after + sending a command. Because dict server does blocking dict accesses, it can + handle only one client at a time. This is why the default timeout is zero, + so that there won't be many dict processes just doing nothing. Zero means + that the socket is disconnected immediately after returning to ioloop. */ +#define DICT_CLIENT_DEFAULT_TIMEOUT_MSECS 0 + +/* Abort dict lookup after this many seconds. */ +#define DICT_CLIENT_REQUEST_TIMEOUT_MSECS 30000 +/* When dict lookup timeout is reached, wait a bit longer if the last dict + ioloop wait was shorter than this. */ +#define DICT_CLIENT_REQUEST_TIMEOUT_MIN_LAST_IOLOOP_WAIT_MSECS 1000 +/* Log a warning if dict lookup takes longer than this many milliseconds. */ +#define DICT_CLIENT_DEFAULT_WARN_SLOW_MSECS 5000 + +struct client_dict_cmd { + int refcount; + struct client_dict *dict; + struct timeval start_time; + char *query; + unsigned int async_id; + struct timeval async_id_received_time; + + uint64_t start_global_ioloop_usecs; + uint64_t start_dict_ioloop_usecs; + uint64_t start_lock_usecs; + + bool reconnected; + bool retry_errors; + bool no_replies; + bool unfinished; + bool background; + + void (*callback)(struct client_dict_cmd *cmd, + enum dict_protocol_reply reply, const char *value, + const char *const *extra_args, const char *error, + bool disconnected); + struct client_dict_iterate_context *iter; + struct client_dict_transaction_context *trans; + + struct { + dict_lookup_callback_t *lookup; + dict_transaction_commit_callback_t *commit; + void *context; + } api_callback; +}; + +struct dict_client_connection { + struct connection conn; + struct client_dict *dict; +}; + +struct client_dict { + struct dict dict; + struct dict_client_connection conn; + + char *uri; + enum dict_data_type value_type; + unsigned warn_slow_msecs; + + time_t last_failed_connect; + char *last_connect_error; + + struct io_wait_timer *wait_timer; + uint64_t last_timer_switch_usecs; + struct timeout *to_requests; + struct timeout *to_idle; + unsigned int idle_msecs; + + ARRAY(struct client_dict_cmd *) cmds; + struct client_dict_transaction_context *transactions; + + unsigned int transaction_id_counter; +}; + +struct client_dict_iter_result { + const char *key, *const *values; +}; + +struct client_dict_iterate_context { + struct dict_iterate_context ctx; + char *error; + char *path; + enum dict_iterate_flags flags; + int refcount; + + pool_t results_pool; + ARRAY(struct client_dict_iter_result) results; + unsigned int result_idx; + + bool cmd_sent; + bool seen_results; + bool finished; + bool deinit; +}; + +struct client_dict_transaction_context { + struct dict_transaction_context ctx; + struct client_dict_transaction_context *prev, *next; + + char *first_query; + char *error; + + unsigned int id; + unsigned int query_count; + + bool sent_begin:1; +}; + +static struct connection_list *dict_connections; + +static int client_dict_connect(struct client_dict *dict, const char **error_r); +static int client_dict_reconnect(struct client_dict *dict, const char *reason, + const char **error_r); +static void client_dict_disconnect(struct client_dict *dict, const char *reason); +static const char *dict_wait_warnings(const struct client_dict_cmd *cmd); + +static struct client_dict_cmd * +client_dict_cmd_init(struct client_dict *dict, const char *query) +{ + struct client_dict_cmd *cmd; + + io_loop_time_refresh(); + + cmd = i_new(struct client_dict_cmd, 1); + cmd->refcount = 1; + cmd->dict = dict; + cmd->query = i_strdup(query); + cmd->start_time = ioloop_timeval; + cmd->start_global_ioloop_usecs = ioloop_global_wait_usecs; + cmd->start_dict_ioloop_usecs = io_wait_timer_get_usecs(dict->wait_timer); + cmd->start_lock_usecs = file_lock_wait_get_total_usecs(); + return cmd; +} + +static void client_dict_cmd_ref(struct client_dict_cmd *cmd) +{ + i_assert(cmd->refcount > 0); + cmd->refcount++; +} + +static bool client_dict_cmd_unref(struct client_dict_cmd *cmd) +{ + i_assert(cmd->refcount > 0); + if (--cmd->refcount > 0) + return TRUE; + + i_assert(cmd->trans == NULL); + + i_free(cmd->query); + i_free(cmd); + return FALSE; +} + +static bool +dict_cmd_callback_line(struct client_dict_cmd *cmd, const char *const *args) +{ + const char *value = args[0]; + enum dict_protocol_reply reply; + + if (value == NULL) { + /* "" is a valid iteration reply */ + reply = DICT_PROTOCOL_REPLY_ITER_FINISHED; + } else { + reply = value[0]; + value++; + args++; + } + + cmd->unfinished = FALSE; + cmd->callback(cmd, reply, value, args, NULL, FALSE); + return !cmd->unfinished; +} + +static void +dict_cmd_callback_error(struct client_dict_cmd *cmd, const char *error, + bool disconnected) +{ + const char *null_arg = NULL; + + cmd->unfinished = FALSE; + if (cmd->callback != NULL) { + cmd->callback(cmd, DICT_PROTOCOL_REPLY_ERROR, + "", &null_arg, error, disconnected); + } + i_assert(!cmd->unfinished); +} + +static struct client_dict_cmd * +client_dict_cmd_first_nonbg(struct client_dict *dict) +{ + struct client_dict_cmd *const *cmds; + unsigned int i, count; + + cmds = array_get(&dict->cmds, &count); + for (i = 0; i < count; i++) { + if (!cmds[i]->background) + return cmds[i]; + } + return NULL; +} + +static void client_dict_input_timeout(struct client_dict *dict) +{ + struct client_dict_cmd *cmd; + const char *error; + uint64_t msecs_in_last_dict_ioloop_wait; + int cmd_diff; + + /* find the first non-background command. there must be at least one. */ + cmd = client_dict_cmd_first_nonbg(dict); + i_assert(cmd != NULL); + + cmd_diff = timeval_diff_msecs(&ioloop_timeval, &cmd->start_time); + if (cmd_diff < DICT_CLIENT_REQUEST_TIMEOUT_MSECS) { + /* need to re-create this timeout. the currently-oldest + command was added when another command was still + running with an older timeout. */ + timeout_remove(&dict->to_requests); + dict->to_requests = + timeout_add(DICT_CLIENT_REQUEST_TIMEOUT_MSECS - cmd_diff, + client_dict_input_timeout, dict); + return; + } + + /* If we've gotten here because all the time was spent in other ioloops + or locks, make sure there's a bit of time waiting for the dict + ioloop as well. There's a good chance that the reply can be read. */ + msecs_in_last_dict_ioloop_wait = + (io_wait_timer_get_usecs(dict->wait_timer) - + dict->last_timer_switch_usecs + 999) / 1000; + if (msecs_in_last_dict_ioloop_wait < DICT_CLIENT_REQUEST_TIMEOUT_MIN_LAST_IOLOOP_WAIT_MSECS) { + timeout_remove(&dict->to_requests); + dict->to_requests = + timeout_add(DICT_CLIENT_REQUEST_TIMEOUT_MIN_LAST_IOLOOP_WAIT_MSECS - + msecs_in_last_dict_ioloop_wait, + client_dict_input_timeout, dict); + return; + } + + (void)client_dict_reconnect(dict, t_strdup_printf( + "Dict server timeout: %s " + "(%u commands pending, oldest sent %u.%03u secs ago: %s, %s)", + connection_input_timeout_reason(&dict->conn.conn), + array_count(&dict->cmds), + cmd_diff/1000, cmd_diff%1000, cmd->query, + dict_wait_warnings(cmd)), &error); +} + +static int +client_dict_cmd_query_send(struct client_dict *dict, const char *query) +{ + struct const_iovec iov[2]; + ssize_t ret; + + iov[0].iov_base = query; + iov[0].iov_len = strlen(query); + iov[1].iov_base = "\n"; + iov[1].iov_len = 1; + ret = o_stream_sendv(dict->conn.conn.output, iov, 2); + if (ret < 0) + return -1; + i_assert((size_t)ret == iov[0].iov_len + 1); + return 0; +} + +static bool +client_dict_cmd_send(struct client_dict *dict, struct client_dict_cmd **_cmd, + const char **error_r) +{ + struct client_dict_cmd *cmd = *_cmd; + const char *error = NULL; + bool retry = cmd->retry_errors; + int ret; + + *_cmd = NULL; + + /* we're no longer idling. even with no_replies=TRUE we're going to + wait for COMMIT/ROLLBACK. */ + timeout_remove(&dict->to_idle); + + if (client_dict_connect(dict, &error) < 0) { + retry = FALSE; + ret = -1; + } else { + ret = client_dict_cmd_query_send(dict, cmd->query); + if (ret < 0) { + error = t_strdup_printf("write(%s) failed: %s", dict->conn.conn.name, + o_stream_get_error(dict->conn.conn.output)); + } + } + if (ret < 0 && retry) { + /* Reconnect and try again. */ + if (client_dict_reconnect(dict, error, &error) < 0) + ; + else if (client_dict_cmd_query_send(dict, cmd->query) < 0) { + error = t_strdup_printf("write(%s) failed: %s", dict->conn.conn.name, + o_stream_get_error(dict->conn.conn.output)); + } else { + ret = 0; + } + } + + if (cmd->no_replies) { + /* just send and forget */ + client_dict_cmd_unref(cmd); + return TRUE; + } else if (ret < 0) { + i_assert(error != NULL); + /* we didn't successfully send this command to dict */ + dict_cmd_callback_error(cmd, error, cmd->reconnected); + client_dict_cmd_unref(cmd); + if (error_r != NULL) + *error_r = error; + return FALSE; + } else { + if (dict->to_requests == NULL && !cmd->background) { + dict->to_requests = + timeout_add(DICT_CLIENT_REQUEST_TIMEOUT_MSECS, + client_dict_input_timeout, dict); + } + array_push_back(&dict->cmds, &cmd); + return TRUE; + } +} + +static bool +client_dict_transaction_send_begin(struct client_dict_transaction_context *ctx, + const struct dict_op_settings_private *set) +{ + struct client_dict *dict = (struct client_dict *)ctx->ctx.dict; + struct client_dict_cmd *cmd; + const char *query, *error; + + i_assert(ctx->error == NULL); + + ctx->sent_begin = TRUE; + + /* transactions commands don't have replies. only COMMIT has. */ + query = t_strdup_printf("%c%u\t%s", DICT_PROTOCOL_CMD_BEGIN, + ctx->id, + set->username == NULL ? "" : str_tabescape(set->username)); + cmd = client_dict_cmd_init(dict, query); + cmd->no_replies = TRUE; + cmd->retry_errors = TRUE; + if (!client_dict_cmd_send(dict, &cmd, &error)) { + ctx->error = i_strdup(error); + return FALSE; + } + return TRUE; +} + +static void +client_dict_send_transaction_query(struct client_dict_transaction_context *ctx, + const char *query) +{ + struct client_dict *dict = (struct client_dict *)ctx->ctx.dict; + const struct dict_op_settings_private *set = &ctx->ctx.set; + struct client_dict_cmd *cmd; + const char *error; + + if (ctx->error != NULL) + return; + + if (!ctx->sent_begin) { + if (!client_dict_transaction_send_begin(ctx, set)) + return; + } + + ctx->query_count++; + if (ctx->first_query == NULL) + ctx->first_query = i_strdup(query); + + cmd = client_dict_cmd_init(dict, query); + cmd->no_replies = TRUE; + if (!client_dict_cmd_send(dict, &cmd, &error)) + ctx->error = i_strdup(error); +} + +static bool client_dict_is_finished(struct client_dict *dict) +{ + return dict->transactions == NULL && array_count(&dict->cmds) == 0; +} + +static void client_dict_timeout(struct client_dict *dict) +{ + if (client_dict_is_finished(dict)) + client_dict_disconnect(dict, "Idle disconnection"); + else + timeout_remove(&dict->to_idle); +} + +static bool client_dict_have_nonbackground_cmds(struct client_dict *dict) +{ + struct client_dict_cmd *cmd; + + array_foreach_elem(&dict->cmds, cmd) { + if (!cmd->background) + return TRUE; + } + return FALSE; +} + +static void client_dict_add_timeout(struct client_dict *dict) +{ + if (dict->to_idle != NULL) { + if (dict->idle_msecs > 0) + timeout_reset(dict->to_idle); + } else if (client_dict_is_finished(dict)) { + dict->to_idle = timeout_add(dict->idle_msecs, + client_dict_timeout, dict); + timeout_remove(&dict->to_requests); + } else if (dict->transactions == NULL && + !client_dict_have_nonbackground_cmds(dict)) { + /* we had non-background commands, but now we're back to + having only background commands. remove timeouts. */ + timeout_remove(&dict->to_requests); + } +} + +static void client_dict_cmd_backgrounded(struct client_dict *dict) +{ + if (dict->to_requests == NULL) + return; + + if (!client_dict_have_nonbackground_cmds(dict)) { + /* we only have background-commands. + remove the request timeout. */ + timeout_remove(&dict->to_requests); + } +} + +static int +dict_conn_assign_next_async_id(struct dict_client_connection *conn, + const char *line) +{ + struct client_dict_cmd *const *cmds; + unsigned int i, count, async_id; + + i_assert(line[0] == DICT_PROTOCOL_REPLY_ASYNC_ID); + + if (str_to_uint(line+1, &async_id) < 0 || async_id == 0) { + e_error(conn->conn.event, "Received invalid async-id line: %s", + line); + return -1; + } + cmds = array_get(&conn->dict->cmds, &count); + for (i = 0; i < count; i++) { + if (cmds[i]->async_id == 0) { + cmds[i]->async_id = async_id; + cmds[i]->async_id_received_time = ioloop_timeval; + return 0; + } + } + e_error(conn->conn.event, "Received async-id line, but all %u " + "commands already have it: %s", + count, line); + return -1; +} + +static int dict_conn_find_async_id(struct dict_client_connection *conn, + const char *async_arg, + const char *line, unsigned int *idx_r) +{ + struct client_dict_cmd *const *cmds; + unsigned int i, count, async_id; + + i_assert(async_arg[0] == DICT_PROTOCOL_REPLY_ASYNC_REPLY); + + if (str_to_uint(async_arg+1, &async_id) < 0 || async_id == 0) { + e_error(conn->conn.event, "Received invalid async-reply line: %s", + line); + return -1; + } + + cmds = array_get(&conn->dict->cmds, &count); + for (i = 0; i < count; i++) { + if (cmds[i]->async_id == async_id) { + *idx_r = i; + return 0; + } + } + e_error(conn->conn.event, "Received reply for nonexistent async-id %u: %s", + async_id, line); + return -1; +} + +static int dict_conn_input_line(struct connection *_conn, const char *line) +{ + struct dict_client_connection *conn = + (struct dict_client_connection *)_conn; + struct client_dict *dict = conn->dict; + struct client_dict_cmd *const *cmds; + const char *const *args; + unsigned int i, count; + bool finished; + + if (dict->to_requests != NULL) + timeout_reset(dict->to_requests); + + if (line[0] == DICT_PROTOCOL_REPLY_ASYNC_ID) + return dict_conn_assign_next_async_id(conn, line) < 0 ? -1 : 1; + + cmds = array_get(&conn->dict->cmds, &count); + if (count == 0) { + e_error(conn->conn.event, "Received reply without pending commands: %s", + line); + return -1; + } + + args = t_strsplit_tabescaped(line); + if (args[0] != NULL && args[0][0] == DICT_PROTOCOL_REPLY_ASYNC_REPLY) { + if (dict_conn_find_async_id(conn, args[0], line, &i) < 0) + return -1; + args++; + } else { + i = 0; + } + i_assert(!cmds[i]->no_replies); + + client_dict_cmd_ref(cmds[i]); + finished = dict_cmd_callback_line(cmds[i], args); + if (!client_dict_cmd_unref(cmds[i])) { + /* disconnected during command handling */ + return -1; + } + if (!finished) { + /* more lines needed for this command */ + return 1; + } + client_dict_cmd_unref(cmds[i]); + array_delete(&dict->cmds, i, 1); + + client_dict_add_timeout(dict); + return 1; +} + +static int client_dict_connect(struct client_dict *dict, const char **error_r) +{ + const char *query, *error; + + if (dict->conn.conn.fd_in != -1) + return 0; + if (dict->last_failed_connect == ioloop_time) { + /* Try again later */ + *error_r = dict->last_connect_error; + return -1; + } + + if (connection_client_connect(&dict->conn.conn) < 0) { + dict->last_failed_connect = ioloop_time; + if (errno == EACCES) { + error = eacces_error_get("net_connect_unix", + dict->conn.conn.name); + } else { + error = t_strdup_printf( + "net_connect_unix(%s) failed: %m", dict->conn.conn.name); + } + i_free(dict->last_connect_error); + dict->last_connect_error = i_strdup(error); + *error_r = error; + return -1; + } + + query = t_strdup_printf("%c%u\t%u\t%d\t%s\t%s\n", + DICT_PROTOCOL_CMD_HELLO, + DICT_CLIENT_PROTOCOL_MAJOR_VERSION, + DICT_CLIENT_PROTOCOL_MINOR_VERSION, + dict->value_type, + "", + str_tabescape(dict->uri)); + o_stream_nsend_str(dict->conn.conn.output, query); + client_dict_add_timeout(dict); + return 0; +} + +static void +client_dict_abort_commands(struct client_dict *dict, const char *reason) +{ + ARRAY(struct client_dict_cmd *) cmds_copy; + struct client_dict_cmd *cmd; + + /* abort all commands */ + t_array_init(&cmds_copy, array_count(&dict->cmds)); + array_append_array(&cmds_copy, &dict->cmds); + array_clear(&dict->cmds); + + array_foreach_elem(&cmds_copy, cmd) { + dict_cmd_callback_error(cmd, reason, TRUE); + client_dict_cmd_unref(cmd); + } +} + +static void client_dict_disconnect(struct client_dict *dict, const char *reason) +{ + struct client_dict_transaction_context *ctx, *next; + + client_dict_abort_commands(dict, reason); + + /* all transactions that have sent BEGIN are no longer valid */ + for (ctx = dict->transactions; ctx != NULL; ctx = next) { + next = ctx->next; + if (ctx->sent_begin && ctx->error == NULL) + ctx->error = i_strdup(reason); + } + + timeout_remove(&dict->to_idle); + timeout_remove(&dict->to_requests); + connection_disconnect(&dict->conn.conn); +} + +static int client_dict_reconnect(struct client_dict *dict, const char *reason, + const char **error_r) +{ + ARRAY(struct client_dict_cmd *) retry_cmds; + struct client_dict_cmd *cmd; + const char *error; + int ret; + + t_array_init(&retry_cmds, array_count(&dict->cmds)); + for (unsigned int i = 0; i < array_count(&dict->cmds); ) { + cmd = array_idx_elem(&dict->cmds, i); + if (!cmd->retry_errors) { + i++; + } else if (cmd->iter != NULL && + cmd->iter->seen_results) { + /* don't retry iteration that already returned + something to the caller. otherwise we'd return + duplicates. */ + i++; + } else { + array_push_back(&retry_cmds, &cmd); + array_delete(&dict->cmds, i, 1); + } + } + client_dict_disconnect(dict, reason); + if (client_dict_connect(dict, error_r) < 0) { + reason = t_strdup_printf("%s - reconnect failed: %s", + reason, *error_r); + array_foreach_elem(&retry_cmds, cmd) { + dict_cmd_callback_error(cmd, reason, TRUE); + client_dict_cmd_unref(cmd); + } + return -1; + } + if (array_count(&retry_cmds) == 0) + return 0; + e_warning(dict->conn.conn.event, "%s - reconnected", reason); + + ret = 0; error = ""; + array_foreach_elem(&retry_cmds, cmd) { + cmd->reconnected = TRUE; + cmd->async_id = 0; + /* if it fails again, don't retry anymore */ + cmd->retry_errors = FALSE; + if (ret < 0) { + dict_cmd_callback_error(cmd, error, TRUE); + client_dict_cmd_unref(cmd); + } else if (!client_dict_cmd_send(dict, &cmd, &error)) + ret = -1; + } + return ret; +} + +static void dict_conn_destroy(struct connection *_conn) +{ + struct dict_client_connection *conn = + (struct dict_client_connection *)_conn; + + client_dict_disconnect(conn->dict, connection_disconnect_reason(_conn)); +} + +static const struct connection_settings dict_conn_set = { + .input_max_size = SIZE_MAX, + .output_max_size = SIZE_MAX, + .unix_client_connect_msecs = 1000, + .client = TRUE +}; + +static const struct connection_vfuncs dict_conn_vfuncs = { + .destroy = dict_conn_destroy, + .input_line = dict_conn_input_line +}; + +static int +client_dict_init(struct dict *driver, const char *uri, + const struct dict_settings *set, + struct dict **dict_r, const char **error_r) +{ + struct ioloop *old_ioloop = current_ioloop; + struct client_dict *dict; + const char *p, *dest_uri, *path; + unsigned int idle_msecs = DICT_CLIENT_DEFAULT_TIMEOUT_MSECS; + unsigned int warn_slow_msecs = DICT_CLIENT_DEFAULT_WARN_SLOW_MSECS; + + /* uri = [idle_msecs=<n>:] [warn_slow_msecs=<n>:] [<path>] ":" <uri> */ + for (;;) { + if (str_begins(uri, "idle_msecs=")) { + p = strchr(uri+11, ':'); + if (p == NULL) { + *error_r = t_strdup_printf("Invalid URI: %s", uri); + return -1; + } + if (str_to_uint(t_strdup_until(uri+11, p), &idle_msecs) < 0) { + *error_r = "Invalid idle_msecs"; + return -1; + } + uri = p+1; + } else if (str_begins(uri, "warn_slow_msecs=")) { + p = strchr(uri+11, ':'); + if (p == NULL) { + *error_r = t_strdup_printf("Invalid URI: %s", uri); + return -1; + } + if (str_to_uint(t_strdup_until(uri+16, p), &warn_slow_msecs) < 0) { + *error_r = "Invalid warn_slow_msecs"; + return -1; + } + uri = p+1; + } else { + break; + } + } + dest_uri = strchr(uri, ':'); + if (dest_uri == NULL) { + *error_r = t_strdup_printf("Invalid URI: %s", uri); + return -1; + } + + if (dict_connections == NULL) { + dict_connections = connection_list_init(&dict_conn_set, + &dict_conn_vfuncs); + } + + dict = i_new(struct client_dict, 1); + dict->dict = *driver; + dict->conn.dict = dict; + dict->conn.conn.event_parent = set->event_parent; + dict->idle_msecs = idle_msecs; + dict->warn_slow_msecs = warn_slow_msecs; + i_array_init(&dict->cmds, 32); + + if (uri[0] == ':') { + /* default path */ + path = t_strconcat(set->base_dir, + "/"DEFAULT_DICT_SERVER_SOCKET_FNAME, NULL); + } else if (uri[0] == '/') { + /* absolute path */ + path = t_strdup_until(uri, dest_uri); + } else { + /* relative path to base_dir */ + path = t_strconcat(set->base_dir, "/", + t_strdup_until(uri, dest_uri), NULL); + } + connection_init_client_unix(dict_connections, &dict->conn.conn, path); + dict->uri = i_strdup(dest_uri + 1); + + dict->dict.ioloop = io_loop_create(); + dict->wait_timer = io_wait_timer_add(); + io_loop_set_current(old_ioloop); + *dict_r = &dict->dict; + return 0; +} + +static void client_dict_deinit(struct dict *_dict) +{ + struct client_dict *dict = (struct client_dict *)_dict; + struct ioloop *old_ioloop = current_ioloop; + + client_dict_disconnect(dict, "Deinit"); + connection_deinit(&dict->conn.conn); + io_wait_timer_remove(&dict->wait_timer); + + i_assert(dict->transactions == NULL); + i_assert(array_count(&dict->cmds) == 0); + + io_loop_set_current(dict->dict.ioloop); + io_loop_destroy(&dict->dict.ioloop); + io_loop_set_current(old_ioloop); + + array_free(&dict->cmds); + i_free(dict->last_connect_error); + i_free(dict->uri); + i_free(dict); + + if (dict_connections->connections == NULL) + connection_list_deinit(&dict_connections); +} + +static void client_dict_wait(struct dict *_dict) +{ + struct client_dict *dict = (struct client_dict *)_dict; + + if (array_count(&dict->cmds) == 0) + return; + + i_assert(io_loop_is_empty(dict->dict.ioloop)); + dict->dict.prev_ioloop = current_ioloop; + io_loop_set_current(dict->dict.ioloop); + dict_switch_ioloop(_dict); + while (array_count(&dict->cmds) > 0) + io_loop_run(dict->dict.ioloop); + + io_loop_set_current(dict->dict.prev_ioloop); + dict->dict.prev_ioloop = NULL; + + dict_switch_ioloop(_dict); + i_assert(io_loop_is_empty(dict->dict.ioloop)); +} + +static bool client_dict_switch_ioloop(struct dict *_dict) +{ + struct client_dict *dict = (struct client_dict *)_dict; + + dict->last_timer_switch_usecs = + io_wait_timer_get_usecs(dict->wait_timer); + dict->wait_timer = io_wait_timer_move(&dict->wait_timer); + if (dict->to_idle != NULL) + dict->to_idle = io_loop_move_timeout(&dict->to_idle); + if (dict->to_requests != NULL) + dict->to_requests = io_loop_move_timeout(&dict->to_requests); + connection_switch_ioloop(&dict->conn.conn); + return array_count(&dict->cmds) > 0; +} + +static const char *dict_wait_warnings(const struct client_dict_cmd *cmd) +{ + int global_ioloop_msecs = (ioloop_global_wait_usecs - + cmd->start_global_ioloop_usecs + 999) / 1000; + int dict_ioloop_msecs = (io_wait_timer_get_usecs(cmd->dict->wait_timer) - + cmd->start_dict_ioloop_usecs + 999) / 1000; + int other_ioloop_msecs = global_ioloop_msecs - dict_ioloop_msecs; + int lock_msecs = (file_lock_wait_get_total_usecs() - + cmd->start_lock_usecs + 999) / 1000; + + return t_strdup_printf( + "%d.%03d in dict wait, %d.%03d in other ioloops, %d.%03d in locks", + dict_ioloop_msecs/1000, dict_ioloop_msecs%1000, + other_ioloop_msecs/1000, other_ioloop_msecs%1000, + lock_msecs/1000, lock_msecs%1000); +} + +static const char * +dict_warnings_sec(const struct client_dict_cmd *cmd, int msecs, + const char *const *extra_args) +{ + string_t *str = t_str_new(64); + struct timeval tv_start, tv_end; + unsigned int tv_start_usec, tv_end_usec; + + str_printfa(str, "%d.%03d secs (%s", msecs/1000, msecs%1000, + dict_wait_warnings(cmd)); + if (cmd->reconnected) { + int reconnected_msecs = + timeval_diff_msecs(&ioloop_timeval, + &cmd->dict->conn.conn.connect_started); + str_printfa(str, ", reconnected %u.%03u secs ago", + reconnected_msecs/1000, reconnected_msecs%1000); + } + if (cmd->async_id != 0) { + int async_reply_msecs = + timeval_diff_msecs(&ioloop_timeval, &cmd->async_id_received_time); + str_printfa(str, ", async-id reply %u.%03u secs ago", + async_reply_msecs/1000, async_reply_msecs%1000); + } + if (extra_args != NULL && + str_array_length(extra_args) >= 4 && + str_to_time(extra_args[0], &tv_start.tv_sec) == 0 && + str_to_uint(extra_args[1], &tv_start_usec) == 0 && + str_to_time(extra_args[2], &tv_end.tv_sec) == 0 && + str_to_uint(extra_args[3], &tv_end_usec) == 0) { + tv_start.tv_usec = tv_start_usec; + tv_end.tv_usec = tv_end_usec; + + int server_msecs_since_start = + timeval_diff_msecs(&ioloop_timeval, &tv_start); + int server_msecs = timeval_diff_msecs(&tv_end, &tv_start); + str_printfa(str, ", started on dict-server %u.%03d secs ago, " + "took %u.%03d secs", + server_msecs_since_start/1000, + server_msecs_since_start%1000, + server_msecs/1000, server_msecs%1000); + } + str_append_c(str, ')'); + return str_c(str); +} + +static void +client_dict_lookup_async_callback(struct client_dict_cmd *cmd, + enum dict_protocol_reply reply, + const char *value, + const char *const *extra_args, + const char *error, + bool disconnected ATTR_UNUSED) +{ + struct client_dict *dict = cmd->dict; + struct dict_lookup_result result; + const char *const values[] = { value, NULL }; + + i_zero(&result); + if (error != NULL) { + result.ret = -1; + result.error = error; + } else switch (reply) { + case DICT_PROTOCOL_REPLY_OK: + result.value = value; + result.values = values; + result.ret = 1; + break; + case DICT_PROTOCOL_REPLY_MULTI_OK: + result.values = t_strsplit_tabescaped(value); + result.value = result.values[0]; + result.ret = 1; + break; + case DICT_PROTOCOL_REPLY_NOTFOUND: + result.ret = 0; + break; + case DICT_PROTOCOL_REPLY_FAIL: + result.error = value[0] == '\0' ? "dict-server returned failure" : + t_strdup_printf("dict-server returned failure: %s", + value); + result.ret = -1; + break; + default: + result.error = t_strdup_printf( + "dict-client: Invalid lookup '%s' reply: %c%s", + cmd->query, reply, value); + client_dict_disconnect(dict, result.error); + result.ret = -1; + break; + } + + int diff = timeval_diff_msecs(&ioloop_timeval, &cmd->start_time); + if (result.error != NULL) { + /* include timing info always in error messages */ + result.error = t_strdup_printf("%s (reply took %s)", + result.error, dict_warnings_sec(cmd, diff, extra_args)); + } else if (!cmd->background && + diff >= (int)dict->warn_slow_msecs) { + e_warning(dict->conn.conn.event, "dict lookup took %s: %s", + dict_warnings_sec(cmd, diff, extra_args), + cmd->query); + } + + dict_pre_api_callback(&dict->dict); + cmd->api_callback.lookup(&result, cmd->api_callback.context); + dict_post_api_callback(&dict->dict); +} + +static void +client_dict_lookup_async(struct dict *_dict, const struct dict_op_settings *set, + const char *key, dict_lookup_callback_t *callback, + void *context) +{ + struct client_dict *dict = (struct client_dict *)_dict; + struct client_dict_cmd *cmd; + const char *query; + + query = t_strdup_printf("%c%s\t%s", DICT_PROTOCOL_CMD_LOOKUP, + str_tabescape(key), + set->username == NULL ? "" : str_tabescape(set->username)); + cmd = client_dict_cmd_init(dict, query); + cmd->callback = client_dict_lookup_async_callback; + cmd->api_callback.lookup = callback; + cmd->api_callback.context = context; + cmd->retry_errors = TRUE; + + client_dict_cmd_send(dict, &cmd, NULL); +} + +struct client_dict_sync_lookup { + char *error; + char *value; + int ret; +}; + +static void client_dict_lookup_callback(const struct dict_lookup_result *result, + struct client_dict_sync_lookup *lookup) +{ + lookup->ret = result->ret; + if (result->ret == -1) + lookup->error = i_strdup(result->error); + else if (result->ret == 1) + lookup->value = i_strdup(result->value); +} + +static int client_dict_lookup(struct dict *_dict, + const struct dict_op_settings *set, + pool_t pool, const char *key, + const char **value_r, const char **error_r) +{ + struct client_dict_sync_lookup lookup; + + i_zero(&lookup); + lookup.ret = -2; + + dict_lookup_async(_dict, set, key, client_dict_lookup_callback, &lookup); + if (lookup.ret == -2) + client_dict_wait(_dict); + + switch (lookup.ret) { + case -1: + *error_r = t_strdup(lookup.error); + i_free(lookup.error); + return -1; + case 0: + i_assert(lookup.value == NULL); + *value_r = NULL; + return 0; + case 1: + *value_r = p_strdup(pool, lookup.value); + i_free(lookup.value); + return 1; + } + i_unreached(); +} + +static void client_dict_iterate_unref(struct client_dict_iterate_context *ctx) +{ + i_assert(ctx->refcount > 0); + if (--ctx->refcount > 0) + return; + i_free(ctx->error); + i_free(ctx); +} + +static void +client_dict_iter_api_callback(struct client_dict_iterate_context *ctx, + struct client_dict_cmd *cmd, + const char *const *extra_args) +{ + struct client_dict *dict = cmd->dict; + + if (ctx->deinit) { + /* Iterator was already deinitialized. Stop if we're in + client_dict_wait(). */ + dict_post_api_callback(&dict->dict); + return; + } + if (ctx->finished) { + int diff = timeval_diff_msecs(&ioloop_timeval, &cmd->start_time); + if (ctx->error != NULL) { + /* include timing info always in error messages */ + char *new_error = i_strdup_printf("%s (reply took %s)", + ctx->error, dict_warnings_sec(cmd, diff, extra_args)); + i_free(ctx->error); + ctx->error = new_error; + } else if (!cmd->background && + diff >= (int)dict->warn_slow_msecs) { + e_warning(dict->conn.conn.event, "dict iteration took %s: %s", + dict_warnings_sec(cmd, diff, extra_args), + cmd->query); + } + } + if (ctx->ctx.async_callback != NULL) { + dict_pre_api_callback(&dict->dict); + ctx->ctx.async_callback(ctx->ctx.async_context); + dict_post_api_callback(&dict->dict); + } else { + /* synchronous lookup */ + io_loop_stop(dict->dict.ioloop); + } +} + +static void +client_dict_iter_async_callback(struct client_dict_cmd *cmd, + enum dict_protocol_reply reply, + const char *value, + const char *const *extra_args, + const char *error, + bool disconnected ATTR_UNUSED) +{ + struct client_dict_iterate_context *ctx = cmd->iter; + struct client_dict *dict = cmd->dict; + struct client_dict_iter_result *result; + const char *iter_key = NULL, *const *iter_values = NULL; + + if (ctx->deinit) { + cmd->background = TRUE; + client_dict_cmd_backgrounded(dict); + } + + if (error != NULL) { + /* failed */ + } else switch (reply) { + case DICT_PROTOCOL_REPLY_ITER_FINISHED: + /* end of iteration */ + i_assert(!ctx->finished); + ctx->finished = TRUE; + client_dict_iter_api_callback(ctx, cmd, extra_args); + client_dict_iterate_unref(ctx); + return; + case DICT_PROTOCOL_REPLY_OK: + /* key \t value */ + iter_key = value; + iter_values = extra_args; + extra_args++; + break; + case DICT_PROTOCOL_REPLY_FAIL: + error = t_strdup_printf("dict-server returned failure: %s", value); + break; + default: + break; + } + if ((iter_values == NULL || iter_values[0] == NULL) && error == NULL) { + /* broken protocol */ + error = t_strdup_printf("dict client (%s) sent broken iterate reply: %c%s", + dict->conn.conn.name, reply, value); + client_dict_disconnect(dict, error); + } + + if (error != NULL) { + if (ctx->error == NULL) + ctx->error = i_strdup(error); + i_assert(!ctx->finished); + ctx->finished = TRUE; + client_dict_iter_api_callback(ctx, cmd, extra_args); + client_dict_iterate_unref(ctx); + return; + } + cmd->unfinished = TRUE; + + if (ctx->deinit) { + /* iterator was already deinitialized */ + return; + } + + result = array_append_space(&ctx->results); + result->key = p_strdup(ctx->results_pool, iter_key); + result->values = p_strarray_dup(ctx->results_pool, iter_values); + + client_dict_iter_api_callback(ctx, cmd, NULL); +} + +static struct dict_iterate_context * +client_dict_iterate_init(struct dict *_dict, + const struct dict_op_settings *set ATTR_UNUSED, + const char *path, enum dict_iterate_flags flags) +{ + struct client_dict_iterate_context *ctx; + + ctx = i_new(struct client_dict_iterate_context, 1); + ctx->ctx.dict = _dict; + ctx->results_pool = pool_alloconly_create("client dict iteration", 512); + ctx->flags = flags; + ctx->path = i_strdup(path); + ctx->refcount = 1; + i_array_init(&ctx->results, 64); + return &ctx->ctx; +} + +static void +client_dict_iterate_cmd_send(struct client_dict_iterate_context *ctx) +{ + struct client_dict *dict = (struct client_dict *)ctx->ctx.dict; + const struct dict_op_settings_private *set = &ctx->ctx.set; + struct client_dict_cmd *cmd; + string_t *query = t_str_new(256); + + /* we can't do this query in _iterate_init(), because + _set_limit() hasn't been called yet at that point. */ + str_printfa(query, "%c%d\t%"PRIu64, DICT_PROTOCOL_CMD_ITERATE, + ctx->flags, ctx->ctx.max_rows); + str_append_c(query, '\t'); + str_append_tabescaped(query, ctx->path); + str_append_c(query, '\t'); + str_append_tabescaped(query, set->username == NULL ? "" : set->username); + + cmd = client_dict_cmd_init(dict, str_c(query)); + cmd->iter = ctx; + cmd->callback = client_dict_iter_async_callback; + cmd->retry_errors = TRUE; + + ctx->refcount++; + client_dict_cmd_send(dict, &cmd, NULL); +} + +static bool client_dict_iterate(struct dict_iterate_context *_ctx, + const char **key_r, const char *const **values_r) +{ + struct client_dict_iterate_context *ctx = + (struct client_dict_iterate_context *)_ctx; + const struct client_dict_iter_result *results; + unsigned int count; + + if (ctx->error != NULL) { + ctx->ctx.has_more = FALSE; + return FALSE; + } + + results = array_get(&ctx->results, &count); + if (ctx->result_idx < count) { + *key_r = results[ctx->result_idx].key; + *values_r = results[ctx->result_idx].values; + ctx->ctx.has_more = TRUE; + ctx->result_idx++; + ctx->seen_results = TRUE; + return TRUE; + } + if (!ctx->cmd_sent) { + ctx->cmd_sent = TRUE; + client_dict_iterate_cmd_send(ctx); + return client_dict_iterate(_ctx, key_r, values_r); + } + ctx->ctx.has_more = !ctx->finished; + ctx->result_idx = 0; + array_clear(&ctx->results); + p_clear(ctx->results_pool); + + if ((ctx->flags & DICT_ITERATE_FLAG_ASYNC) == 0 && ctx->ctx.has_more) { + client_dict_wait(_ctx->dict); + return client_dict_iterate(_ctx, key_r, values_r); + } + return FALSE; +} + +static int client_dict_iterate_deinit(struct dict_iterate_context *_ctx, + const char **error_r) +{ + struct client_dict *dict = (struct client_dict *)_ctx->dict; + struct client_dict_iterate_context *ctx = + (struct client_dict_iterate_context *)_ctx; + int ret = ctx->error != NULL ? -1 : 0; + + i_assert(!ctx->deinit); + ctx->deinit = TRUE; + + *error_r = t_strdup(ctx->error); + array_free(&ctx->results); + pool_unref(&ctx->results_pool); + i_free(ctx->path); + client_dict_iterate_unref(ctx); + + client_dict_add_timeout(dict); + return ret; +} + +static struct dict_transaction_context * +client_dict_transaction_init(struct dict *_dict) +{ + struct client_dict *dict = (struct client_dict *)_dict; + struct client_dict_transaction_context *ctx; + + ctx = i_new(struct client_dict_transaction_context, 1); + ctx->ctx.dict = _dict; + ctx->id = ++dict->transaction_id_counter; + + DLLIST_PREPEND(&dict->transactions, ctx); + return &ctx->ctx; +} + +static void +client_dict_transaction_free(struct client_dict_transaction_context **_ctx) +{ + struct client_dict_transaction_context *ctx = *_ctx; + + *_ctx = NULL; + i_free(ctx->first_query); + i_free(ctx->error); + i_free(ctx); +} + +static void +client_dict_transaction_commit_callback(struct client_dict_cmd *cmd, + enum dict_protocol_reply reply, + const char *value, + const char *const *extra_args, + const char *error, bool disconnected) +{ + struct client_dict *dict = cmd->dict; + struct dict_commit_result result = { + .ret = DICT_COMMIT_RET_FAILED, .error = NULL + }; + + i_assert(cmd->trans != NULL); + + if (error != NULL) { + /* failed */ + if (disconnected) + result.ret = DICT_COMMIT_RET_WRITE_UNCERTAIN; + result.error = error; + } else switch (reply) { + case DICT_PROTOCOL_REPLY_OK: + result.ret = DICT_COMMIT_RET_OK; + break; + case DICT_PROTOCOL_REPLY_NOTFOUND: + result.ret = DICT_COMMIT_RET_NOTFOUND; + break; + case DICT_PROTOCOL_REPLY_WRITE_UNCERTAIN: + result.ret = DICT_COMMIT_RET_WRITE_UNCERTAIN; + /* fallthrough */ + case DICT_PROTOCOL_REPLY_FAIL: { + /* value contains the obsolete trans_id */ + const char *error = extra_args[0]; + + result.error = t_strdup_printf("dict-server returned failure: %s", + error != NULL ? t_str_tabunescape(error) : ""); + if (error != NULL) + extra_args++; + break; + } + default: + result.ret = DICT_COMMIT_RET_FAILED; + result.error = t_strdup_printf( + "dict-client: Invalid commit reply: %c%s", + reply, value); + client_dict_disconnect(dict, result.error); + break; + } + + int diff = timeval_diff_msecs(&ioloop_timeval, &cmd->start_time); + if (result.error != NULL) { + /* include timing info always in error messages */ + result.error = t_strdup_printf("%s (reply took %s)", + result.error, dict_warnings_sec(cmd, diff, extra_args)); + } else if (!cmd->background && !cmd->trans->ctx.no_slowness_warning && + diff >= (int)dict->warn_slow_msecs) { + e_warning(dict->conn.conn.event, "dict commit took %s: " + "%s (%u commands, first: %s)", + dict_warnings_sec(cmd, diff, extra_args), + cmd->query, cmd->trans->query_count, + cmd->trans->first_query); + } + client_dict_transaction_free(&cmd->trans); + + dict_pre_api_callback(&dict->dict); + cmd->api_callback.commit(&result, cmd->api_callback.context); + dict_post_api_callback(&dict->dict); +} + + +static void +client_dict_transaction_commit(struct dict_transaction_context *_ctx, + bool async, + dict_transaction_commit_callback_t *callback, + void *context) +{ + struct client_dict_transaction_context *ctx = + (struct client_dict_transaction_context *)_ctx; + struct client_dict *dict = (struct client_dict *)_ctx->dict; + struct client_dict_cmd *cmd; + const char *query; + + DLLIST_REMOVE(&dict->transactions, ctx); + + if (ctx->sent_begin && ctx->error == NULL) { + query = t_strdup_printf("%c%u", DICT_PROTOCOL_CMD_COMMIT, ctx->id); + cmd = client_dict_cmd_init(dict, query); + cmd->trans = ctx; + + cmd->callback = client_dict_transaction_commit_callback; + cmd->api_callback.commit = callback; + cmd->api_callback.context = context; + if (callback == dict_transaction_commit_async_noop_callback) + cmd->background = TRUE; + if (client_dict_cmd_send(dict, &cmd, NULL)) { + if (!async) + client_dict_wait(_ctx->dict); + } + } else if (ctx->error != NULL) { + /* already failed */ + struct dict_commit_result result = { + .ret = DICT_COMMIT_RET_FAILED, .error = ctx->error + }; + callback(&result, context); + client_dict_transaction_free(&ctx); + } else { + /* nothing changed */ + struct dict_commit_result result = { + .ret = DICT_COMMIT_RET_OK, .error = NULL + }; + callback(&result, context); + client_dict_transaction_free(&ctx); + } + + client_dict_add_timeout(dict); +} + +static void +client_dict_transaction_rollback(struct dict_transaction_context *_ctx) +{ + struct client_dict_transaction_context *ctx = + (struct client_dict_transaction_context *)_ctx; + struct client_dict *dict = (struct client_dict *)_ctx->dict; + + if (ctx->sent_begin) { + const char *query; + + query = t_strdup_printf("%c%u", DICT_PROTOCOL_CMD_ROLLBACK, + ctx->id); + client_dict_send_transaction_query(ctx, query); + } + + DLLIST_REMOVE(&dict->transactions, ctx); + client_dict_transaction_free(&ctx); + client_dict_add_timeout(dict); +} + +static void client_dict_set(struct dict_transaction_context *_ctx, + const char *key, const char *value) +{ + struct client_dict_transaction_context *ctx = + (struct client_dict_transaction_context *)_ctx; + const char *query; + + query = t_strdup_printf("%c%u\t%s\t%s", + DICT_PROTOCOL_CMD_SET, ctx->id, + str_tabescape(key), + str_tabescape(value)); + client_dict_send_transaction_query(ctx, query); +} + +static void client_dict_unset(struct dict_transaction_context *_ctx, + const char *key) +{ + struct client_dict_transaction_context *ctx = + (struct client_dict_transaction_context *)_ctx; + const char *query; + + query = t_strdup_printf("%c%u\t%s", + DICT_PROTOCOL_CMD_UNSET, ctx->id, + str_tabescape(key)); + client_dict_send_transaction_query(ctx, query); +} + +static void client_dict_atomic_inc(struct dict_transaction_context *_ctx, + const char *key, long long diff) +{ + struct client_dict_transaction_context *ctx = + (struct client_dict_transaction_context *)_ctx; + const char *query; + + query = t_strdup_printf("%c%u\t%s\t%lld", + DICT_PROTOCOL_CMD_ATOMIC_INC, + ctx->id, str_tabescape(key), diff); + client_dict_send_transaction_query(ctx, query); +} + +static void client_dict_set_timestamp(struct dict_transaction_context *_ctx, + const struct timespec *ts) +{ + struct client_dict_transaction_context *ctx = + (struct client_dict_transaction_context *)_ctx; + const char *query; + + query = t_strdup_printf("%c%u\t%s\t%u", + DICT_PROTOCOL_CMD_TIMESTAMP, + ctx->id, dec2str(ts->tv_sec), + (unsigned int)ts->tv_nsec); + client_dict_send_transaction_query(ctx, query); +} + +static void client_dict_set_hide_log_values(struct dict_transaction_context *_ctx, + bool hide_log_values) +{ + struct client_dict_transaction_context *ctx = + container_of(_ctx, struct client_dict_transaction_context, ctx); + const char *query; + + query = t_strdup_printf("%c%u\t%s", + DICT_PROTOCOL_CMD_HIDE_LOG_VALUES, + ctx->id, + hide_log_values ? "yes" : "no"); + + client_dict_send_transaction_query(ctx, query); +} + +struct dict dict_driver_client = { + .name = "proxy", + + { + .init = client_dict_init, + .deinit = client_dict_deinit, + .wait = client_dict_wait, + .lookup = client_dict_lookup, + .iterate_init = client_dict_iterate_init, + .iterate = client_dict_iterate, + .iterate_deinit = client_dict_iterate_deinit, + .transaction_init = client_dict_transaction_init, + .transaction_commit = client_dict_transaction_commit, + .transaction_rollback = client_dict_transaction_rollback, + .set = client_dict_set, + .unset = client_dict_unset, + .atomic_inc = client_dict_atomic_inc, + .lookup_async = client_dict_lookup_async, + .switch_ioloop = client_dict_switch_ioloop, + .set_timestamp = client_dict_set_timestamp, + .set_hide_log_values = client_dict_set_hide_log_values, + } +}; diff --git a/src/lib-dict/dict-client.h b/src/lib-dict/dict-client.h new file mode 100644 index 0000000..95e2fb0 --- /dev/null +++ b/src/lib-dict/dict-client.h @@ -0,0 +1,48 @@ +#ifndef DICT_CLIENT_H +#define DICT_CLIENT_H + +#include "dict.h" + +#define DEFAULT_DICT_SERVER_SOCKET_FNAME "dict" + +#define DICT_CLIENT_PROTOCOL_MAJOR_VERSION 3 +#define DICT_CLIENT_PROTOCOL_MINOR_VERSION 2 + +#define DICT_CLIENT_PROTOCOL_VERSION_MIN_MULTI_OK 2 + +#define DICT_CLIENT_MAX_LINE_LENGTH (64*1024) + +enum dict_protocol_cmd { + /* <major-version> <minor-version> <value type> <user> <dict name> */ + DICT_PROTOCOL_CMD_HELLO = 'H', + + DICT_PROTOCOL_CMD_LOOKUP = 'L', /* <key> */ + DICT_PROTOCOL_CMD_ITERATE = 'I', /* <flags> <path> */ + + DICT_PROTOCOL_CMD_BEGIN = 'B', /* <id> */ + DICT_PROTOCOL_CMD_COMMIT = 'C', /* <id> */ + DICT_PROTOCOL_CMD_COMMIT_ASYNC = 'D', /* <id> */ + DICT_PROTOCOL_CMD_ROLLBACK = 'R', /* <id> */ + + DICT_PROTOCOL_CMD_SET = 'S', /* <id> <key> <value> */ + DICT_PROTOCOL_CMD_UNSET = 'U', /* <id> <key> */ + DICT_PROTOCOL_CMD_ATOMIC_INC = 'A', /* <id> <key> <diff> */ + DICT_PROTOCOL_CMD_TIMESTAMP = 'T', /* <id> <secs> <nsecs> */ + DICT_PROTOCOL_CMD_HIDE_LOG_VALUES = 'V', /* <id> <hide_log_values> */ +}; + +enum dict_protocol_reply { + DICT_PROTOCOL_REPLY_ERROR = -1, + + DICT_PROTOCOL_REPLY_OK = 'O', /* <value> */ + DICT_PROTOCOL_REPLY_MULTI_OK = 'M', /* protocol v2.2+ */ + DICT_PROTOCOL_REPLY_NOTFOUND = 'N', + DICT_PROTOCOL_REPLY_FAIL = 'F', + DICT_PROTOCOL_REPLY_WRITE_UNCERTAIN = 'W', + DICT_PROTOCOL_REPLY_ASYNC_COMMIT = 'A', + DICT_PROTOCOL_REPLY_ITER_FINISHED = '\0', + DICT_PROTOCOL_REPLY_ASYNC_ID = '*', + DICT_PROTOCOL_REPLY_ASYNC_REPLY = '+', +}; + +#endif diff --git a/src/lib-dict/dict-fail.c b/src/lib-dict/dict-fail.c new file mode 100644 index 0000000..101750b --- /dev/null +++ b/src/lib-dict/dict-fail.c @@ -0,0 +1,134 @@ +/* Copyright (c) 2017-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "dict.h" +#include "dict-private.h" + +struct dict_iterate_context dict_iter_unsupported = +{ + .dict = &dict_driver_fail, +}; + +struct dict_transaction_context dict_transaction_unsupported = +{ + .dict = &dict_driver_fail, +}; + +static int dict_fail_init(struct dict *dict_driver ATTR_UNUSED, + const char *uri ATTR_UNUSED, + const struct dict_settings *set ATTR_UNUSED, + struct dict **dict_r ATTR_UNUSED, const char **error_r) +{ + *error_r = "Unsupported operation (dict does not support this feature)"; + return -1; +} + +static void dict_fail_deinit(struct dict *dict ATTR_UNUSED) +{ +} + +static void dict_fail_wait(struct dict *dict ATTR_UNUSED) +{ +} + +static int dict_fail_lookup(struct dict *dict ATTR_UNUSED, + const struct dict_op_settings *set ATTR_UNUSED, + pool_t pool ATTR_UNUSED, + const char *key ATTR_UNUSED, const char **value_r ATTR_UNUSED, + const char **error_r) +{ + *error_r = "Unsupported operation (dict does not support this feature)"; + return -1; +} + +static struct dict_iterate_context * +dict_fail_iterate_init(struct dict *dict ATTR_UNUSED, + const struct dict_op_settings *set ATTR_UNUSED, + const char *path ATTR_UNUSED, + enum dict_iterate_flags flags ATTR_UNUSED) +{ + return &dict_iter_unsupported; +} + +static bool dict_fail_iterate(struct dict_iterate_context *ctx ATTR_UNUSED, + const char **key_r ATTR_UNUSED, + const char *const **values_r ATTR_UNUSED) +{ + return FALSE; +} + +static int dict_fail_iterate_deinit(struct dict_iterate_context *ctx ATTR_UNUSED, + const char **error_r) +{ + *error_r = "Unsupported operation (dict does not support this feature)"; + return -1; +} + +static struct dict_transaction_context *dict_fail_transaction_init(struct dict *dict ATTR_UNUSED) +{ + return &dict_transaction_unsupported; +} + +static void dict_fail_transaction_commit(struct dict_transaction_context *ctx ATTR_UNUSED, + bool async ATTR_UNUSED, + dict_transaction_commit_callback_t *callback, + void *context) +{ + struct dict_commit_result res = { + .ret = DICT_COMMIT_RET_FAILED, + .error = "Unsupported operation (dict does not support this feature)" + }; + if (callback != NULL) + callback(&res, context); +} + +static void dict_fail_transaction_rollback(struct dict_transaction_context *ctx ATTR_UNUSED) +{ +} + +static void dict_fail_set(struct dict_transaction_context *ctx ATTR_UNUSED, + const char *key ATTR_UNUSED, const char *value ATTR_UNUSED) +{ +} + +static void dict_fail_unset(struct dict_transaction_context *ctx ATTR_UNUSED, + const char *key ATTR_UNUSED) +{ +} + +static void dict_fail_atomic_inc(struct dict_transaction_context *ctx ATTR_UNUSED, + const char *key ATTR_UNUSED, long long diff ATTR_UNUSED) +{ +} + +static bool dict_fail_switch_ioloop(struct dict *dict ATTR_UNUSED) +{ + return TRUE; +} + +static void dict_fail_set_timestamp(struct dict_transaction_context *ctx ATTR_UNUSED, + const struct timespec *ts ATTR_UNUSED) +{ +} + +struct dict dict_driver_fail = { + .name = "fail", + .v = { + .init = dict_fail_init, + .deinit = dict_fail_deinit, + .wait = dict_fail_wait, + .lookup = dict_fail_lookup, + .iterate_init = dict_fail_iterate_init, + .iterate = dict_fail_iterate, + .iterate_deinit = dict_fail_iterate_deinit, + .transaction_init = dict_fail_transaction_init, + .transaction_commit = dict_fail_transaction_commit, + .transaction_rollback = dict_fail_transaction_rollback, + .set = dict_fail_set, + .unset = dict_fail_unset, + .atomic_inc = dict_fail_atomic_inc, + .lookup_async = NULL, + .switch_ioloop = dict_fail_switch_ioloop, + .set_timestamp = dict_fail_set_timestamp + }, +}; diff --git a/src/lib-dict/dict-file.c b/src/lib-dict/dict-file.c new file mode 100644 index 0000000..c9228a8 --- /dev/null +++ b/src/lib-dict/dict-file.c @@ -0,0 +1,709 @@ +/* Copyright (c) 2008-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "hash.h" +#include "str.h" +#include "strescape.h" +#include "home-expand.h" +#include "mkdir-parents.h" +#include "eacces-error.h" +#include "file-lock.h" +#include "file-dotlock.h" +#include "nfs-workarounds.h" +#include "istream.h" +#include "ostream.h" +#include "dict-transaction-memory.h" +#include "dict-private.h" + +#include <stdio.h> +#include <unistd.h> +#include <fcntl.h> +#include <sys/stat.h> + +struct file_dict { + struct dict dict; + pool_t hash_pool; + enum file_lock_method lock_method; + + char *path; + char *home_dir; + bool dict_path_checked; + HASH_TABLE(char *, char *) hash; + int fd; + + bool refreshed; +}; + +struct file_dict_iterate_context { + struct dict_iterate_context ctx; + pool_t pool; + + struct hash_iterate_context *iter; + const char *path; + size_t path_len; + + enum dict_iterate_flags flags; + const char *values[2]; + const char *error; +}; + +static struct dotlock_settings file_dict_dotlock_settings = { + .timeout = 60*2, + .stale_timeout = 60, + .use_io_notify = TRUE +}; + +static int +file_dict_ensure_path_home_dir(struct file_dict *dict, const char *home_dir, + const char **error_r) +{ + if (null_strcmp(dict->home_dir, home_dir) == 0) + return 0; + + if (dict->dict_path_checked) { + *error_r = t_strdup_printf("home_dir changed from %s to %s " + "(requested dict was: %s)", dict->home_dir, + home_dir, dict->path); + return -1; + } + + char *_p = dict->path; + dict->path = i_strdup(home_expand_tilde(dict->path, home_dir)); + dict->home_dir = i_strdup(home_dir); + i_free(_p); + dict->dict_path_checked = TRUE; + return 0; +} + +static int +file_dict_init(struct dict *driver, const char *uri, + const struct dict_settings *set ATTR_UNUSED, + struct dict **dict_r, const char **error_r) +{ + struct file_dict *dict; + const char *p, *path; + + dict = i_new(struct file_dict, 1); + dict->lock_method = FILE_LOCK_METHOD_DOTLOCK; + + p = strchr(uri, ':'); + if (p == NULL) { + /* no parameters */ + path = uri; + } else { + path = t_strdup_until(uri, p++); + if (strcmp(p, "lock=fcntl") == 0) + dict->lock_method = FILE_LOCK_METHOD_FCNTL; + else if (strcmp(p, "lock=flock") == 0) + dict->lock_method = FILE_LOCK_METHOD_FLOCK; + else { + *error_r = t_strdup_printf("Invalid parameter: %s", p+1); + i_free(dict); + return -1; + } + } + + /* keep the path for now, later in dict operations check if home_dir + should be prepended. */ + dict->path = i_strdup(path); + + dict->dict = *driver; + dict->hash_pool = pool_alloconly_create("file dict", 1024); + hash_table_create(&dict->hash, dict->hash_pool, 0, str_hash, strcmp); + dict->fd = -1; + *dict_r = &dict->dict; + return 0; +} + +static void file_dict_deinit(struct dict *_dict) +{ + struct file_dict *dict = (struct file_dict *)_dict; + + i_close_fd_path(&dict->fd, dict->path); + hash_table_destroy(&dict->hash); + pool_unref(&dict->hash_pool); + i_free(dict->path); + i_free(dict->home_dir); + i_free(dict); +} + +static bool file_dict_need_refresh(struct file_dict *dict) +{ + struct stat st1, st2; + + if (dict->dict.iter_count > 0) { + /* Change nothing while there are iterators or they can crash + because the hash table content recreated. */ + return FALSE; + } + + if (dict->fd == -1) + return TRUE; + + /* Disable NFS flushing for now since it can cause unnecessary + problems and there's no easy way for us to know here if + mail_nfs_storage=yes. In any case it's pretty much an unsupported + setting nowadays. */ + /*nfs_flush_file_handle_cache(dict->path);*/ + if (nfs_safe_stat(dict->path, &st1) < 0) { + e_error(dict->dict.event, "stat(%s) failed: %m", dict->path); + return FALSE; + } + + if (fstat(dict->fd, &st2) < 0) { + if (errno != ESTALE) + e_error(dict->dict.event, "fstat(%s) failed: %m", dict->path); + return TRUE; + } + if (st1.st_ino != st2.st_ino || + !CMP_DEV_T(st1.st_dev, st2.st_dev)) { + /* file changed */ + return TRUE; + } + return FALSE; +} + +static int file_dict_open_latest(struct file_dict *dict, const char **error_r) +{ + int open_type; + + if (!file_dict_need_refresh(dict)) + return 0; + + i_close_fd_path(&dict->fd, dict->path); + + open_type = dict->lock_method == FILE_LOCK_METHOD_DOTLOCK ? + O_RDONLY : O_RDWR; + dict->fd = open(dict->path, open_type); + if (dict->fd == -1) { + if (errno == ENOENT) + return 0; + if (errno == EACCES) + *error_r = eacces_error_get("open", dict->path); + else + *error_r = t_strdup_printf("open(%s) failed: %m", dict->path); + return -1; + } + dict->refreshed = FALSE; + return 1; +} + +static int file_dict_refresh(struct file_dict *dict, const char **error_r) +{ + struct istream *input; + char *key, *value; + + if (file_dict_open_latest(dict, error_r) < 0) + return -1; + if (dict->refreshed || dict->dict.iter_count > 0) + return 0; + + hash_table_clear(dict->hash, TRUE); + p_clear(dict->hash_pool); + + if (dict->fd != -1) { + input = i_stream_create_fd(dict->fd, SIZE_MAX); + + while ((key = i_stream_read_next_line(input)) != NULL) { + /* strdup() before the second read */ + key = str_tabunescape(p_strdup(dict->hash_pool, key)); + + if ((value = i_stream_read_next_line(input)) == NULL) + break; + + value = str_tabunescape(p_strdup(dict->hash_pool, value)); + hash_table_update(dict->hash, key, value); + } + i_stream_destroy(&input); + } + dict->refreshed = TRUE; + return 0; +} + +static int file_dict_lookup(struct dict *_dict, + const struct dict_op_settings *set, + pool_t pool, const char *key, + const char **value_r, const char **error_r) +{ + struct file_dict *dict = (struct file_dict *)_dict; + + if (file_dict_ensure_path_home_dir(dict, set->home_dir, error_r) < 0) + return -1; + + if (file_dict_refresh(dict, error_r) < 0) + return -1; + + *value_r = p_strdup(pool, hash_table_lookup(dict->hash, key)); + return *value_r == NULL ? 0 : 1; +} + +static struct dict_iterate_context * +file_dict_iterate_init(struct dict *_dict, + const struct dict_op_settings *set ATTR_UNUSED, + const char *path, enum dict_iterate_flags flags) +{ + struct file_dict_iterate_context *ctx; + struct file_dict *dict = (struct file_dict *)_dict; + const char *error; + pool_t pool; + + pool = pool_alloconly_create("file dict iterate", 256); + ctx = p_new(pool, struct file_dict_iterate_context, 1); + ctx->ctx.dict = _dict; + ctx->pool = pool; + + ctx->path = p_strdup(pool, path); + ctx->path_len = strlen(path); + ctx->flags = flags; + + if (file_dict_ensure_path_home_dir(dict, set->home_dir, &error) < 0 || + file_dict_refresh(dict, &error) < 0) + ctx->error = p_strdup(pool, error); + + ctx->iter = hash_table_iterate_init(dict->hash); + return &ctx->ctx; +} + +static bool +file_dict_iterate_key_matches(struct file_dict_iterate_context *ctx, + const char *key) +{ + if (strncmp(ctx->path, key, ctx->path_len) == 0) + return TRUE; + return FALSE; +} + + +static bool file_dict_iterate(struct dict_iterate_context *_ctx, + const char **key_r, const char *const **values_r) +{ + struct file_dict_iterate_context *ctx = + (struct file_dict_iterate_context *)_ctx; + char *key, *value; + + while (hash_table_iterate(ctx->iter, + ((struct file_dict *)_ctx->dict)->hash, + &key, &value)) { + if (!file_dict_iterate_key_matches(ctx, key)) + continue; + + if ((ctx->flags & DICT_ITERATE_FLAG_RECURSE) != 0) { + /* match everything */ + } else if ((ctx->flags & DICT_ITERATE_FLAG_EXACT_KEY) != 0) { + if (key[ctx->path_len] != '\0') + continue; + } else { + if (strchr(key + ctx->path_len, '/') != NULL) + continue; + } + + *key_r = key; + ctx->values[0] = value; + *values_r = ctx->values; + return TRUE; + } + return FALSE; +} + +static int file_dict_iterate_deinit(struct dict_iterate_context *_ctx, + const char **error_r) +{ + struct file_dict_iterate_context *ctx = + (struct file_dict_iterate_context *)_ctx; + int ret = ctx->error != NULL ? -1 : 0; + + *error_r = t_strdup(ctx->error); + hash_table_iterate_deinit(&ctx->iter); + pool_unref(&ctx->pool); + return ret; +} + +static struct dict_transaction_context * +file_dict_transaction_init(struct dict *_dict) +{ + struct dict_transaction_memory_context *ctx; + pool_t pool; + + pool = pool_alloconly_create("file dict transaction", 2048); + ctx = p_new(pool, struct dict_transaction_memory_context, 1); + dict_transaction_memory_init(ctx, _dict, pool); + return &ctx->ctx; +} + +static void file_dict_apply_changes(struct dict_transaction_memory_context *ctx, + bool *atomic_inc_not_found_r) +{ + struct file_dict *dict = (struct file_dict *)ctx->ctx.dict; + const char *tmp; + char *key, *value, *old_value; + char *orig_key, *orig_value; + const struct dict_transaction_memory_change *change; + size_t new_len; + long long diff; + + array_foreach(&ctx->changes, change) { + if (hash_table_lookup_full(dict->hash, change->key, + &orig_key, &orig_value)) { + key = orig_key; + old_value = orig_value; + } else { + key = NULL; + old_value = NULL; + } + value = NULL; + + switch (change->type) { + case DICT_CHANGE_TYPE_INC: + if (old_value == NULL) { + *atomic_inc_not_found_r = TRUE; + break; + } + if (str_to_llong(old_value, &diff) < 0) + i_unreached(); + diff += change->value.diff; + tmp = t_strdup_printf("%lld", diff); + new_len = strlen(tmp); + if (old_value == NULL || new_len > strlen(old_value)) + value = p_strdup(dict->hash_pool, tmp); + else { + memcpy(old_value, tmp, new_len + 1); + value = old_value; + } + /* fall through */ + case DICT_CHANGE_TYPE_SET: + if (key == NULL) + key = p_strdup(dict->hash_pool, change->key); + if (value == NULL) { + value = p_strdup(dict->hash_pool, + change->value.str); + } + hash_table_update(dict->hash, key, value); + break; + case DICT_CHANGE_TYPE_UNSET: + if (old_value != NULL) + hash_table_remove(dict->hash, key); + break; + } + } +} + +static int +fd_copy_stat_permissions(const struct stat *src_st, + int dest_fd, const char *dest_path, + const char **error_r) +{ + struct stat dest_st; + + if (fstat(dest_fd, &dest_st) < 0) { + *error_r = t_strdup_printf("fstat(%s) failed: %m", dest_path); + return -1; + } + + if (src_st->st_gid != dest_st.st_gid && + ((src_st->st_mode & 0070) >> 3 != (src_st->st_mode & 0007))) { + /* group has different permissions from world. + preserve the group. */ + if (fchown(dest_fd, (uid_t)-1, src_st->st_gid) < 0) { + *error_r = t_strdup_printf("fchown(%s, -1, %s) failed: %m", + dest_path, dec2str(src_st->st_gid)); + return -1; + } + } + + if ((src_st->st_mode & 07777) != (dest_st.st_mode & 07777)) { + if (fchmod(dest_fd, src_st->st_mode & 07777) < 0) { + *error_r = t_strdup_printf("fchmod(%s, %o) failed: %m", + dest_path, (int)(src_st->st_mode & 0777)); + return -1; + } + } + return 0; +} + +static int fd_copy_permissions(int src_fd, const char *src_path, + int dest_fd, const char *dest_path, + const char **error_r) +{ + struct stat src_st; + + if (fstat(src_fd, &src_st) < 0) { + *error_r = t_strdup_printf("fstat(%s) failed: %m", src_path); + return -1; + } + return fd_copy_stat_permissions(&src_st, dest_fd, dest_path, error_r); +} + +static int +fd_copy_parent_dir_permissions(const char *src_path, int dest_fd, + const char *dest_path, const char **error_r) +{ + struct stat src_st; + const char *src_dir, *p; + + p = strrchr(src_path, '/'); + if (p == NULL) + src_dir = "."; + else + src_dir = t_strdup_until(src_path, p); + if (stat(src_dir, &src_st) < 0) { + *error_r = t_strdup_printf("stat(%s) failed: %m", src_dir); + return -1; + } + src_st.st_mode &= 0666; + return fd_copy_stat_permissions(&src_st, dest_fd, dest_path, error_r); +} + +static int file_dict_mkdir(struct file_dict *dict, const char **error_r) +{ + const char *path, *p, *root; + struct stat st; + mode_t mode = 0700; + + p = strrchr(dict->path, '/'); + if (p == NULL) + return 0; + path = t_strdup_until(dict->path, p); + + if (stat_first_parent(path, &root, &st) < 0) { + if (errno == EACCES) + *error_r = eacces_error_get("stat", root); + else + *error_r = t_strdup_printf("stat(%s) failed: %m", root); + return -1; + } + if ((st.st_mode & S_ISGID) != 0) { + /* preserve parent's permissions when it has setgid bit */ + mode = st.st_mode; + } + + if (mkdir_parents(path, mode) < 0 && errno != EEXIST) { + if (errno == EACCES) + *error_r = eacces_error_get("mkdir_parents", path); + else + *error_r = t_strdup_printf("mkdir_parents(%s) failed: %m", path); + return -1; + } + return 0; +} + +static int +file_dict_lock(struct file_dict *dict, struct file_lock **lock_r, + const char **error_r) +{ + int ret; + const char *error; + + if (file_dict_open_latest(dict, error_r) < 0) + return -1; + + if (dict->fd == -1) { + /* quota file doesn't exist yet, we need to create it */ + dict->fd = open(dict->path, O_CREAT | O_RDWR, 0600); + if (dict->fd == -1 && errno == ENOENT) { + if (file_dict_mkdir(dict, error_r) < 0) + return -1; + dict->fd = open(dict->path, O_CREAT | O_RDWR, 0600); + } + if (dict->fd == -1) { + if (errno == EACCES) + *error_r = eacces_error_get("creat", dict->path); + else { + *error_r = t_strdup_printf( + "creat(%s) failed: %m", dict->path); + } + return -1; + } + if (fd_copy_parent_dir_permissions(dict->path, dict->fd, + dict->path, &error) < 0) + e_error(dict->dict.event, "%s", error); + } + + *lock_r = NULL; + struct file_lock_settings lock_set = { + .lock_method = dict->lock_method, + }; + do { + file_lock_free(lock_r); + if (file_wait_lock(dict->fd, dict->path, F_WRLCK, &lock_set, + file_dict_dotlock_settings.timeout, + lock_r, &error) <= 0) { + *error_r = t_strdup_printf( + "file_wait_lock(%s) failed: %s", + dict->path, error); + return -1; + } + /* check again if we need to reopen the file because it was + just replaced */ + } while ((ret = file_dict_open_latest(dict, error_r)) > 0); + + return ret < 0 ? -1 : 0; +} + +static int +file_dict_write_changes(struct dict_transaction_memory_context *ctx, + bool *atomic_inc_not_found_r, const char **error_r) +{ + struct file_dict *dict = (struct file_dict *)ctx->ctx.dict; + struct dotlock *dotlock = NULL; + struct file_lock *lock = NULL; + const char *temp_path = NULL; + const char *error; + struct hash_iterate_context *iter; + struct ostream *output; + char *key, *value; + string_t *str; + int fd = -1; + + *atomic_inc_not_found_r = FALSE; + + if (file_dict_ensure_path_home_dir(dict, ctx->ctx.set.home_dir, error_r) < 0) + return -1; + + switch (dict->lock_method) { + case FILE_LOCK_METHOD_FCNTL: + case FILE_LOCK_METHOD_FLOCK: + if (file_dict_lock(dict, &lock, error_r) < 0) + return -1; + temp_path = t_strdup_printf("%s.tmp", dict->path); + fd = creat(temp_path, 0600); + if (fd == -1) { + *error_r = t_strdup_printf( + "dict-file: creat(%s) failed: %m", temp_path); + file_unlock(&lock); + return -1; + } + break; + case FILE_LOCK_METHOD_DOTLOCK: + fd = file_dotlock_open(&file_dict_dotlock_settings, dict->path, 0, + &dotlock); + if (fd == -1 && errno == ENOENT) { + if (file_dict_mkdir(dict, error_r) < 0) + return -1; + fd = file_dotlock_open(&file_dict_dotlock_settings, + dict->path, 0, &dotlock); + } + if (fd == -1) { + *error_r = t_strdup_printf( + "dict-file: file_dotlock_open(%s) failed: %m", + dict->path); + return -1; + } + temp_path = file_dotlock_get_lock_path(dotlock); + break; + } + + /* refresh once more now that we're locked */ + if (file_dict_refresh(dict, error_r) < 0) { + if (dotlock != NULL) + file_dotlock_delete(&dotlock); + else { + i_close_fd(&fd); + file_unlock(&lock); + } + return -1; + } + if (dict->fd != -1) { + /* preserve the permissions */ + if (fd_copy_permissions(dict->fd, dict->path, fd, temp_path, &error) < 0) + e_error(ctx->ctx.event, "%s", error); + } else { + /* get initial permissions from parent directory */ + if (fd_copy_parent_dir_permissions(dict->path, fd, temp_path, &error) < 0) + e_error(ctx->ctx.event, "%s", error); + } + file_dict_apply_changes(ctx, atomic_inc_not_found_r); + + output = o_stream_create_fd(fd, 0); + o_stream_cork(output); + iter = hash_table_iterate_init(dict->hash); + str = t_str_new(256); + while (hash_table_iterate(iter, dict->hash, &key, &value)) { + str_truncate(str, 0); + str_append_tabescaped(str, key); + str_append_c(str, '\n'); + str_append_tabescaped(str, value); + str_append_c(str, '\n'); + o_stream_nsend(output, str_data(str), str_len(str)); + } + hash_table_iterate_deinit(&iter); + + if (o_stream_finish(output) <= 0) { + *error_r = t_strdup_printf("write(%s) failed: %s", temp_path, + o_stream_get_error(output)); + o_stream_destroy(&output); + if (dotlock != NULL) + file_dotlock_delete(&dotlock); + else { + i_close_fd(&fd); + file_unlock(&lock); + } + return -1; + } + o_stream_destroy(&output); + + if (dotlock != NULL) { + if (file_dotlock_replace(&dotlock, + DOTLOCK_REPLACE_FLAG_DONT_CLOSE_FD) < 0) { + *error_r = t_strdup_printf("file_dotlock_replace() failed: %m"); + i_close_fd(&fd); + return -1; + } + } else { + if (rename(temp_path, dict->path) < 0) { + *error_r = t_strdup_printf("rename(%s, %s) failed: %m", + temp_path, dict->path); + file_unlock(&lock); + i_close_fd(&fd); + return -1; + } + /* dict->fd is locked, not the new fd. We're closing dict->fd + so we can just free the lock struct. */ + file_lock_free(&lock); + } + + i_close_fd(&dict->fd); + dict->fd = fd; + return 0; +} + +static void +file_dict_transaction_commit(struct dict_transaction_context *_ctx, + bool async ATTR_UNUSED, + dict_transaction_commit_callback_t *callback, + void *context) +{ + struct dict_transaction_memory_context *ctx = + (struct dict_transaction_memory_context *)_ctx; + struct dict_commit_result result; + bool atomic_inc_not_found; + + i_zero(&result); + if (file_dict_write_changes(ctx, &atomic_inc_not_found, &result.error) < 0) + result.ret = DICT_COMMIT_RET_FAILED; + else if (atomic_inc_not_found) + result.ret = DICT_COMMIT_RET_NOTFOUND; + else + result.ret = DICT_COMMIT_RET_OK; + pool_unref(&ctx->pool); + + callback(&result, context); +} + +struct dict dict_driver_file = { + .name = "file", + { + .init = file_dict_init, + .deinit = file_dict_deinit, + .lookup = file_dict_lookup, + .iterate_init = file_dict_iterate_init, + .iterate = file_dict_iterate, + .iterate_deinit = file_dict_iterate_deinit, + .transaction_init = file_dict_transaction_init, + .transaction_commit = file_dict_transaction_commit, + .transaction_rollback = dict_transaction_memory_rollback, + .set = dict_transaction_memory_set, + .unset = dict_transaction_memory_unset, + .atomic_inc = dict_transaction_memory_atomic_inc, + } +}; diff --git a/src/lib-dict/dict-iter-lua.c b/src/lib-dict/dict-iter-lua.c new file mode 100644 index 0000000..780de03 --- /dev/null +++ b/src/lib-dict/dict-iter-lua.c @@ -0,0 +1,193 @@ +/* Copyright (c) 2021 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "dict.h" +#include "dlua-script-private.h" +#include "dict-lua-private.h" +#include "dlua-wrapper.h" + +struct lua_dict_iter { + pool_t pool; + struct dict_iterate_context *iter; + ARRAY(int) refs; + int error_ref; + + lua_State *L; + bool yielded:1; +}; + +static void lua_dict_iter_unref(struct lua_dict_iter *iter) +{ + const char *error; + + /* deinit iteration if it hasn't been done yet */ + if (dict_iterate_deinit(&iter->iter, &error) < 0) { + e_error(dlua_script_from_state(iter->L)->event, + "Dict iteration failed: %s", error); + } + + pool_unref(&iter->pool); +} + +DLUA_WRAP_C_DATA(dict_iter, struct lua_dict_iter, lua_dict_iter_unref, NULL); + +static int lua_dict_iterate_step(lua_State *L); + +/* resume after a yield */ +static int lua_dict_iterate_step_continue(lua_State *L, + int status ATTR_UNUSED, + lua_KContext ctx ATTR_UNUSED) +{ + return lua_dict_iterate_step(L); +} + +static void lua_dict_iterate_more(struct lua_dict_iter *iter); + +/* + * Iteration step function + * + * Takes two args (a userdata state, and previous value) and returns the + * next value. + */ +static int lua_dict_iterate_step(lua_State *L) +{ + struct lua_dict_iter *iter; + const int *refs; + unsigned nrefs; + + DLUA_REQUIRE_ARGS(L, 2); + + iter = xlua_dict_iter_getptr(L, 1, NULL); + iter->yielded = FALSE; + + lua_dict_iterate_more(iter); + + if (iter->iter != NULL) { + /* iteration didn't end yet - yield */ + return lua_dict_iterate_step_continue(L, + lua_yieldk(L, 0, 0, lua_dict_iterate_step_continue), 0); + } + + /* dict iteration ended - return first key-value pair */ + refs = array_get(&iter->refs, &nrefs); + i_assert(nrefs % 2 == 0); + + if (nrefs == 0) { + if (iter->error_ref != 0) { + /* dict iteration generated an error - raise it now */ + lua_rawgeti(L, LUA_REGISTRYINDEX, iter->error_ref); + luaL_unref(L, LUA_REGISTRYINDEX, iter->error_ref); + return lua_error(L); + } + + return 0; /* return nil */ + } + + /* get the key & value from the registry */ + lua_rawgeti(L, LUA_REGISTRYINDEX, refs[0]); + lua_rawgeti(L, LUA_REGISTRYINDEX, refs[1]); + luaL_unref(L, LUA_REGISTRYINDEX, refs[0]); + luaL_unref(L, LUA_REGISTRYINDEX, refs[1]); + + array_delete(&iter->refs, 0, 2); + + return 2; +} + +static void lua_dict_iterate_more(struct lua_dict_iter *iter) +{ + const char *key, *const *values; + lua_State *L = iter->L; + const char *error; + + if (iter->iter == NULL) + return; /* done iterating the dict */ + + while (dict_iterate_values(iter->iter, &key, &values)) { + int ref; + + /* stash key */ + lua_pushstring(L, key); + ref = luaL_ref(L, LUA_REGISTRYINDEX); + array_push_back(&iter->refs, &ref); + + /* stash values */ + lua_newtable(L); + for (unsigned int i = 0; values[i] != NULL; i++) { + lua_pushstring(L, values[i]); + lua_seti(L, -2, i + 1); + } + ref = luaL_ref(L, LUA_REGISTRYINDEX); + array_push_back(&iter->refs, &ref); + } + + if (dict_iterate_has_more(iter->iter)) + return; + + if (dict_iterate_deinit(&iter->iter, &error) < 0) { + lua_pushstring(L, error); + iter->error_ref = luaL_ref(L, LUA_REGISTRYINDEX); + } +} + +/* dict iter callback */ +static void lua_dict_iterate_callback(struct lua_dict_iter *iter) +{ + if (iter->yielded) + return; + iter->yielded = TRUE; + dlua_pcall_yieldable_resume(iter->L, 1); +} + +/* + * Iterate a dict at key [-(3|4),+2,e] + * + * Args: + * 1) userdata: sturct dict *dict + * 2) string: key + * 3) integer: flags + * 4*) string: username + * + * Returns: + * Returns a iteration step function and dict iter userdata. + * Username will be NULL if not provided in args. + */ +int lua_dict_iterate(lua_State *L) +{ + enum dict_iterate_flags flags; + struct lua_dict_iter *iter; + struct dict *dict; + const char *path, *username = NULL; + pool_t pool; + + DLUA_REQUIRE_ARGS_IN(L, 3, 4); + + dict = dlua_check_dict(L, 1); + path = luaL_checkstring(L, 2); + flags = luaL_checkinteger(L, 3); + if (lua_gettop(L) >= 4) + username = luaL_checkstring(L, 4); + lua_dict_check_key_prefix(L, path, username); + + struct dict_op_settings set = { + .username = username, + }; + + /* set up iteration */ + pool = pool_alloconly_create("lua dict iter", 128); + iter = p_new(pool, struct lua_dict_iter, 1); + iter->pool = pool; + iter->iter = dict_iterate_init(dict, &set, path, flags | + DICT_ITERATE_FLAG_ASYNC); + p_array_init(&iter->refs, iter->pool, 32); + iter->L = L; + + dict_iterate_set_async_callback(iter->iter, + lua_dict_iterate_callback, iter); + + /* push return values: func, state */ + lua_pushcfunction(L, lua_dict_iterate_step); + xlua_pushdict_iter(L, iter, FALSE); + return 2; +} diff --git a/src/lib-dict/dict-lua-private.h b/src/lib-dict/dict-lua-private.h new file mode 100644 index 0000000..f9f9943 --- /dev/null +++ b/src/lib-dict/dict-lua-private.h @@ -0,0 +1,9 @@ +#ifndef DICT_LUA_PRIVATE_H +#define DICT_LUA_PRIVATE_H + +#include "dict-lua.h" + +int lua_dict_iterate(lua_State *l); +int lua_dict_transaction_begin(lua_State *l); + +#endif diff --git a/src/lib-dict/dict-lua.c b/src/lib-dict/dict-lua.c new file mode 100644 index 0000000..d5de534 --- /dev/null +++ b/src/lib-dict/dict-lua.c @@ -0,0 +1,117 @@ +/* Copyright (c) 2021 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "dict.h" +#include "dlua-script-private.h" +#include "dict-lua-private.h" +#include "dlua-wrapper.h" + +static int lua_dict_lookup(lua_State *); + +static luaL_Reg lua_dict_methods[] = { + { "lookup", lua_dict_lookup }, + { "iterate", lua_dict_iterate }, + { "transaction_begin", lua_dict_transaction_begin }, + { NULL, NULL }, +}; + +/* no actual ref counting */ +static void lua_dict_unref(struct dict *dict ATTR_UNUSED) +{ +} + +DLUA_WRAP_C_DATA(dict, struct dict, lua_dict_unref, lua_dict_methods); + +static int lua_dict_async_continue(lua_State *L, + int status ATTR_UNUSED, + lua_KContext ctx ATTR_UNUSED) +{ + /* + * lua_dict_*_callback() already pushed the result table/nil or error + * string. We simply need to return/error out. + */ + + if (lua_istable(L, -1) || lua_isnil(L, -1)) + return 1; + else + return lua_error(L); +} + +static void lua_dict_lookup_callback(const struct dict_lookup_result *result, + lua_State *L) +{ + if (result->ret < 0) { + lua_pushstring(L, result->error); + } else if (result->ret == 0) { + lua_pushnil(L); + } else { + unsigned int i; + + lua_newtable(L); + + for (i = 0; i < str_array_length(result->values); i++) { + lua_pushstring(L, result->values[i]); + lua_seti(L, -2, i + 1); + } + } + + dlua_pcall_yieldable_resume(L, 1); +} + +void lua_dict_check_key_prefix(lua_State *L, const char *key, + const char *username) +{ + if (str_begins(key, DICT_PATH_SHARED)) + ; + else if (str_begins(key, DICT_PATH_PRIVATE)) { + if (username == NULL || username[0] == '\0') + luaL_error(L, DICT_PATH_PRIVATE" dict key prefix requires username"); + } else { + luaL_error(L, "Invalid dict key prefix"); + } +} + +/* + * Lookup a key in dict [-(2|3),+1,e] + * + * Args: + * 1) userdata: struct dict *dict + * 2) string: key + * 3*) string: username + * + * Returns: + * If key is found, returns a table with values. If key is not found, + * returns nil. + * Username will be NULL if not provided in args. + */ +static int lua_dict_lookup(lua_State *L) +{ + struct dict *dict; + const char *key, *username = NULL; + + DLUA_REQUIRE_ARGS_IN(L, 2, 3); + + dict = xlua_dict_getptr(L, 1, NULL); + key = luaL_checkstring(L, 2); + if (lua_gettop(L) >= 3) + username = luaL_checkstring(L, 3); + lua_dict_check_key_prefix(L, key, username); + + struct dict_op_settings set = { + .username = username, + }; + dict_lookup_async(dict, &set, key, lua_dict_lookup_callback, L); + + return lua_dict_async_continue(L, + lua_yieldk(L, 0, 0, lua_dict_async_continue), 0); +} + +void dlua_push_dict(lua_State *L, struct dict *dict) +{ + xlua_pushdict(L, dict, FALSE); +} + +struct dict *dlua_check_dict(lua_State *L, int idx) +{ + return xlua_dict_getptr(L, idx, NULL); +} diff --git a/src/lib-dict/dict-lua.h b/src/lib-dict/dict-lua.h new file mode 100644 index 0000000..bf4255c --- /dev/null +++ b/src/lib-dict/dict-lua.h @@ -0,0 +1,18 @@ +#ifndef DICT_LUA_H +#define DICT_LUA_H + +#ifdef DLUA_WITH_YIELDS +/* + * Internally, the dict methods yield via lua_yieldk() as implemented in Lua + * 5.3 and newer. + */ + +void lua_dict_check_key_prefix(lua_State *L, const char *key, + const char *username); + +void dlua_push_dict(lua_State *L, struct dict *dict); +struct dict *dlua_check_dict(lua_State *L, int idx); + +#endif + +#endif diff --git a/src/lib-dict/dict-memcached-ascii.c b/src/lib-dict/dict-memcached-ascii.c new file mode 100644 index 0000000..6ae5443 --- /dev/null +++ b/src/lib-dict/dict-memcached-ascii.c @@ -0,0 +1,685 @@ +/* Copyright (c) 2008-2018 Dovecot authors, see the included COPYING memcached_ascii */ + +#include "lib.h" +#include "array.h" +#include "str.h" +#include "istream.h" +#include "ostream.h" +#include "connection.h" +#include "dict-transaction-memory.h" +#include "dict-private.h" + +#define MEMCACHED_DEFAULT_PORT 11211 +#define MEMCACHED_DEFAULT_LOOKUP_TIMEOUT_MSECS (1000*30) +#define DICT_USERNAME_SEPARATOR '/' + +enum memcached_ascii_input_state { + /* GET: expecting VALUE or END */ + MEMCACHED_INPUT_STATE_GET, + /* SET: expecting STORED / NOT_STORED */ + MEMCACHED_INPUT_STATE_STORED, + /* DELETE: expecting DELETED */ + MEMCACHED_INPUT_STATE_DELETED, + /* (INCR+ADD)/DECR: expecting number / NOT_FOUND / STORED / NOT_STORED */ + MEMCACHED_INPUT_STATE_INCRDECR +}; + +struct memcached_ascii_connection { + struct connection conn; + struct memcached_ascii_dict *dict; + + string_t *reply_str; + unsigned int reply_bytes_left; + bool value_received; + bool value_waiting_end; +}; + +struct memcached_ascii_dict_reply { + unsigned int reply_count; + dict_transaction_commit_callback_t *callback; + void *context; +}; + +struct dict_memcached_ascii_commit_ctx { + struct memcached_ascii_dict *dict; + struct dict_transaction_memory_context *memctx; + string_t *str; + + dict_transaction_commit_callback_t *callback; + void *context; +}; + +struct memcached_ascii_dict { + struct dict dict; + struct ip_addr ip; + char *key_prefix; + in_port_t port; + unsigned int timeout_msecs; + + struct timeout *to; + struct memcached_ascii_connection conn; + + ARRAY(enum memcached_ascii_input_state) input_states; + ARRAY(struct memcached_ascii_dict_reply) replies; +}; + +static struct connection_list *memcached_ascii_connections; + +static void +memcached_ascii_callback(struct memcached_ascii_dict *dict, + const struct memcached_ascii_dict_reply *reply, + const struct dict_commit_result *result) +{ + if (reply->callback != NULL) { + if (dict->dict.prev_ioloop != NULL) { + /* Don't let callback see that we've created our + internal ioloop in case it wants to add some ios + or timeouts. */ + current_ioloop = dict->dict.prev_ioloop; + } + reply->callback(result, reply->context); + if (dict->dict.prev_ioloop != NULL) + current_ioloop = dict->dict.ioloop; + } +} + +static void +memcached_ascii_disconnected(struct memcached_ascii_connection *conn, + const char *reason) +{ + const struct dict_commit_result result = { + DICT_COMMIT_RET_FAILED, reason + }; + const struct memcached_ascii_dict_reply *reply; + + connection_disconnect(&conn->conn); + if (conn->dict->dict.ioloop != NULL) + io_loop_stop(conn->dict->dict.ioloop); + + array_foreach(&conn->dict->replies, reply) + memcached_ascii_callback(conn->dict, reply, &result); + array_clear(&conn->dict->replies); + array_clear(&conn->dict->input_states); + conn->reply_bytes_left = 0; +} + +static void memcached_ascii_conn_destroy(struct connection *_conn) +{ + struct memcached_ascii_connection *conn = + (struct memcached_ascii_connection *)_conn; + + memcached_ascii_disconnected(conn, connection_disconnect_reason(_conn)); +} + +static bool memcached_ascii_input_value(struct memcached_ascii_connection *conn) +{ + const unsigned char *data; + size_t size; + + data = i_stream_get_data(conn->conn.input, &size); + if (size > conn->reply_bytes_left) + size = conn->reply_bytes_left; + conn->reply_bytes_left -= size; + + str_append_data(conn->reply_str, data, size); + i_stream_skip(conn->conn.input, size); + if (conn->reply_bytes_left > 0) + return FALSE; + + /* finished. drop the trailing CRLF */ + str_truncate(conn->reply_str, str_len(conn->reply_str)-2); + conn->value_received = TRUE; + return TRUE; +} + +static int memcached_ascii_input_reply_read(struct memcached_ascii_dict *dict, + const char **error_r) +{ + struct memcached_ascii_connection *conn = &dict->conn; + const enum memcached_ascii_input_state *states; + const char *line, *p; + unsigned int count; + long long num; + + if (conn->reply_bytes_left > 0) { + /* continue reading bulk reply */ + if (!memcached_ascii_input_value(conn)) + return 0; + conn->value_waiting_end = TRUE; + } else if (conn->value_waiting_end) { + conn->value_waiting_end = FALSE; + } else { + str_truncate(conn->reply_str, 0); + conn->value_received = FALSE; + } + + line = i_stream_next_line(conn->conn.input); + if (line == NULL) + return 0; + + states = array_get(&dict->input_states, &count); + if (count == 0) { + *error_r = t_strdup_printf( + "memcached_ascii: Unexpected input (expected nothing): %s", line); + return -1; + } + switch (states[0]) { + case MEMCACHED_INPUT_STATE_GET: + /* VALUE <key> <flags> <bytes> + END */ + if (str_begins(line, "VALUE ")) { + p = strrchr(line, ' '); + if (str_to_uint(p+1, &conn->reply_bytes_left) < 0) + break; + conn->reply_bytes_left += 2; /* CRLF */ + return memcached_ascii_input_reply_read(dict, error_r); + } else if (strcmp(line, "END") == 0) + return 1; + break; + case MEMCACHED_INPUT_STATE_STORED: + if (strcmp(line, "STORED") != 0 && + strcmp(line, "NOT_STORED") != 0) + break; + return 1; + case MEMCACHED_INPUT_STATE_DELETED: + if (strcmp(line, "DELETED") != 0) + break; + return 1; + case MEMCACHED_INPUT_STATE_INCRDECR: + if (strcmp(line, "NOT_FOUND") != 0 && + strcmp(line, "STORED") != 0 && + strcmp(line, "NOT_STORED") != 0 && + str_to_llong(line, &num) < 0) + break; + return 1; + } + *error_r = t_strdup_printf( + "memcached_ascii: Unexpected input (state=%d): %s", + states[0], line); + return -1; +} + +static int memcached_ascii_input_reply(struct memcached_ascii_dict *dict, + const char **error_r) +{ + const struct dict_commit_result result = { + DICT_COMMIT_RET_OK, NULL + }; + struct memcached_ascii_dict_reply *replies; + unsigned int count; + int ret; + + if ((ret = memcached_ascii_input_reply_read(dict, error_r)) <= 0) + return ret; + /* finished a reply */ + array_pop_front(&dict->input_states); + + replies = array_get_modifiable(&dict->replies, &count); + i_assert(count > 0); + i_assert(replies[0].reply_count > 0); + if (--replies[0].reply_count == 0) { + memcached_ascii_callback(dict, &replies[0], &result); + array_pop_front(&dict->replies); + } + return 1; +} + +static void memcached_ascii_conn_input(struct connection *_conn) +{ + struct memcached_ascii_connection *conn = + (struct memcached_ascii_connection *)_conn; + const char *error; + int ret; + + switch (i_stream_read(_conn->input)) { + case 0: + return; + case -1: + memcached_ascii_disconnected(conn, + i_stream_get_disconnect_reason(_conn->input)); + return; + default: + break; + } + + while ((ret = memcached_ascii_input_reply(conn->dict, &error)) > 0) ; + if (ret < 0) + memcached_ascii_disconnected(conn, error); + io_loop_stop(conn->dict->dict.ioloop); +} + +static int memcached_ascii_input_wait(struct memcached_ascii_dict *dict, + const char **error_r) +{ + i_assert(io_loop_is_empty(dict->dict.ioloop)); + dict->dict.prev_ioloop = current_ioloop; + io_loop_set_current(dict->dict.ioloop); + if (dict->to != NULL) + dict->to = io_loop_move_timeout(&dict->to); + connection_switch_ioloop(&dict->conn.conn); + io_loop_run(dict->dict.ioloop); + + io_loop_set_current(dict->dict.prev_ioloop); + dict->dict.prev_ioloop = NULL; + + if (dict->to != NULL) + dict->to = io_loop_move_timeout(&dict->to); + connection_switch_ioloop(&dict->conn.conn); + i_assert(io_loop_is_empty(dict->dict.ioloop)); + + if (dict->conn.conn.fd_in == -1) { + *error_r = "memcached_ascii: Communication failure"; + return -1; + } + return 0; +} + +static void memcached_ascii_input_timeout(struct memcached_ascii_dict *dict) +{ + const char *reason = t_strdup_printf( + "memcached_ascii: Request timed out in %u.%03u secs", + dict->timeout_msecs/1000, dict->timeout_msecs%1000); + memcached_ascii_disconnected(&dict->conn, reason); +} + +static int memcached_ascii_wait_replies(struct memcached_ascii_dict *dict, + const char **error_r) +{ + int ret = 0; + + dict->to = timeout_add(dict->timeout_msecs, + memcached_ascii_input_timeout, dict); + while (array_count(&dict->input_states) > 0) { + i_assert(array_count(&dict->replies) > 0); + + if ((ret = memcached_ascii_input_reply(dict, error_r)) != 0) { + if (ret < 0) + memcached_ascii_disconnected(&dict->conn, *error_r); + break; + } + ret = memcached_ascii_input_wait(dict, error_r); + if (ret != 0) + break; + } + + timeout_remove(&dict->to); + return ret < 0 ? -1 : 0; +} + +static int memcached_ascii_wait(struct memcached_ascii_dict *dict, + const char **error_r) +{ + int ret; + + i_assert(dict->conn.conn.fd_in != -1); + + if (dict->conn.conn.input == NULL) { + /* waiting for connection to finish */ + dict->to = timeout_add(dict->timeout_msecs, + memcached_ascii_input_timeout, dict); + ret = memcached_ascii_input_wait(dict, error_r); + timeout_remove(&dict->to); + if (ret < 0) + return -1; + } + if (memcached_ascii_wait_replies(dict, error_r) < 0) + return -1; + i_assert(array_count(&dict->input_states) == 0); + i_assert(array_count(&dict->replies) == 0); + return 0; +} + +static void +memcached_ascii_conn_connected(struct connection *_conn, bool success) +{ + struct memcached_ascii_connection *conn = (struct memcached_ascii_connection *)_conn; + + if (!success) { + e_error(conn->conn.event, "connect() failed: %m"); + } + if (conn->dict->dict.ioloop != NULL) + io_loop_stop(conn->dict->dict.ioloop); +} + +static const struct connection_settings memcached_ascii_conn_set = { + .input_max_size = SIZE_MAX, + .output_max_size = SIZE_MAX, + .client = TRUE +}; + +static const struct connection_vfuncs memcached_ascii_conn_vfuncs = { + .destroy = memcached_ascii_conn_destroy, + .input = memcached_ascii_conn_input, + .client_connected = memcached_ascii_conn_connected +}; + +static const char *memcached_ascii_escape_username(const char *username) +{ + const char *p; + string_t *str = t_str_new(64); + + for (p = username; *p != '\0'; p++) { + switch (*p) { + case DICT_USERNAME_SEPARATOR: + str_append(str, "\\-"); + break; + case '\\': + str_append(str, "\\\\"); + break; + default: + str_append_c(str, *p); + } + } + return str_c(str); +} + +static int +memcached_ascii_dict_init(struct dict *driver, const char *uri, + const struct dict_settings *set, + struct dict **dict_r, const char **error_r) +{ + struct memcached_ascii_dict *dict; + const char *const *args; + struct ioloop *old_ioloop = current_ioloop; + int ret = 0; + + if (memcached_ascii_connections == NULL) { + memcached_ascii_connections = + connection_list_init(&memcached_ascii_conn_set, + &memcached_ascii_conn_vfuncs); + } + + dict = i_new(struct memcached_ascii_dict, 1); + if (net_addr2ip("127.0.0.1", &dict->ip) < 0) + i_unreached(); + dict->port = MEMCACHED_DEFAULT_PORT; + dict->timeout_msecs = MEMCACHED_DEFAULT_LOOKUP_TIMEOUT_MSECS; + dict->key_prefix = i_strdup(""); + + args = t_strsplit(uri, ":"); + for (; *args != NULL; args++) { + if (str_begins(*args, "host=")) { + if (net_addr2ip(*args+5, &dict->ip) < 0) { + *error_r = t_strdup_printf("Invalid IP: %s", + *args+5); + ret = -1; + } + } else if (str_begins(*args, "port=")) { + if (net_str2port(*args+5, &dict->port) < 0) { + *error_r = t_strdup_printf("Invalid port: %s", + *args+5); + ret = -1; + } + } else if (str_begins(*args, "prefix=")) { + i_free(dict->key_prefix); + dict->key_prefix = i_strdup(*args + 7); + } else if (str_begins(*args, "timeout_msecs=")) { + if (str_to_uint(*args+14, &dict->timeout_msecs) < 0) { + *error_r = t_strdup_printf( + "Invalid timeout_msecs: %s", *args+14); + ret = -1; + } + } else { + *error_r = t_strdup_printf("Unknown parameter: %s", + *args); + ret = -1; + } + } + if (ret < 0) { + i_free(dict->key_prefix); + i_free(dict); + return -1; + } + + dict->conn.conn.event_parent = set->event_parent; + connection_init_client_ip(memcached_ascii_connections, &dict->conn.conn, + NULL, &dict->ip, dict->port); + event_set_append_log_prefix(dict->conn.conn.event, "memcached: "); + dict->dict = *driver; + dict->conn.reply_str = str_new(default_pool, 256); + dict->conn.dict = dict; + + i_array_init(&dict->input_states, 4); + i_array_init(&dict->replies, 4); + + dict->dict.ioloop = io_loop_create(); + io_loop_set_current(old_ioloop); + *dict_r = &dict->dict; + return 0; +} + +static void memcached_ascii_dict_deinit(struct dict *_dict) +{ + struct memcached_ascii_dict *dict = + (struct memcached_ascii_dict *)_dict; + struct ioloop *old_ioloop = current_ioloop; + const char *error; + + if (array_count(&dict->input_states) > 0) { + if (memcached_ascii_wait(dict, &error) < 0) + i_error("%s", error); + } + connection_deinit(&dict->conn.conn); + + io_loop_set_current(dict->dict.ioloop); + io_loop_destroy(&dict->dict.ioloop); + io_loop_set_current(old_ioloop); + + str_free(&dict->conn.reply_str); + array_free(&dict->replies); + array_free(&dict->input_states); + i_free(dict->key_prefix); + i_free(dict); + + if (memcached_ascii_connections->connections == NULL) + connection_list_deinit(&memcached_ascii_connections); +} + +static int memcached_ascii_connect(struct memcached_ascii_dict *dict, + const char **error_r) +{ + if (dict->conn.conn.input != NULL) + return 0; + + if (dict->conn.conn.fd_in == -1) { + if (connection_client_connect(&dict->conn.conn) < 0) { + *error_r = t_strdup_printf( + "memcached_ascii: Couldn't connect to %s:%u", + net_ip2addr(&dict->ip), dict->port); + return -1; + } + } + return memcached_ascii_wait(dict, error_r); +} + +static const char * +memcached_ascii_dict_get_full_key(struct memcached_ascii_dict *dict, + const char *username, const char *key) +{ + if (str_begins(key, DICT_PATH_SHARED)) + key += strlen(DICT_PATH_SHARED); + else if (str_begins(key, DICT_PATH_PRIVATE)) { + if (strchr(username, DICT_USERNAME_SEPARATOR) == NULL) { + key = t_strdup_printf("%s%c%s", username, + DICT_USERNAME_SEPARATOR, + key + strlen(DICT_PATH_PRIVATE)); + } else { + /* escape the username */ + key = t_strdup_printf("%s%c%s", memcached_ascii_escape_username(username), + DICT_USERNAME_SEPARATOR, + key + strlen(DICT_PATH_PRIVATE)); + } + } else { + i_unreached(); + } + if (*dict->key_prefix != '\0') + key = t_strconcat(dict->key_prefix, key, NULL); + return key; +} + +static int +memcached_ascii_dict_lookup(struct dict *_dict, + const struct dict_op_settings *set, + pool_t pool, const char *key, const char **value_r, + const char **error_r) +{ + struct memcached_ascii_dict *dict = (struct memcached_ascii_dict *)_dict; + struct memcached_ascii_dict_reply *reply; + enum memcached_ascii_input_state state = MEMCACHED_INPUT_STATE_GET; + + if (memcached_ascii_connect(dict, error_r) < 0) + return -1; + + key = memcached_ascii_dict_get_full_key(dict, set->username, key); + o_stream_nsend_str(dict->conn.conn.output, + t_strdup_printf("get %s\r\n", key)); + array_push_back(&dict->input_states, &state); + + reply = array_append_space(&dict->replies); + reply->reply_count = 1; + + if (memcached_ascii_wait(dict, error_r) < 0) + return -1; + + *value_r = p_strdup(pool, str_c(dict->conn.reply_str)); + return dict->conn.value_received ? 1 : 0; +} + +static struct dict_transaction_context * +memcached_ascii_transaction_init(struct dict *_dict) +{ + struct dict_transaction_memory_context *ctx; + pool_t pool; + + pool = pool_alloconly_create("file dict transaction", 2048); + ctx = p_new(pool, struct dict_transaction_memory_context, 1); + dict_transaction_memory_init(ctx, _dict, pool); + return &ctx->ctx; +} + +static void +memcached_send_change(struct dict_memcached_ascii_commit_ctx *ctx, + const struct dict_op_settings_private *set, + const struct dict_transaction_memory_change *change) +{ + enum memcached_ascii_input_state state; + const char *key, *value; + + key = memcached_ascii_dict_get_full_key(ctx->dict, set->username, + change->key); + + str_truncate(ctx->str, 0); + switch (change->type) { + case DICT_CHANGE_TYPE_SET: + state = MEMCACHED_INPUT_STATE_STORED; + str_printfa(ctx->str, "set %s 0 0 %zu\r\n%s\r\n", + key, strlen(change->value.str), change->value.str); + break; + case DICT_CHANGE_TYPE_UNSET: + state = MEMCACHED_INPUT_STATE_DELETED; + str_printfa(ctx->str, "delete %s\r\n", key); + break; + case DICT_CHANGE_TYPE_INC: + state = MEMCACHED_INPUT_STATE_INCRDECR; + if (change->value.diff > 0) { + str_printfa(ctx->str, "incr %s %lld\r\n", + key, change->value.diff); + array_push_back(&ctx->dict->input_states, &state); + /* same kludge as with append */ + value = t_strdup_printf("%lld", change->value.diff); + str_printfa(ctx->str, "add %s 0 0 %u\r\n%s\r\n", + key, (unsigned int)strlen(value), value); + } else { + str_printfa(ctx->str, "decr %s %lld\r\n", + key, -change->value.diff); + } + break; + } + array_push_back(&ctx->dict->input_states, &state); + o_stream_nsend(ctx->dict->conn.conn.output, + str_data(ctx->str), str_len(ctx->str)); +} + +static int +memcached_ascii_transaction_send(struct dict_memcached_ascii_commit_ctx *ctx, + const struct dict_op_settings_private *set, + const char **error_r) +{ + struct memcached_ascii_dict *dict = ctx->dict; + struct memcached_ascii_dict_reply *reply; + const struct dict_transaction_memory_change *changes; + unsigned int i, count, old_state_count; + + if (memcached_ascii_connect(dict, error_r) < 0) + return -1; + + old_state_count = array_count(&dict->input_states); + changes = array_get(&ctx->memctx->changes, &count); + i_assert(count > 0); + + o_stream_cork(dict->conn.conn.output); + for (i = 0; i < count; i++) T_BEGIN { + memcached_send_change(ctx, set, &changes[i]); + } T_END; + o_stream_uncork(dict->conn.conn.output); + + reply = array_append_space(&dict->replies); + reply->callback = ctx->callback; + reply->context = ctx->context; + reply->reply_count = array_count(&dict->input_states) - old_state_count; + return 0; +} + +static void +memcached_ascii_transaction_commit(struct dict_transaction_context *_ctx, + bool async, + dict_transaction_commit_callback_t *callback, + void *context) +{ + struct dict_transaction_memory_context *ctx = + (struct dict_transaction_memory_context *)_ctx; + struct memcached_ascii_dict *dict = + (struct memcached_ascii_dict *)_ctx->dict; + struct dict_memcached_ascii_commit_ctx commit_ctx; + struct dict_commit_result result = { DICT_COMMIT_RET_OK, NULL }; + const struct dict_op_settings_private *set = &_ctx->set; + + if (_ctx->changed) { + i_zero(&commit_ctx); + commit_ctx.dict = dict; + commit_ctx.memctx = ctx; + commit_ctx.callback = callback; + commit_ctx.context = context; + commit_ctx.str = str_new(default_pool, 128); + + result.ret = memcached_ascii_transaction_send(&commit_ctx, set, &result.error); + str_free(&commit_ctx.str); + + if (async && result.ret == 0) { + pool_unref(&ctx->pool); + return; + } + + if (result.ret == 0) { + if (memcached_ascii_wait(dict, &result.error) < 0) + result.ret = -1; + } + } + callback(&result, context); + pool_unref(&ctx->pool); +} + +struct dict dict_driver_memcached_ascii = { + .name = "memcached_ascii", + { + .init = memcached_ascii_dict_init, + .deinit = memcached_ascii_dict_deinit, + .lookup = memcached_ascii_dict_lookup, + .transaction_init = memcached_ascii_transaction_init, + .transaction_commit = memcached_ascii_transaction_commit, + .transaction_rollback = dict_transaction_memory_rollback, + .set = dict_transaction_memory_set, + .unset = dict_transaction_memory_unset, + .atomic_inc = dict_transaction_memory_atomic_inc, + } +}; diff --git a/src/lib-dict/dict-memcached.c b/src/lib-dict/dict-memcached.c new file mode 100644 index 0000000..c5b7ce6 --- /dev/null +++ b/src/lib-dict/dict-memcached.c @@ -0,0 +1,373 @@ +/* Copyright (c) 2013-2018 Dovecot authors, see the included COPYING memcached */ + +#include "lib.h" +#include "array.h" +#include "str.h" +#include "istream.h" +#include "ostream.h" +#include "connection.h" +#include "dict-private.h" + +#define MEMCACHED_DEFAULT_PORT 11211 +#define MEMCACHED_DEFAULT_LOOKUP_TIMEOUT_MSECS (1000*30) + +/* we need only very limited memcached functionality, so just define the binary + protocol ourself instead requiring protocol_binary.h */ +#define MEMCACHED_REQUEST_HDR_MAGIC 0x80 +#define MEMCACHED_REPLY_HDR_MAGIC 0x81 + +#define MEMCACHED_REQUEST_HDR_LENGTH 24 +#define MEMCACHED_REPLY_HDR_LENGTH 24 + +#define MEMCACHED_CMD_GET 0x00 + +#define MEMCACHED_DATA_TYPE_RAW 0x00 + +enum memcached_response { + MEMCACHED_RESPONSE_OK = 0x0000, + MEMCACHED_RESPONSE_NOTFOUND = 0x0001, + MEMCACHED_RESPONSE_INTERNALERROR= 0x0084, + MEMCACHED_RESPONSE_BUSY = 0x0085, + MEMCACHED_RESPONSE_TEMPFAILURE = 0x0086, +}; + +struct memcached_connection { + struct connection conn; + struct memcached_dict *dict; + + buffer_t *cmd; + struct { + const unsigned char *value; + size_t value_len; + uint16_t status; /* enum memcached_response */ + bool reply_received; + } reply; +}; + +struct memcached_dict { + struct dict dict; + struct ip_addr ip; + char *key_prefix; + in_port_t port; + unsigned int timeout_msecs; + + struct memcached_connection conn; + + bool connected; +}; + +static struct connection_list *memcached_connections; + +static void memcached_conn_destroy(struct connection *_conn) +{ + struct memcached_connection *conn = (struct memcached_connection *)_conn; + + conn->dict->connected = FALSE; + connection_disconnect(_conn); + + if (conn->dict->dict.ioloop != NULL) + io_loop_stop(conn->dict->dict.ioloop); +} + +static int memcached_input_get(struct memcached_connection *conn) +{ + const unsigned char *data; + size_t size; + uint32_t body_len, value_pos; + uint16_t key_len, key_pos, status; + uint8_t extras_len, data_type; + + data = i_stream_get_data(conn->conn.input, &size); + if (size < MEMCACHED_REPLY_HDR_LENGTH) + return 0; + + if (data[0] != MEMCACHED_REPLY_HDR_MAGIC) { + e_error(conn->conn.event, "Invalid reply magic: %u != %u", + data[0], MEMCACHED_REPLY_HDR_MAGIC); + return -1; + } + memcpy(&body_len, data+8, 4); body_len = ntohl(body_len); + body_len += MEMCACHED_REPLY_HDR_LENGTH; + if (size < body_len) { + /* we haven't read the whole response yet */ + return 0; + } + + memcpy(&key_len, data+2, 2); key_len = ntohs(key_len); + extras_len = data[4]; + data_type = data[5]; + memcpy(&status, data+6, 2); status = ntohs(status); + if (data_type != MEMCACHED_DATA_TYPE_RAW) { + e_error(conn->conn.event, "Unsupported data type: %u != %u", + data[0], MEMCACHED_DATA_TYPE_RAW); + return -1; + } + + key_pos = MEMCACHED_REPLY_HDR_LENGTH + extras_len; + value_pos = key_pos + key_len; + if (value_pos > body_len) { + e_error(conn->conn.event, "Invalid key/extras lengths"); + return -1; + } + conn->reply.value = data + value_pos; + conn->reply.value_len = body_len - value_pos; + conn->reply.status = status; + + i_stream_skip(conn->conn.input, body_len); + conn->reply.reply_received = TRUE; + + if (conn->dict->dict.ioloop != NULL) + io_loop_stop(conn->dict->dict.ioloop); + return 1; +} + +static void memcached_conn_input(struct connection *_conn) +{ + struct memcached_connection *conn = (struct memcached_connection *)_conn; + + switch (i_stream_read(_conn->input)) { + case 0: + return; + case -1: + memcached_conn_destroy(_conn); + return; + default: + break; + } + + if (memcached_input_get(conn) < 0) + memcached_conn_destroy(_conn); +} + +static void memcached_conn_connected(struct connection *_conn, bool success) +{ + struct memcached_connection *conn = + (struct memcached_connection *)_conn; + + if (!success) { + e_error(conn->conn.event, "connect() failed: %m"); + } else { + conn->dict->connected = TRUE; + } + if (conn->dict->dict.ioloop != NULL) + io_loop_stop(conn->dict->dict.ioloop); +} + +static const struct connection_settings memcached_conn_set = { + .input_max_size = SIZE_MAX, + .output_max_size = SIZE_MAX, + .client = TRUE +}; + +static const struct connection_vfuncs memcached_conn_vfuncs = { + .destroy = memcached_conn_destroy, + .input = memcached_conn_input, + .client_connected = memcached_conn_connected +}; + +static int +memcached_dict_init(struct dict *driver, const char *uri, + const struct dict_settings *set, + struct dict **dict_r, const char **error_r) +{ + struct memcached_dict *dict; + const char *const *args; + int ret = 0; + + if (memcached_connections == NULL) { + memcached_connections = + connection_list_init(&memcached_conn_set, + &memcached_conn_vfuncs); + } + + dict = i_new(struct memcached_dict, 1); + if (net_addr2ip("127.0.0.1", &dict->ip) < 0) + i_unreached(); + dict->port = MEMCACHED_DEFAULT_PORT; + dict->timeout_msecs = MEMCACHED_DEFAULT_LOOKUP_TIMEOUT_MSECS; + dict->key_prefix = i_strdup(""); + + args = t_strsplit(uri, ":"); + for (; *args != NULL; args++) { + if (str_begins(*args, "host=")) { + if (net_addr2ip(*args+5, &dict->ip) < 0) { + *error_r = t_strdup_printf("Invalid IP: %s", + *args+5); + ret = -1; + } + } else if (str_begins(*args, "port=")) { + if (net_str2port(*args+5, &dict->port) < 0) { + *error_r = t_strdup_printf("Invalid port: %s", + *args+5); + ret = -1; + } + } else if (str_begins(*args, "prefix=")) { + i_free(dict->key_prefix); + dict->key_prefix = i_strdup(*args + 7); + } else if (str_begins(*args, "timeout_msecs=")) { + if (str_to_uint(*args+14, &dict->timeout_msecs) < 0) { + *error_r = t_strdup_printf( + "Invalid timeout_msecs: %s", *args+14); + ret = -1; + } + } else { + *error_r = t_strdup_printf("Unknown parameter: %s", + *args); + ret = -1; + } + } + if (ret < 0) { + i_free(dict->key_prefix); + i_free(dict); + return -1; + } + + dict->conn.conn.event_parent = set->event_parent; + + connection_init_client_ip(memcached_connections, &dict->conn.conn, + NULL, &dict->ip, dict->port); + event_set_append_log_prefix(dict->conn.conn.event, "memcached: "); + dict->dict = *driver; + dict->conn.cmd = buffer_create_dynamic(default_pool, 256); + dict->conn.dict = dict; + *dict_r = &dict->dict; + return 0; +} + +static void memcached_dict_deinit(struct dict *_dict) +{ + struct memcached_dict *dict = (struct memcached_dict *)_dict; + + connection_deinit(&dict->conn.conn); + buffer_free(&dict->conn.cmd); + i_free(dict->key_prefix); + i_free(dict); + + if (memcached_connections->connections == NULL) + connection_list_deinit(&memcached_connections); +} + +static void memcached_dict_lookup_timeout(struct memcached_dict *dict) +{ + e_error(dict->dict.event, "Lookup timed out in %u.%03u secs", + dict->timeout_msecs/1000, dict->timeout_msecs%1000); + io_loop_stop(dict->dict.ioloop); +} + +static void memcached_add_header(buffer_t *buf, unsigned int key_len) +{ + uint32_t body_len = htonl(key_len); + + i_assert(key_len <= 0xffff); + + buffer_append_c(buf, MEMCACHED_REQUEST_HDR_MAGIC); + buffer_append_c(buf, MEMCACHED_CMD_GET); + buffer_append_c(buf, (key_len >> 8) & 0xff); + buffer_append_c(buf, key_len & 0xff); + buffer_append_c(buf, 0); /* extras length */ + buffer_append_c(buf, MEMCACHED_DATA_TYPE_RAW); + buffer_append_zero(buf, 2); /* vbucket id - we probably don't care? */ + buffer_append(buf, &body_len, sizeof(body_len)); + buffer_append_zero(buf, 4+8); /* opaque + cas */ + i_assert(buf->used == MEMCACHED_REQUEST_HDR_LENGTH); +} + +static int +memcached_dict_lookup(struct dict *_dict, const struct dict_op_settings *set ATTR_UNUSED, + pool_t pool, const char *key, const char **value_r, + const char **error_r) +{ + struct memcached_dict *dict = (struct memcached_dict *)_dict; + struct ioloop *prev_ioloop = current_ioloop; + struct timeout *to; + size_t key_len; + + if (str_begins(key, DICT_PATH_SHARED)) + key += strlen(DICT_PATH_SHARED); + else { + *error_r = t_strdup_printf("memcached: Only shared keys supported currently"); + return -1; + } + if (*dict->key_prefix != '\0') + key = t_strconcat(dict->key_prefix, key, NULL); + key_len = strlen(key); + if (key_len > 0xffff) { + *error_r = t_strdup_printf( + "memcached: Key is too long (%zu bytes): %s", key_len, key); + return -1; + } + + i_assert(dict->dict.ioloop == NULL); + + dict->dict.ioloop = io_loop_create(); + connection_switch_ioloop(&dict->conn.conn); + + if (dict->conn.conn.fd_in == -1 && + connection_client_connect(&dict->conn.conn) < 0) { + e_error(dict->conn.conn.event, "Couldn't connect"); + } else { + to = timeout_add(dict->timeout_msecs, + memcached_dict_lookup_timeout, dict); + if (!dict->connected) { + /* wait for connection */ + io_loop_run(dict->dict.ioloop); + } + + if (dict->connected) { + buffer_set_used_size(dict->conn.cmd, 0); + memcached_add_header(dict->conn.cmd, key_len); + buffer_append(dict->conn.cmd, key, key_len); + + o_stream_nsend(dict->conn.conn.output, + dict->conn.cmd->data, + dict->conn.cmd->used); + + i_zero(&dict->conn.reply); + io_loop_run(dict->dict.ioloop); + } + timeout_remove(&to); + } + + io_loop_set_current(prev_ioloop); + connection_switch_ioloop(&dict->conn.conn); + io_loop_set_current(dict->dict.ioloop); + io_loop_destroy(&dict->dict.ioloop); + + if (!dict->conn.reply.reply_received) { + /* we failed in some way. make sure we disconnect since the + connection state isn't known anymore */ + memcached_conn_destroy(&dict->conn.conn); + *error_r = "Communication failure"; + return -1; + } + switch (dict->conn.reply.status) { + case MEMCACHED_RESPONSE_OK: + *value_r = p_strndup(pool, dict->conn.reply.value, + dict->conn.reply.value_len); + return 1; + case MEMCACHED_RESPONSE_NOTFOUND: + return 0; + case MEMCACHED_RESPONSE_INTERNALERROR: + *error_r = "Lookup failed: Internal error"; + return -1; + case MEMCACHED_RESPONSE_BUSY: + *error_r = "Lookup failed: Busy"; + return -1; + case MEMCACHED_RESPONSE_TEMPFAILURE: + *error_r = "Lookup failed: Temporary failure"; + return -1; + } + + *error_r = t_strdup_printf("Lookup failed: Error code=%u", + dict->conn.reply.status); + return -1; +} + +struct dict dict_driver_memcached = { + .name = "memcached", + { + .init = memcached_dict_init, + .deinit = memcached_dict_deinit, + .lookup = memcached_dict_lookup, + } +}; diff --git a/src/lib-dict/dict-private.h b/src/lib-dict/dict-private.h new file mode 100644 index 0000000..7354175 --- /dev/null +++ b/src/lib-dict/dict-private.h @@ -0,0 +1,127 @@ +#ifndef DICT_PRIVATE_H +#define DICT_PRIVATE_H + +#include <time.h> +#include "dict.h" + +struct ioloop; + +struct dict_vfuncs { + int (*init)(struct dict *dict_driver, const char *uri, + const struct dict_settings *set, + struct dict **dict_r, const char **error_r); + void (*deinit)(struct dict *dict); + void (*wait)(struct dict *dict); + + int (*lookup)(struct dict *dict, const struct dict_op_settings *set, + pool_t pool, const char *key, const char **value_r, + const char **error_r); + + struct dict_iterate_context * + (*iterate_init)(struct dict *dict, + const struct dict_op_settings *set, + const char *path, + enum dict_iterate_flags flags); + bool (*iterate)(struct dict_iterate_context *ctx, + const char **key_r, const char *const **values_r); + int (*iterate_deinit)(struct dict_iterate_context *ctx, + const char **error_r); + + struct dict_transaction_context *(*transaction_init)(struct dict *dict); + /* call the callback before returning if non-async commits */ + void (*transaction_commit)(struct dict_transaction_context *ctx, + bool async, + dict_transaction_commit_callback_t *callback, + void *context); + void (*transaction_rollback)(struct dict_transaction_context *ctx); + + void (*set)(struct dict_transaction_context *ctx, + const char *key, const char *value); + void (*unset)(struct dict_transaction_context *ctx, + const char *key); + void (*atomic_inc)(struct dict_transaction_context *ctx, + const char *key, long long diff); + + void (*lookup_async)(struct dict *dict, const struct dict_op_settings *set, + const char *key, dict_lookup_callback_t *callback, + void *context); + bool (*switch_ioloop)(struct dict *dict); + void (*set_timestamp)(struct dict_transaction_context *ctx, + const struct timespec *ts); + void (*set_hide_log_values)(struct dict_transaction_context *ctx, + bool hide_log_values); +}; + +struct dict_commit_callback_ctx; + +struct dict_op_settings_private { + char *username; + char *home_dir; + + bool hide_log_values; +}; + +struct dict { + const char *name; + + struct dict_vfuncs v; + unsigned int iter_count; + unsigned int transaction_count; + struct dict_transaction_context *transactions; + int refcount; + struct event *event; + struct ioloop *ioloop, *prev_ioloop; + struct dict_commit_callback_ctx *commits; +}; + +struct dict_iterate_context { + struct dict *dict; + struct event *event; + struct dict_op_settings_private set; + enum dict_iterate_flags flags; + + dict_iterate_callback_t *async_callback; + void *async_context; + + uint64_t row_count, max_rows; + + bool has_more:1; +}; + +struct dict_transaction_context { + struct dict *dict; + struct dict_op_settings_private set; + struct dict_transaction_context *prev, *next; + + struct event *event; + struct timespec timestamp; + + bool changed:1; + bool no_slowness_warning:1; +}; + +void dict_transaction_commit_async_noop_callback( + const struct dict_commit_result *result, void *context); + +extern struct dict dict_driver_client; +extern struct dict dict_driver_file; +extern struct dict dict_driver_fs; +extern struct dict dict_driver_memcached; +extern struct dict dict_driver_memcached_ascii; +extern struct dict dict_driver_redis; +extern struct dict dict_driver_cdb; +extern struct dict dict_driver_fail; + +extern struct dict_iterate_context dict_iter_unsupported; +extern struct dict_transaction_context dict_transaction_unsupported; + +void dict_pre_api_callback(struct dict *dict); +void dict_post_api_callback(struct dict *dict); + +/* Duplicate an object of type dict_op_settings. Used for initializing/freeing + iterator and transaction contexts. */ +void dict_op_settings_dup(const struct dict_op_settings *source, + struct dict_op_settings_private *dest_r); +void dict_op_settings_private_free(struct dict_op_settings_private *set); + +#endif diff --git a/src/lib-dict/dict-redis.c b/src/lib-dict/dict-redis.c new file mode 100644 index 0000000..51eb322 --- /dev/null +++ b/src/lib-dict/dict-redis.c @@ -0,0 +1,831 @@ +/* Copyright (c) 2008-2018 Dovecot authors, see the included COPYING redis */ + +#include "lib.h" +#include "array.h" +#include "str.h" +#include "istream.h" +#include "ostream.h" +#include "connection.h" +#include "dict-private.h" + +#define REDIS_DEFAULT_PORT 6379 +#define REDIS_DEFAULT_LOOKUP_TIMEOUT_MSECS (1000*30) +#define DICT_USERNAME_SEPARATOR '/' + +enum redis_input_state { + /* expecting +OK reply for AUTH */ + REDIS_INPUT_STATE_AUTH, + /* expecting +OK reply for SELECT */ + REDIS_INPUT_STATE_SELECT, + /* expecting $-1 / $<size> followed by GET reply */ + REDIS_INPUT_STATE_GET, + /* expecting +QUEUED */ + REDIS_INPUT_STATE_MULTI, + /* expecting +OK reply for DISCARD */ + REDIS_INPUT_STATE_DISCARD, + /* expecting *<nreplies> */ + REDIS_INPUT_STATE_EXEC, + /* expecting EXEC reply */ + REDIS_INPUT_STATE_EXEC_REPLY +}; + +struct redis_connection { + struct connection conn; + struct redis_dict *dict; + + string_t *last_reply; + unsigned int bytes_left; + bool value_not_found; + bool value_received; +}; + +struct redis_dict_reply { + unsigned int reply_count; + dict_transaction_commit_callback_t *callback; + void *context; +}; + +struct redis_dict { + struct dict dict; + char *password, *key_prefix, *expire_value; + unsigned int timeout_msecs, db_id; + + struct redis_connection conn; + + ARRAY(enum redis_input_state) input_states; + ARRAY(struct redis_dict_reply) replies; + + bool connected; + bool transaction_open; + bool db_id_set; +}; + +struct redis_dict_transaction_context { + struct dict_transaction_context ctx; + unsigned int cmd_count; + char *error; +}; + +static struct connection_list *redis_connections; + +static void +redis_input_state_add(struct redis_dict *dict, enum redis_input_state state) +{ + array_push_back(&dict->input_states, &state); +} + +static void redis_input_state_remove(struct redis_dict *dict) +{ + array_pop_front(&dict->input_states); +} + +static void redis_reply_callback(struct redis_connection *conn, + const struct redis_dict_reply *reply, + const struct dict_commit_result *result) +{ + i_assert(reply->callback != NULL); + if (conn->dict->dict.prev_ioloop != NULL) + io_loop_set_current(conn->dict->dict.prev_ioloop); + reply->callback(result, reply->context); + if (conn->dict->dict.prev_ioloop != NULL) + io_loop_set_current(conn->dict->dict.ioloop); +} + +static void +redis_disconnected(struct redis_connection *conn, const char *reason) +{ + const struct dict_commit_result result = { + DICT_COMMIT_RET_FAILED, reason + }; + const struct redis_dict_reply *reply; + + conn->dict->db_id_set = FALSE; + conn->dict->connected = FALSE; + connection_disconnect(&conn->conn); + + array_foreach(&conn->dict->replies, reply) + redis_reply_callback(conn, reply, &result); + array_clear(&conn->dict->replies); + array_clear(&conn->dict->input_states); + + if (conn->dict->dict.ioloop != NULL) + io_loop_stop(conn->dict->dict.ioloop); +} + +static void redis_conn_destroy(struct connection *_conn) +{ + struct redis_connection *conn = (struct redis_connection *)_conn; + + redis_disconnected(conn, connection_disconnect_reason(_conn)); +} + +static void redis_dict_wait_timeout(struct redis_dict *dict) +{ + const char *reason = t_strdup_printf( + "redis: Commit timed out in %u.%03u secs", + dict->timeout_msecs/1000, dict->timeout_msecs%1000); + redis_disconnected(&dict->conn, reason); +} + +static void redis_wait(struct redis_dict *dict) +{ + struct timeout *to; + + i_assert(dict->dict.ioloop == NULL); + + dict->dict.prev_ioloop = current_ioloop; + dict->dict.ioloop = io_loop_create(); + to = timeout_add(dict->timeout_msecs, redis_dict_wait_timeout, dict); + connection_switch_ioloop(&dict->conn.conn); + + do { + io_loop_run(dict->dict.ioloop); + } while (array_count(&dict->input_states) > 0); + + timeout_remove(&to); + io_loop_set_current(dict->dict.prev_ioloop); + connection_switch_ioloop(&dict->conn.conn); + io_loop_set_current(dict->dict.ioloop); + io_loop_destroy(&dict->dict.ioloop); + dict->dict.prev_ioloop = NULL; +} + +static int redis_input_get(struct redis_connection *conn, const char **error_r) +{ + const unsigned char *data; + size_t size; + const char *line; + + if (conn->bytes_left == 0) { + /* read the size first */ + line = i_stream_next_line(conn->conn.input); + if (line == NULL) + return 0; + if (strcmp(line, "$-1") == 0) { + conn->value_received = TRUE; + conn->value_not_found = TRUE; + if (conn->dict->dict.ioloop != NULL) + io_loop_stop(conn->dict->dict.ioloop); + redis_input_state_remove(conn->dict); + return 1; + } + if (line[0] != '$' || str_to_uint(line+1, &conn->bytes_left) < 0) { + *error_r = t_strdup_printf( + "redis: Unexpected input (wanted $size): %s", line); + return -1; + } + conn->bytes_left += 2; /* include trailing CRLF */ + } + + data = i_stream_get_data(conn->conn.input, &size); + if (size > conn->bytes_left) + size = conn->bytes_left; + str_append_data(conn->last_reply, data, size); + + conn->bytes_left -= size; + i_stream_skip(conn->conn.input, size); + + if (conn->bytes_left > 0) + return 0; + + /* reply fully read - drop trailing CRLF */ + conn->value_received = TRUE; + str_truncate(conn->last_reply, str_len(conn->last_reply)-2); + + if (conn->dict->dict.ioloop != NULL) + io_loop_stop(conn->dict->dict.ioloop); + redis_input_state_remove(conn->dict); + return 1; +} + +static int +redis_conn_input_more(struct redis_connection *conn, const char **error_r) +{ + struct redis_dict *dict = conn->dict; + struct redis_dict_reply *reply; + const enum redis_input_state *states; + enum redis_input_state state; + unsigned int count, num_replies; + const char *line; + + states = array_get(&dict->input_states, &count); + if (count == 0) { + line = i_stream_next_line(conn->conn.input); + if (line == NULL) + return 0; + *error_r = t_strdup_printf( + "redis: Unexpected input (expected nothing): %s", line); + return -1; + } + state = states[0]; + if (state == REDIS_INPUT_STATE_GET) + return redis_input_get(conn, error_r); + + line = i_stream_next_line(conn->conn.input); + if (line == NULL) + return 0; + + redis_input_state_remove(dict); + switch (state) { + case REDIS_INPUT_STATE_GET: + i_unreached(); + case REDIS_INPUT_STATE_AUTH: + case REDIS_INPUT_STATE_SELECT: + case REDIS_INPUT_STATE_MULTI: + case REDIS_INPUT_STATE_DISCARD: + if (line[0] != '+') + break; + return 1; + case REDIS_INPUT_STATE_EXEC: + if (line[0] != '*' || str_to_uint(line+1, &num_replies) < 0) + break; + + reply = array_front_modifiable(&dict->replies); + i_assert(reply->reply_count > 0); + if (reply->reply_count != num_replies) { + *error_r = t_strdup_printf( + "redis: EXEC expected %u replies, not %u", + reply->reply_count, num_replies); + return -1; + } + return 1; + case REDIS_INPUT_STATE_EXEC_REPLY: + if (*line != '+' && *line != ':') + break; + /* success, just ignore the actual reply */ + reply = array_front_modifiable(&dict->replies); + i_assert(reply->reply_count > 0); + if (--reply->reply_count == 0) { + const struct dict_commit_result result = { + DICT_COMMIT_RET_OK, NULL + }; + redis_reply_callback(conn, reply, &result); + array_pop_front(&dict->replies); + /* if we're running in a dict-ioloop, we're handling a + synchronous commit and need to stop now */ + if (array_count(&dict->replies) == 0 && + conn->dict->dict.ioloop != NULL) + io_loop_stop(conn->dict->dict.ioloop); + } + return 1; + } + str_truncate(dict->conn.last_reply, 0); + str_append(dict->conn.last_reply, line); + *error_r = t_strdup_printf("redis: Unexpected input (state=%d): %s", state, line); + return -1; +} + +static void redis_conn_input(struct connection *_conn) +{ + struct redis_connection *conn = (struct redis_connection *)_conn; + const char *error = NULL; + int ret; + + switch (i_stream_read(_conn->input)) { + case 0: + return; + case -1: + redis_disconnected(conn, i_stream_get_error(_conn->input)); + return; + default: + break; + } + + while ((ret = redis_conn_input_more(conn, &error)) > 0) ; + if (ret < 0) { + i_assert(error != NULL); + redis_disconnected(conn, error); + } +} + +static void redis_conn_connected(struct connection *_conn, bool success) +{ + struct redis_connection *conn = (struct redis_connection *)_conn; + + if (!success) { + e_error(conn->conn.event, "connect() failed: %m"); + } else { + conn->dict->connected = TRUE; + } + if (conn->dict->dict.ioloop != NULL) + io_loop_stop(conn->dict->dict.ioloop); +} + +static const struct connection_settings redis_conn_set = { + .input_max_size = SIZE_MAX, + .output_max_size = SIZE_MAX, + .client = TRUE +}; + +static const struct connection_vfuncs redis_conn_vfuncs = { + .destroy = redis_conn_destroy, + .input = redis_conn_input, + .client_connected = redis_conn_connected +}; + +static const char *redis_escape_username(const char *username) +{ + const char *p; + string_t *str = t_str_new(64); + + for (p = username; *p != '\0'; p++) { + switch (*p) { + case DICT_USERNAME_SEPARATOR: + str_append(str, "\\-"); + break; + case '\\': + str_append(str, "\\\\"); + break; + default: + str_append_c(str, *p); + } + } + return str_c(str); +} + +static int +redis_dict_init(struct dict *driver, const char *uri, + const struct dict_settings *set, + struct dict **dict_r, const char **error_r) +{ + struct redis_dict *dict; + struct ip_addr ip; + unsigned int secs; + in_port_t port = REDIS_DEFAULT_PORT; + const char *const *args, *unix_path = NULL; + int ret = 0; + + if (redis_connections == NULL) { + redis_connections = + connection_list_init(&redis_conn_set, + &redis_conn_vfuncs); + } + + dict = i_new(struct redis_dict, 1); + if (net_addr2ip("127.0.0.1", &ip) < 0) + i_unreached(); + dict->timeout_msecs = REDIS_DEFAULT_LOOKUP_TIMEOUT_MSECS; + dict->key_prefix = i_strdup(""); + dict->password = i_strdup(""); + + args = t_strsplit(uri, ":"); + for (; *args != NULL; args++) { + if (str_begins(*args, "path=")) { + unix_path = *args + 5; + } else if (str_begins(*args, "host=")) { + if (net_addr2ip(*args+5, &ip) < 0) { + *error_r = t_strdup_printf("Invalid IP: %s", + *args+5); + ret = -1; + } + } else if (str_begins(*args, "port=")) { + if (net_str2port(*args+5, &port) < 0) { + *error_r = t_strdup_printf("Invalid port: %s", + *args+5); + ret = -1; + } + } else if (str_begins(*args, "prefix=")) { + i_free(dict->key_prefix); + dict->key_prefix = i_strdup(*args + 7); + } else if (str_begins(*args, "db=")) { + if (str_to_uint(*args+3, &dict->db_id) < 0) { + *error_r = t_strdup_printf( + "Invalid db number: %s", *args+3); + ret = -1; + } + } else if (str_begins(*args, "expire_secs=")) { + const char *value = *args + 12; + + if (str_to_uint(value, &secs) < 0 || secs == 0) { + *error_r = t_strdup_printf( + "Invalid expire_secs: %s", value); + ret = -1; + } + i_free(dict->expire_value); + dict->expire_value = i_strdup(value); + } else if (str_begins(*args, "timeout_msecs=")) { + if (str_to_uint(*args+14, &dict->timeout_msecs) < 0) { + *error_r = t_strdup_printf( + "Invalid timeout_msecs: %s", *args+14); + ret = -1; + } + } else if (str_begins(*args, "password=")) { + i_free(dict->password); + dict->password = i_strdup(*args + 9); + } else { + *error_r = t_strdup_printf("Unknown parameter: %s", + *args); + ret = -1; + } + } + if (ret < 0) { + i_free(dict->password); + i_free(dict->key_prefix); + i_free(dict); + return -1; + } + + dict->conn.conn.event_parent = set->event_parent; + + if (unix_path != NULL) { + connection_init_client_unix(redis_connections, &dict->conn.conn, + unix_path); + } else { + connection_init_client_ip(redis_connections, &dict->conn.conn, + NULL, &ip, port); + } + event_set_append_log_prefix(dict->conn.conn.event, "redis: "); + dict->dict = *driver; + dict->conn.last_reply = str_new(default_pool, 256); + dict->conn.dict = dict; + + i_array_init(&dict->input_states, 4); + i_array_init(&dict->replies, 4); + + *dict_r = &dict->dict; + return 0; +} + +static void redis_dict_deinit(struct dict *_dict) +{ + struct redis_dict *dict = (struct redis_dict *)_dict; + + if (array_count(&dict->input_states) > 0) { + i_assert(dict->connected); + redis_wait(dict); + } + connection_deinit(&dict->conn.conn); + str_free(&dict->conn.last_reply); + array_free(&dict->replies); + array_free(&dict->input_states); + i_free(dict->expire_value); + i_free(dict->key_prefix); + i_free(dict->password); + i_free(dict); + + if (redis_connections->connections == NULL) + connection_list_deinit(&redis_connections); +} + +static void redis_dict_wait(struct dict *_dict) +{ + struct redis_dict *dict = (struct redis_dict *)_dict; + + if (array_count(&dict->input_states) > 0) + redis_wait(dict); +} + +static void redis_dict_lookup_timeout(struct redis_dict *dict) +{ + const char *reason = t_strdup_printf( + "redis: Lookup timed out in %u.%03u secs", + dict->timeout_msecs/1000, dict->timeout_msecs%1000); + redis_disconnected(&dict->conn, reason); +} + +static const char * +redis_dict_get_full_key(struct redis_dict *dict, const char *username, + const char *key) +{ + const char *username_sp = strchr(username, DICT_USERNAME_SEPARATOR); + + if (str_begins(key, DICT_PATH_SHARED)) + key += strlen(DICT_PATH_SHARED); + else if (str_begins(key, DICT_PATH_PRIVATE)) { + key = t_strdup_printf("%s%c%s", + username_sp == NULL ? username : + redis_escape_username(username), + DICT_USERNAME_SEPARATOR, + key + strlen(DICT_PATH_PRIVATE)); + } else { + i_unreached(); + } + if (*dict->key_prefix != '\0') + key = t_strconcat(dict->key_prefix, key, NULL); + return key; +} + +static void redis_dict_auth(struct redis_dict *dict) +{ + const char *cmd; + + if (*dict->password == '\0') + return; + + cmd = t_strdup_printf("*2\r\n$4\r\nAUTH\r\n$%d\r\n%s\r\n", + (int)strlen(dict->password), dict->password); + o_stream_nsend_str(dict->conn.conn.output, cmd); + redis_input_state_add(dict, REDIS_INPUT_STATE_AUTH); +} + +static void redis_dict_select_db(struct redis_dict *dict) +{ + const char *cmd, *db_str; + + if (dict->db_id_set) + return; + dict->db_id_set = TRUE; + if (dict->db_id == 0) { + /* 0 is the default */ + return; + } + db_str = dec2str(dict->db_id); + cmd = t_strdup_printf("*2\r\n$6\r\nSELECT\r\n$%d\r\n%s\r\n", + (int)strlen(db_str), db_str); + o_stream_nsend_str(dict->conn.conn.output, cmd); + redis_input_state_add(dict, REDIS_INPUT_STATE_SELECT); +} + +static int redis_dict_lookup(struct dict *_dict, + const struct dict_op_settings *set, + pool_t pool, const char *key, + const char **value_r, const char **error_r) +{ + struct redis_dict *dict = (struct redis_dict *)_dict; + struct timeout *to; + const char *cmd; + + key = redis_dict_get_full_key(dict, set->username, key); + + dict->conn.value_received = FALSE; + dict->conn.value_not_found = FALSE; + + i_assert(dict->dict.ioloop == NULL); + + dict->dict.prev_ioloop = current_ioloop; + dict->dict.ioloop = io_loop_create(); + connection_switch_ioloop(&dict->conn.conn); + + if (dict->conn.conn.fd_in == -1 && + connection_client_connect(&dict->conn.conn) < 0) { + e_error(dict->conn.conn.event, "Couldn't connect"); + } else { + to = timeout_add(dict->timeout_msecs, + redis_dict_lookup_timeout, dict); + if (!dict->connected) { + /* wait for connection */ + io_loop_run(dict->dict.ioloop); + if (dict->connected) + redis_dict_auth(dict); + } + + if (dict->connected) { + redis_dict_select_db(dict); + cmd = t_strdup_printf("*2\r\n$3\r\nGET\r\n$%d\r\n%s\r\n", + (int)strlen(key), key); + o_stream_nsend_str(dict->conn.conn.output, cmd); + + str_truncate(dict->conn.last_reply, 0); + redis_input_state_add(dict, REDIS_INPUT_STATE_GET); + do { + io_loop_run(dict->dict.ioloop); + } while (array_count(&dict->input_states) > 0); + } + timeout_remove(&to); + } + + io_loop_set_current(dict->dict.prev_ioloop); + connection_switch_ioloop(&dict->conn.conn); + io_loop_set_current(dict->dict.ioloop); + io_loop_destroy(&dict->dict.ioloop); + dict->dict.prev_ioloop = NULL; + + if (!dict->conn.value_received) { + /* we failed in some way. make sure we disconnect since the + connection state isn't known anymore */ + *error_r = t_strdup_printf("redis: Communication failure (last reply: %s)", + str_c(dict->conn.last_reply)); + redis_disconnected(&dict->conn, *error_r); + return -1; + } + if (dict->conn.value_not_found) + return 0; + + *value_r = p_strdup(pool, str_c(dict->conn.last_reply)); + return 1; +} + +static struct dict_transaction_context * +redis_transaction_init(struct dict *_dict) +{ + struct redis_dict *dict = (struct redis_dict *)_dict; + struct redis_dict_transaction_context *ctx; + + i_assert(!dict->transaction_open); + dict->transaction_open = TRUE; + + ctx = i_new(struct redis_dict_transaction_context, 1); + ctx->ctx.dict = _dict; + + if (dict->conn.conn.fd_in == -1 && + connection_client_connect(&dict->conn.conn) < 0) { + e_error(dict->conn.conn.event, "Couldn't connect"); + } else if (!dict->connected) { + /* wait for connection */ + redis_wait(dict); + if (dict->connected) + redis_dict_auth(dict); + } + if (dict->connected) + redis_dict_select_db(dict); + return &ctx->ctx; +} + +static void +redis_transaction_commit(struct dict_transaction_context *_ctx, bool async, + dict_transaction_commit_callback_t *callback, + void *context) +{ + struct redis_dict_transaction_context *ctx = + (struct redis_dict_transaction_context *)_ctx; + struct redis_dict *dict = (struct redis_dict *)_ctx->dict; + struct redis_dict_reply *reply; + unsigned int i; + struct dict_commit_result result = { .ret = DICT_COMMIT_RET_OK }; + + i_assert(dict->transaction_open); + dict->transaction_open = FALSE; + + if (ctx->error != NULL) { + /* make sure we're disconnected */ + redis_disconnected(&dict->conn, ctx->error); + result.ret = -1; + result.error = ctx->error; + callback(&result, context); + } else if (_ctx->changed) { + i_assert(ctx->cmd_count > 0); + + o_stream_nsend_str(dict->conn.conn.output, + "*1\r\n$4\r\nEXEC\r\n"); + reply = array_append_space(&dict->replies); + reply->callback = callback; + reply->context = context; + reply->reply_count = ctx->cmd_count; + redis_input_state_add(dict, REDIS_INPUT_STATE_EXEC); + for (i = 0; i < ctx->cmd_count; i++) + redis_input_state_add(dict, REDIS_INPUT_STATE_EXEC_REPLY); + if (async) { + i_free(ctx); + return; + } + redis_wait(dict); + } else { + callback(&result, context); + } + i_free(ctx->error); + i_free(ctx); +} + +static void redis_transaction_rollback(struct dict_transaction_context *_ctx) +{ + struct redis_dict_transaction_context *ctx = + (struct redis_dict_transaction_context *)_ctx; + struct redis_dict *dict = (struct redis_dict *)_ctx->dict; + + i_assert(dict->transaction_open); + dict->transaction_open = FALSE; + + if (ctx->error != NULL) { + /* make sure we're disconnected */ + redis_disconnected(&dict->conn, ctx->error); + } else if (_ctx->changed) { + o_stream_nsend_str(dict->conn.conn.output, + "*1\r\n$7\r\nDISCARD\r\n"); + redis_input_state_add(dict, REDIS_INPUT_STATE_DISCARD); + } + i_free(ctx->error); + i_free(ctx); +} + +static int redis_check_transaction(struct redis_dict_transaction_context *ctx) +{ + struct redis_dict *dict = (struct redis_dict *)ctx->ctx.dict; + + if (ctx->error != NULL) + return -1; + if (!dict->connected) { + ctx->error = i_strdup("Disconnected during transaction"); + return -1; + } + if (ctx->ctx.changed) + return 0; + + redis_input_state_add(dict, REDIS_INPUT_STATE_MULTI); + if (o_stream_send_str(dict->conn.conn.output, + "*1\r\n$5\r\nMULTI\r\n") < 0) { + ctx->error = i_strdup_printf("write() failed: %s", + o_stream_get_error(dict->conn.conn.output)); + return -1; + } + return 0; +} + +static void +redis_append_expire(struct redis_dict_transaction_context *ctx, + string_t *cmd, const char *key) +{ + struct redis_dict *dict = (struct redis_dict *)ctx->ctx.dict; + + if (dict->expire_value == NULL) + return; + + str_printfa(cmd, "*3\r\n$6\r\nEXPIRE\r\n$%u\r\n%s\r\n$%u\r\n%s\r\n", + (unsigned int)strlen(key), key, + (unsigned int)strlen(dict->expire_value), + dict->expire_value); + redis_input_state_add(dict, REDIS_INPUT_STATE_MULTI); + ctx->cmd_count++; +} + +static void redis_set(struct dict_transaction_context *_ctx, + const char *key, const char *value) +{ + struct redis_dict_transaction_context *ctx = + (struct redis_dict_transaction_context *)_ctx; + struct redis_dict *dict = (struct redis_dict *)_ctx->dict; + const struct dict_op_settings_private *set = &_ctx->set; + string_t *cmd; + + if (redis_check_transaction(ctx) < 0) + return; + + key = redis_dict_get_full_key(dict, set->username, key); + cmd = t_str_new(128); + str_printfa(cmd, "*3\r\n$3\r\nSET\r\n$%u\r\n%s\r\n$%u\r\n%s\r\n", + (unsigned int)strlen(key), key, + (unsigned int)strlen(value), value); + redis_input_state_add(dict, REDIS_INPUT_STATE_MULTI); + ctx->cmd_count++; + redis_append_expire(ctx, cmd, key); + if (o_stream_send(dict->conn.conn.output, str_data(cmd), str_len(cmd)) < 0) { + ctx->error = i_strdup_printf("write() failed: %s", + o_stream_get_error(dict->conn.conn.output)); + } +} + +static void redis_unset(struct dict_transaction_context *_ctx, + const char *key) +{ + struct redis_dict_transaction_context *ctx = + (struct redis_dict_transaction_context *)_ctx; + struct redis_dict *dict = (struct redis_dict *)_ctx->dict; + const struct dict_op_settings_private *set = &_ctx->set; + const char *cmd; + + if (redis_check_transaction(ctx) < 0) + return; + + key = redis_dict_get_full_key(dict, set->username, key); + cmd = t_strdup_printf("*2\r\n$3\r\nDEL\r\n$%u\r\n%s\r\n", + (unsigned int)strlen(key), key); + if (o_stream_send_str(dict->conn.conn.output, cmd) < 0) { + ctx->error = i_strdup_printf("write() failed: %s", + o_stream_get_error(dict->conn.conn.output)); + } + redis_input_state_add(dict, REDIS_INPUT_STATE_MULTI); + ctx->cmd_count++; +} + +static void redis_atomic_inc(struct dict_transaction_context *_ctx, + const char *key, long long diff) +{ + struct redis_dict_transaction_context *ctx = + (struct redis_dict_transaction_context *)_ctx; + struct redis_dict *dict = (struct redis_dict *)_ctx->dict; + const struct dict_op_settings_private *set = &_ctx->set; + const char *diffstr; + string_t *cmd; + + if (redis_check_transaction(ctx) < 0) + return; + + key = redis_dict_get_full_key(dict, set->username, key); + diffstr = t_strdup_printf("%lld", diff); + cmd = t_str_new(128); + str_printfa(cmd, "*3\r\n$6\r\nINCRBY\r\n$%u\r\n%s\r\n$%u\r\n%s\r\n", + (unsigned int)strlen(key), key, + (unsigned int)strlen(diffstr), diffstr); + redis_input_state_add(dict, REDIS_INPUT_STATE_MULTI); + ctx->cmd_count++; + redis_append_expire(ctx, cmd, key); + if (o_stream_send(dict->conn.conn.output, str_data(cmd), str_len(cmd)) < 0) { + ctx->error = i_strdup_printf("write() failed: %s", + o_stream_get_error(dict->conn.conn.output)); + } +} + +struct dict dict_driver_redis = { + .name = "redis", + { + .init = redis_dict_init, + .deinit = redis_dict_deinit, + .wait = redis_dict_wait, + .lookup = redis_dict_lookup, + .transaction_init = redis_transaction_init, + .transaction_commit = redis_transaction_commit, + .transaction_rollback = redis_transaction_rollback, + .set = redis_set, + .unset = redis_unset, + .atomic_inc = redis_atomic_inc, + } +}; diff --git a/src/lib-dict/dict-transaction-memory.c b/src/lib-dict/dict-transaction-memory.c new file mode 100644 index 0000000..4793ad3 --- /dev/null +++ b/src/lib-dict/dict-transaction-memory.c @@ -0,0 +1,59 @@ +/* Copyright (c) 2005-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "dict-transaction-memory.h" + +void dict_transaction_memory_init(struct dict_transaction_memory_context *ctx, + struct dict *dict, pool_t pool) +{ + ctx->ctx.dict = dict; + ctx->pool = pool; + p_array_init(&ctx->changes, pool, 32); +} + +void dict_transaction_memory_rollback(struct dict_transaction_context *_ctx) +{ + struct dict_transaction_memory_context *ctx = + (struct dict_transaction_memory_context *)_ctx; + + pool_unref(&ctx->pool); +} + +void dict_transaction_memory_set(struct dict_transaction_context *_ctx, + const char *key, const char *value) +{ + struct dict_transaction_memory_context *ctx = + (struct dict_transaction_memory_context *)_ctx; + struct dict_transaction_memory_change *change; + + change = array_append_space(&ctx->changes); + change->type = DICT_CHANGE_TYPE_SET; + change->key = p_strdup(ctx->pool, key); + change->value.str = p_strdup(ctx->pool, value); +} + +void dict_transaction_memory_unset(struct dict_transaction_context *_ctx, + const char *key) +{ + struct dict_transaction_memory_context *ctx = + (struct dict_transaction_memory_context *)_ctx; + struct dict_transaction_memory_change *change; + + change = array_append_space(&ctx->changes); + change->type = DICT_CHANGE_TYPE_UNSET; + change->key = p_strdup(ctx->pool, key); +} + +void dict_transaction_memory_atomic_inc(struct dict_transaction_context *_ctx, + const char *key, long long diff) +{ + struct dict_transaction_memory_context *ctx = + (struct dict_transaction_memory_context *)_ctx; + struct dict_transaction_memory_change *change; + + change = array_append_space(&ctx->changes); + change->type = DICT_CHANGE_TYPE_INC; + change->key = p_strdup(ctx->pool, key); + change->value.diff = diff; +} diff --git a/src/lib-dict/dict-transaction-memory.h b/src/lib-dict/dict-transaction-memory.h new file mode 100644 index 0000000..2164c1e --- /dev/null +++ b/src/lib-dict/dict-transaction-memory.h @@ -0,0 +1,38 @@ +#ifndef DICT_TRANSACTION_MEMORY_H +#define DICT_TRANSACTION_MEMORY_H + +#include "dict-private.h" + +enum dict_change_type { + DICT_CHANGE_TYPE_SET, + DICT_CHANGE_TYPE_UNSET, + DICT_CHANGE_TYPE_INC +}; + +struct dict_transaction_memory_change { + enum dict_change_type type; + const char *key; + union { + const char *str; + long long diff; + } value; +}; + +struct dict_transaction_memory_context { + struct dict_transaction_context ctx; + pool_t pool; + ARRAY(struct dict_transaction_memory_change) changes; +}; + +void dict_transaction_memory_init(struct dict_transaction_memory_context *ctx, + struct dict *dict, pool_t pool); +void dict_transaction_memory_rollback(struct dict_transaction_context *ctx); + +void dict_transaction_memory_set(struct dict_transaction_context *ctx, + const char *key, const char *value); +void dict_transaction_memory_unset(struct dict_transaction_context *ctx, + const char *key); +void dict_transaction_memory_atomic_inc(struct dict_transaction_context *ctx, + const char *key, long long diff); + +#endif diff --git a/src/lib-dict/dict-txn-lua.c b/src/lib-dict/dict-txn-lua.c new file mode 100644 index 0000000..34a475d --- /dev/null +++ b/src/lib-dict/dict-txn-lua.c @@ -0,0 +1,262 @@ +/* Copyright (c) 2021 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "dict.h" +#include "dlua-script-private.h" +#include "dict-lua-private.h" +#include "dlua-wrapper.h" + +struct lua_dict_txn { + pool_t pool; + struct dict_transaction_context *txn; + enum { + STATE_OPEN, + STATE_COMMITTED, + STATE_ABORTED, + } state; + + lua_State *L; + const char *username; +}; + +static int lua_dict_transaction_rollback(lua_State *L); +static int lua_dict_transaction_commit(lua_State *L); +static int lua_dict_set(lua_State *L); +static int lua_dict_unset(lua_State *L); +static int lua_dict_set_timestamp(lua_State *L); + +static luaL_Reg lua_dict_txn_methods[] = { + { "rollback", lua_dict_transaction_rollback }, + { "commit", lua_dict_transaction_commit }, + { "set", lua_dict_set }, + { "unset", lua_dict_unset }, + { "set_timestamp", lua_dict_set_timestamp }, + { NULL, NULL }, +}; + +static void sanity_check_txn(lua_State *L, struct lua_dict_txn *txn) +{ + switch (txn->state) { + case STATE_OPEN: + return; + case STATE_COMMITTED: + luaL_error(L, "dict transaction already committed"); + return; + case STATE_ABORTED: + luaL_error(L, "dict transaction already aborted"); + return; + } + + i_unreached(); +} + +/* no actual ref counting, but we use it for clean up */ +static void lua_dict_txn_unref(struct lua_dict_txn *txn) +{ + /* rollback any transactions that were forgotten about */ + dict_transaction_rollback(&txn->txn); + + pool_unref(&txn->pool); +} + +DLUA_WRAP_C_DATA(dict_txn, struct lua_dict_txn, lua_dict_txn_unref, + lua_dict_txn_methods); + +/* + * Abort a transaction [-1,+0,e] + * + * Args: + * 1) userdata: struct lua_dict_txn * + */ +static int lua_dict_transaction_rollback(lua_State *L) +{ + struct lua_dict_txn *txn; + + DLUA_REQUIRE_ARGS(L, 1); + + txn = xlua_dict_txn_getptr(L, 1, NULL); + sanity_check_txn(L, txn); + + txn->state = STATE_ABORTED; + dict_transaction_rollback(&txn->txn); + + return 0; +} + +static int lua_dict_transaction_commit_continue(lua_State *L, + int status ATTR_UNUSED, + lua_KContext ctx ATTR_UNUSED) +{ + if (!lua_isnil(L, -1)) + lua_error(L); /* commit failed */ + + lua_pop(L, 1); /* pop the nil indicating the lack of error */ + + return 0; +} + +static void +lua_dict_transaction_commit_callback(const struct dict_commit_result *result, + struct lua_dict_txn *txn) +{ + + switch (result->ret) { + case DICT_COMMIT_RET_OK: + /* push a nil to indicate everything is ok */ + lua_pushnil(txn->L); + break; + case DICT_COMMIT_RET_NOTFOUND: + /* we don't expose dict_atomic_inc(), so this should never happen */ + i_unreached(); + case DICT_COMMIT_RET_FAILED: + case DICT_COMMIT_RET_WRITE_UNCERTAIN: + /* push the error we'll raise when we resume */ + i_assert(result->error != NULL); + lua_pushfstring(txn->L, "dict transaction commit failed: %s", + result->error); + break; + } + + dlua_pcall_yieldable_resume(txn->L, 1); +} + +/* + * Commit a transaction [-1,+0,e] + * + * Args: + * 1) userdata: struct lua_dict_txn * + */ +static int lua_dict_transaction_commit(lua_State *L) +{ + struct lua_dict_txn *txn; + + DLUA_REQUIRE_ARGS(L, 1); + + txn = xlua_dict_txn_getptr(L, 1, NULL); + sanity_check_txn(L, txn); + + txn->state = STATE_COMMITTED; + dict_transaction_commit_async(&txn->txn, + lua_dict_transaction_commit_callback, txn); + + return lua_dict_transaction_commit_continue(L, + lua_yieldk(L, 0, 0, lua_dict_transaction_commit_continue), 0); +} + +/* + * Set key to value [-3,+0,e] + * + * Args: + * 1) userdata: struct lua_dict_txn * + * 2) string: key + * 3) string: value + */ +static int lua_dict_set(lua_State *L) +{ + struct lua_dict_txn *txn; + const char *key, *value; + + DLUA_REQUIRE_ARGS(L, 3); + + txn = xlua_dict_txn_getptr(L, 1, NULL); + key = luaL_checkstring(L, 2); + value = luaL_checkstring(L, 3); + lua_dict_check_key_prefix(L, key, txn->username); + + dict_set(txn->txn, key, value); + + return 0; +} + +/* + * Unset key [-2,+0,e] + * + * Args: + * 1) userdata: struct lua_dict_txn * + * 2) string: key + */ +static int lua_dict_unset(lua_State *L) +{ + struct lua_dict_txn *txn; + const char *key; + + DLUA_REQUIRE_ARGS(L, 2); + + txn = xlua_dict_txn_getptr(L, 1, NULL); + key = luaL_checkstring(L, 2); + lua_dict_check_key_prefix(L, key, txn->username); + + dict_unset(txn->txn, key); + + return 0; +} + +/* + * Start a dict transaction [-(1|2),+1,e] + * + * Args: + * 1) userdata: struct dict * + * 2*) string: username + * + * Returns: + * Returns a new transaction object. + * Username will be NULL if not provided in args. + */ +int lua_dict_transaction_begin(lua_State *L) +{ + struct lua_dict_txn *txn; + struct dict *dict; + const char *username = NULL; + pool_t pool; + + DLUA_REQUIRE_ARGS_IN(L, 1, 2); + + dict = dlua_check_dict(L, 1); + if (lua_gettop(L) >= 2) + username = luaL_checkstring(L, 2); + + pool = pool_alloconly_create("lua dict txn", 128); + txn = p_new(pool, struct lua_dict_txn, 1); + txn->pool = pool; + + struct dict_op_settings set = { + .username = username, + }; + txn->txn = dict_transaction_begin(dict, &set); + txn->state = STATE_OPEN; + txn->L = L; + txn->username = p_strdup(txn->pool, username); + + xlua_pushdict_txn(L, txn, FALSE); + + return 1; +} + +/* + * Set timestamp to the transaction [-2,+0,e] + * + * Args: + * 1) userdata: struct lua_dict_txn * + * 2) PosixTimespec : { tv_sec, tv_nsec } + */ +static int lua_dict_set_timestamp(lua_State *L) +{ + struct lua_dict_txn *txn; + lua_Number tv_sec, tv_nsec; + + DLUA_REQUIRE_ARGS(L, 2); + + txn = xlua_dict_txn_getptr(L, 1, NULL); + if (dlua_table_get_number_by_str(L, 2, "tv_sec", &tv_sec) <= 0) + luaL_error(L, "tv_sec missing from table"); + if (dlua_table_get_number_by_str(L, 2, "tv_nsec", &tv_nsec) <= 0) + luaL_error(L, "tv_nsec missing from table"); + + struct timespec ts = { + .tv_sec = tv_sec, + .tv_nsec = tv_nsec + }; + dict_transaction_set_timestamp(txn->txn, &ts); + return 0; +} diff --git a/src/lib-dict/dict.c b/src/lib-dict/dict.c new file mode 100644 index 0000000..bda34ae --- /dev/null +++ b/src/lib-dict/dict.c @@ -0,0 +1,769 @@ +/* Copyright (c) 2005-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "guid.h" +#include "llist.h" +#include "ioloop.h" +#include "str.h" +#include "ioloop.h" +#include "dict-private.h" + +struct dict_commit_callback_ctx { + pool_t pool; + struct dict_commit_callback_ctx *prev, *next; + struct dict *dict; + struct event *event; + dict_transaction_commit_callback_t *callback; + struct dict_op_settings_private set; + struct timeout *to; + void *context; + struct dict_commit_result result; + bool delayed_callback:1; +}; + +struct dict_lookup_callback_ctx { + struct dict *dict; + struct event *event; + dict_lookup_callback_t *callback; + void *context; +}; + +static ARRAY(struct dict *) dict_drivers; + +static void +dict_commit_async_timeout(struct dict_commit_callback_ctx *ctx); + +static struct event_category event_category_dict = { + .name = "dict", +}; + +static struct dict *dict_driver_lookup(const char *name) +{ + struct dict *dict; + + array_foreach_elem(&dict_drivers, dict) { + if (strcmp(dict->name, name) == 0) + return dict; + } + return NULL; +} + +void dict_transaction_commit_async_noop_callback( + const struct dict_commit_result *result ATTR_UNUSED, + void *context ATTR_UNUSED) +{ + /* do nothing */ +} + +void dict_driver_register(struct dict *driver) +{ + if (!array_is_created(&dict_drivers)) + i_array_init(&dict_drivers, 8); + + if (dict_driver_lookup(driver->name) != NULL) { + i_fatal("dict_driver_register(%s): Already registered", + driver->name); + } + array_push_back(&dict_drivers, &driver); +} + +void dict_driver_unregister(struct dict *driver) +{ + struct dict *const *dicts; + unsigned int idx = UINT_MAX; + + array_foreach(&dict_drivers, dicts) { + if (*dicts == driver) { + idx = array_foreach_idx(&dict_drivers, dicts); + break; + } + } + i_assert(idx != UINT_MAX); + array_delete(&dict_drivers, idx, 1); + + if (array_count(&dict_drivers) == 0) + array_free(&dict_drivers); +} + +int dict_init(const char *uri, const struct dict_settings *set, + struct dict **dict_r, const char **error_r) +{ + struct dict_settings set_dup = *set; + struct dict *dict; + const char *p, *name, *error; + + p = strchr(uri, ':'); + if (p == NULL) { + *error_r = t_strdup_printf("Dictionary URI is missing ':': %s", + uri); + return -1; + } + + name = t_strdup_until(uri, p); + dict = dict_driver_lookup(name); + if (dict == NULL) { + *error_r = t_strdup_printf("Unknown dict module: %s", name); + return -1; + } + struct event *event = event_create(set->event_parent); + event_add_category(event, &event_category_dict); + event_add_str(event, "driver", dict->name); + event_set_append_log_prefix(event, t_strdup_printf("dict(%s): ", + dict->name)); + set_dup.event_parent = event; + if (dict->v.init(dict, p+1, &set_dup, dict_r, &error) < 0) { + *error_r = t_strdup_printf("dict %s: %s", name, error); + event_unref(&event); + return -1; + } + i_assert(*dict_r != NULL); + (*dict_r)->refcount++; + (*dict_r)->event = event; + e_debug(event_create_passthrough(event)->set_name("dict_created")->event(), + "dict created (uri=%s, base_dir=%s)", uri, set->base_dir); + + return 0; +} + +static void dict_ref(struct dict *dict) +{ + i_assert(dict->refcount > 0); + + dict->refcount++; +} + +static void dict_unref(struct dict **_dict) +{ + struct dict *dict = *_dict; + *_dict = NULL; + if (dict == NULL) + return; + struct event *event = dict->event; + i_assert(dict->refcount > 0); + if (--dict->refcount == 0) { + dict->v.deinit(dict); + e_debug(event_create_passthrough(event)-> + set_name("dict_destroyed")->event(), "dict destroyed"); + event_unref(&event); + } +} + +void dict_deinit(struct dict **_dict) +{ + struct dict *dict = *_dict; + + *_dict = NULL; + + i_assert(dict->iter_count == 0); + i_assert(dict->transaction_count == 0); + i_assert(dict->transactions == NULL); + i_assert(dict->commits == NULL); + dict_unref(&dict); +} + +void dict_wait(struct dict *dict) +{ + struct dict_commit_callback_ctx *commit, *next; + + e_debug(dict->event, "Waiting for dict to finish pending operations"); + if (dict->v.wait != NULL) + dict->v.wait(dict); + for (commit = dict->commits; commit != NULL; commit = next) { + next = commit->next; + dict_commit_async_timeout(commit); + } +} + +bool dict_switch_ioloop(struct dict *dict) +{ + struct dict_commit_callback_ctx *commit; + bool ret = FALSE; + + for (commit = dict->commits; commit != NULL; commit = commit->next) { + commit->to = io_loop_move_timeout(&commit->to); + ret = TRUE; + } + if (dict->v.switch_ioloop != NULL) { + if (dict->v.switch_ioloop(dict)) + return TRUE; + } + return ret; +} + +static bool dict_key_prefix_is_valid(const char *key, const char *username) +{ + if (str_begins(key, DICT_PATH_SHARED)) + return TRUE; + if (str_begins(key, DICT_PATH_PRIVATE)) { + i_assert(username != NULL && username[0] != '\0'); + return TRUE; + } + return FALSE; + +} + +void dict_pre_api_callback(struct dict *dict) +{ + if (dict->prev_ioloop != NULL) { + /* Don't let callback see that we've created our + internal ioloop in case it wants to add some ios + or timeouts. */ + io_loop_set_current(dict->prev_ioloop); + } +} + +void dict_post_api_callback(struct dict *dict) +{ + if (dict->prev_ioloop != NULL) { + io_loop_set_current(dict->ioloop); + io_loop_stop(dict->ioloop); + } +} + +static void dict_lookup_finished(struct event *event, int ret, const char *error) +{ + i_assert(ret >= 0 || error != NULL); + const char *key = event_find_field_recursive_str(event, "key"); + if (ret < 0) + event_add_str(event, "error", error); + else if (ret == 0) + event_add_str(event, "key_not_found", "yes"); + event_set_name(event, "dict_lookup_finished"); + e_debug(event, "Lookup finished for '%s': %s", key, ret > 0 ? + "found" : + "not found"); +} + +static void dict_transaction_finished(struct event *event, enum dict_commit_ret ret, + bool rollback, const char *error) +{ + i_assert(ret > DICT_COMMIT_RET_FAILED || error != NULL); + if (ret == DICT_COMMIT_RET_FAILED || ret == DICT_COMMIT_RET_WRITE_UNCERTAIN) { + if (ret == DICT_COMMIT_RET_WRITE_UNCERTAIN) + event_add_str(event, "write_uncertain", "yes"); + event_add_str(event, "error", error); + } else if (rollback) { + event_add_str(event, "rollback", "yes"); + } else if (ret == 0) { + event_add_str(event, "key_not_found", "yes"); + } + event_set_name(event, "dict_transaction_finished"); + e_debug(event, "Dict transaction finished"); +} + +static void +dict_lookup_callback(const struct dict_lookup_result *result, + void *context) +{ + struct dict_lookup_callback_ctx *ctx = context; + + dict_pre_api_callback(ctx->dict); + ctx->callback(result, ctx->context); + dict_post_api_callback(ctx->dict); + dict_lookup_finished(ctx->event, result->ret, result->error); + event_unref(&ctx->event); + + dict_unref(&ctx->dict); + i_free(ctx); +} + +static void +dict_commit_async_timeout(struct dict_commit_callback_ctx *ctx) +{ + DLLIST_REMOVE(&ctx->dict->commits, ctx); + timeout_remove(&ctx->to); + dict_pre_api_callback(ctx->dict); + if (ctx->callback != NULL) + ctx->callback(&ctx->result, ctx->context); + else if (ctx->result.ret < 0) + e_error(ctx->event, "Commit failed: %s", ctx->result.error); + dict_post_api_callback(ctx->dict); + + dict_transaction_finished(ctx->event, ctx->result.ret, FALSE, ctx->result.error); + dict_op_settings_private_free(&ctx->set); + event_unref(&ctx->event); + dict_unref(&ctx->dict); + pool_unref(&ctx->pool); +} + +static void dict_commit_callback(const struct dict_commit_result *result, + void *context) +{ + struct dict_commit_callback_ctx *ctx = context; + + i_assert(result->ret >= 0 || result->error != NULL); + ctx->result = *result; + if (ctx->delayed_callback) { + ctx->result.error = p_strdup(ctx->pool, ctx->result.error); + ctx->to = timeout_add_short(0, dict_commit_async_timeout, ctx); + } else { + dict_commit_async_timeout(ctx); + } +} + +static struct event * +dict_event_create(struct dict *dict, const struct dict_op_settings *set) +{ + struct event *event = event_create(dict->event); + if (set->username != NULL) + event_add_str(event, "user", set->username); + return event; +} + +int dict_lookup(struct dict *dict, const struct dict_op_settings *set, + pool_t pool, const char *key, + const char **value_r, const char **error_r) +{ + struct event *event = dict_event_create(dict, set); + int ret; + i_assert(dict_key_prefix_is_valid(key, set->username)); + + e_debug(event, "Looking up '%s'", key); + event_add_str(event, "key", key); + ret = dict->v.lookup(dict, set, pool, key, value_r, error_r); + dict_lookup_finished(event, ret, *error_r); + event_unref(&event); + return ret; +} + +#undef dict_lookup_async +void dict_lookup_async(struct dict *dict, const struct dict_op_settings *set, + const char *key, dict_lookup_callback_t *callback, + void *context) +{ + i_assert(dict_key_prefix_is_valid(key, set->username)); + if (dict->v.lookup_async == NULL) { + struct dict_lookup_result result; + + i_zero(&result); + /* event is going to be sent by dict_lookup */ + result.ret = dict_lookup(dict, set, pool_datastack_create(), + key, &result.value, &result.error); + const char *const values[] = { result.value, NULL }; + result.values = values; + callback(&result, context); + return; + } + struct dict_lookup_callback_ctx *lctx = + i_new(struct dict_lookup_callback_ctx, 1); + lctx->dict = dict; + dict_ref(lctx->dict); + lctx->callback = callback; + lctx->context = context; + lctx->event = dict_event_create(dict, set); + event_add_str(lctx->event, "key", key); + e_debug(lctx->event, "Looking up (async) '%s'", key); + dict->v.lookup_async(dict, set, key, dict_lookup_callback, lctx); +} + +struct dict_iterate_context * +dict_iterate_init(struct dict *dict, const struct dict_op_settings *set, + const char *path, enum dict_iterate_flags flags) +{ + struct dict_iterate_context *ctx; + + i_assert(path != NULL); + i_assert(dict_key_prefix_is_valid(path, set->username)); + + if (dict->v.iterate_init == NULL) { + /* not supported by backend */ + ctx = &dict_iter_unsupported; + } else { + ctx = dict->v.iterate_init(dict, set, path, flags); + } + /* the dict in context can differ from the dict + passed as parameter, e.g. it can be dict-fail when + iteration is not supported. */ + ctx->event = dict_event_create(dict, set); + ctx->flags = flags; + dict_op_settings_dup(set, &ctx->set); + + event_add_str(ctx->event, "key", path); + event_set_name(ctx->event, "dict_iteration_started"); + e_debug(ctx->event, "Iterating prefix %s", path); + ctx->dict->iter_count++; + return ctx; +} + +bool dict_iterate(struct dict_iterate_context *ctx, + const char **key_r, const char **value_r) +{ + const char *const *values; + + if (!dict_iterate_values(ctx, key_r, &values)) + return FALSE; + if ((ctx->flags & DICT_ITERATE_FLAG_NO_VALUE) == 0) + *value_r = values[0]; + return TRUE; +} + +bool dict_iterate_values(struct dict_iterate_context *ctx, + const char **key_r, const char *const **values_r) +{ + + if (ctx->max_rows > 0 && ctx->row_count >= ctx->max_rows) { + e_debug(ctx->event, "Maximum row count (%"PRIu64") reached", + ctx->max_rows); + /* row count was limited */ + ctx->has_more = FALSE; + return FALSE; + } + if (!ctx->dict->v.iterate(ctx, key_r, values_r)) + return FALSE; + if ((ctx->flags & DICT_ITERATE_FLAG_NO_VALUE) != 0) { + /* always return value as NULL to be consistent across + drivers */ + *values_r = NULL; + } else { + i_assert(values_r[0] != NULL); + } + ctx->row_count++; + return TRUE; +} + +#undef dict_iterate_set_async_callback +void dict_iterate_set_async_callback(struct dict_iterate_context *ctx, + dict_iterate_callback_t *callback, + void *context) +{ + ctx->async_callback = callback; + ctx->async_context = context; +} + +void dict_iterate_set_limit(struct dict_iterate_context *ctx, + uint64_t max_rows) +{ + ctx->max_rows = max_rows; +} + +bool dict_iterate_has_more(struct dict_iterate_context *ctx) +{ + return ctx->has_more; +} + +int dict_iterate_deinit(struct dict_iterate_context **_ctx, + const char **error_r) +{ + struct dict_iterate_context *ctx = *_ctx; + + if (ctx == NULL) + return 0; + + struct event *event = ctx->event; + int ret; + uint64_t rows; + + i_assert(ctx->dict->iter_count > 0); + ctx->dict->iter_count--; + + *_ctx = NULL; + rows = ctx->row_count; + struct dict_op_settings_private set_copy = ctx->set; + ret = ctx->dict->v.iterate_deinit(ctx, error_r); + dict_op_settings_private_free(&set_copy); + + event_add_int(event, "rows", rows); + event_set_name(event, "dict_iteration_finished"); + + if (ret < 0) { + event_add_str(event, "error", *error_r); + e_debug(event, "Iteration finished: %s", *error_r); + } else { + if (rows == 0) + event_add_str(event, "key_not_found", "yes"); + e_debug(event, "Iteration finished, got %"PRIu64" rows", rows); + } + + event_unref(&event); + return ret; +} + +struct dict_transaction_context * +dict_transaction_begin(struct dict *dict, const struct dict_op_settings *set) +{ + struct dict_transaction_context *ctx; + guid_128_t guid; + if (dict->v.transaction_init == NULL) + ctx = &dict_transaction_unsupported; + else + ctx = dict->v.transaction_init(dict); + /* the dict in context can differ from the dict + passed as parameter, e.g. it can be dict-fail when + transactions are not supported. */ + ctx->dict->transaction_count++; + DLLIST_PREPEND(&ctx->dict->transactions, ctx); + ctx->event = dict_event_create(dict, set); + dict_op_settings_dup(set, &ctx->set); + guid_128_generate(guid); + event_add_str(ctx->event, "txid", guid_128_to_string(guid)); + event_set_name(ctx->event, "dict_transaction_started"); + e_debug(ctx->event, "Starting transaction"); + return ctx; +} + +void dict_transaction_no_slowness_warning(struct dict_transaction_context *ctx) +{ + ctx->no_slowness_warning = TRUE; +} + +void dict_transaction_set_hide_log_values(struct dict_transaction_context *ctx, + bool hide_log_values) +{ + /* Apply hide_log_values to the current transactions dict op settings */ + ctx->set.hide_log_values = hide_log_values; + if (ctx->dict->v.set_hide_log_values != NULL) + ctx->dict->v.set_hide_log_values(ctx, hide_log_values); +} + +void dict_transaction_set_timestamp(struct dict_transaction_context *ctx, + const struct timespec *ts) +{ + /* These asserts are mainly here to guarantee a possibility in future + to change the API to support multiple timestamps within the same + transaction, so this call would apply only to the following + changes. */ + i_assert(!ctx->changed); + i_assert(ctx->timestamp.tv_sec == 0); + i_assert(ts->tv_sec > 0); + + ctx->timestamp = *ts; + struct event_passthrough *e = event_create_passthrough(ctx->event)-> + set_name("dict_set_timestamp"); + + e_debug(e->event(), "Setting timestamp on transaction to (%"PRIdTIME_T", %ld)", + ts->tv_sec, ts->tv_nsec); + if (ctx->dict->v.set_timestamp != NULL) + ctx->dict->v.set_timestamp(ctx, ts); +} + +struct dict_commit_sync_result { + int ret; + char *error; +}; + +static void +dict_transaction_commit_sync_callback(const struct dict_commit_result *result, + void *context) +{ + struct dict_commit_sync_result *sync_result = context; + + sync_result->ret = result->ret; + sync_result->error = i_strdup(result->error); +} + +int dict_transaction_commit(struct dict_transaction_context **_ctx, + const char **error_r) +{ + pool_t pool = pool_alloconly_create("dict_commit_callback_ctx", 64); + struct dict_commit_callback_ctx *cctx = + p_new(pool, struct dict_commit_callback_ctx, 1); + struct dict_transaction_context *ctx = *_ctx; + struct dict_commit_sync_result result; + + *_ctx = NULL; + cctx->pool = pool; + i_zero(&result); + i_assert(ctx->dict->transaction_count > 0); + ctx->dict->transaction_count--; + DLLIST_REMOVE(&ctx->dict->transactions, ctx); + DLLIST_PREPEND(&ctx->dict->commits, cctx); + cctx->dict = ctx->dict; + dict_ref(cctx->dict); + cctx->callback = dict_transaction_commit_sync_callback; + cctx->context = &result; + cctx->event = ctx->event; + cctx->set = ctx->set; + + ctx->dict->v.transaction_commit(ctx, FALSE, dict_commit_callback, cctx); + *error_r = t_strdup(result.error); + i_free(result.error); + return result.ret; +} + +#undef dict_transaction_commit_async +void dict_transaction_commit_async(struct dict_transaction_context **_ctx, + dict_transaction_commit_callback_t *callback, + void *context) +{ + pool_t pool = pool_alloconly_create("dict_commit_callback_ctx", 64); + struct dict_commit_callback_ctx *cctx = + p_new(pool, struct dict_commit_callback_ctx, 1); + struct dict_transaction_context *ctx = *_ctx; + + *_ctx = NULL; + i_assert(ctx->dict->transaction_count > 0); + ctx->dict->transaction_count--; + DLLIST_REMOVE(&ctx->dict->transactions, ctx); + DLLIST_PREPEND(&ctx->dict->commits, cctx); + if (callback == NULL) + callback = dict_transaction_commit_async_noop_callback; + cctx->pool = pool; + cctx->dict = ctx->dict; + dict_ref(cctx->dict); + cctx->callback = callback; + cctx->context = context; + cctx->event = ctx->event; + cctx->set = ctx->set; + cctx->delayed_callback = TRUE; + ctx->dict->v.transaction_commit(ctx, TRUE, dict_commit_callback, cctx); + cctx->delayed_callback = FALSE; +} + +void dict_transaction_commit_async_nocallback( + struct dict_transaction_context **ctx) +{ + dict_transaction_commit_async(ctx, NULL, NULL); +} + +void dict_transaction_rollback(struct dict_transaction_context **_ctx) +{ + struct dict_transaction_context *ctx = *_ctx; + + if (ctx == NULL) + return; + + struct event *event = ctx->event; + + *_ctx = NULL; + i_assert(ctx->dict->transaction_count > 0); + ctx->dict->transaction_count--; + DLLIST_REMOVE(&ctx->dict->transactions, ctx); + struct dict_op_settings_private set_copy = ctx->set; + ctx->dict->v.transaction_rollback(ctx); + dict_transaction_finished(event, DICT_COMMIT_RET_OK, TRUE, NULL); + dict_op_settings_private_free(&set_copy); + event_unref(&event); +} + +void dict_set(struct dict_transaction_context *ctx, + const char *key, const char *value) +{ + i_assert(dict_key_prefix_is_valid(key, ctx->set.username)); + struct event_passthrough *e = event_create_passthrough(ctx->event)-> + set_name("dict_set_key")-> + add_str("key", key); + + e_debug(e->event(), "Setting '%s' to '%s'", key, value); + + T_BEGIN { + ctx->dict->v.set(ctx, key, value); + } T_END; + ctx->changed = TRUE; +} + +void dict_unset(struct dict_transaction_context *ctx, + const char *key) +{ + i_assert(dict_key_prefix_is_valid(key, ctx->set.username)); + struct event_passthrough *e = event_create_passthrough(ctx->event)-> + set_name("dict_unset_key")-> + add_str("key", key); + + e_debug(e->event(), "Unsetting '%s'", key); + + T_BEGIN { + ctx->dict->v.unset(ctx, key); + } T_END; + ctx->changed = TRUE; +} + +void dict_atomic_inc(struct dict_transaction_context *ctx, + const char *key, long long diff) +{ + i_assert(dict_key_prefix_is_valid(key, ctx->set.username)); + struct event_passthrough *e = event_create_passthrough(ctx->event)-> + set_name("dict_increment_key")-> + add_str("key", key); + + e_debug(e->event(), "Incrementing '%s' with %lld", key, diff); + + if (diff != 0) T_BEGIN { + ctx->dict->v.atomic_inc(ctx, key, diff); + ctx->changed = TRUE; + } T_END; +} + +const char *dict_escape_string(const char *str) +{ + const char *p; + string_t *ret; + + /* see if we need to escape it */ + for (p = str; *p != '\0'; p++) { + if (*p == '/' || *p == '\\') + break; + } + + if (*p == '\0') + return str; + + /* escape */ + ret = t_str_new((size_t) (p - str) + 128); + str_append_data(ret, str, (size_t) (p - str)); + + for (; *p != '\0'; p++) { + switch (*p) { + case '/': + str_append_c(ret, '\\'); + str_append_c(ret, '|'); + break; + case '\\': + str_append_c(ret, '\\'); + str_append_c(ret, '\\'); + break; + default: + str_append_c(ret, *p); + break; + } + } + return str_c(ret); +} + +const char *dict_unescape_string(const char *str) +{ + const char *p; + string_t *ret; + + /* see if we need to unescape it */ + for (p = str; *p != '\0'; p++) { + if (*p == '\\') + break; + } + + if (*p == '\0') + return str; + + /* unescape */ + ret = t_str_new((size_t) (p - str) + strlen(p) + 1); + str_append_data(ret, str, (size_t) (p - str)); + + for (; *p != '\0'; p++) { + if (*p != '\\') + str_append_c(ret, *p); + else { + if (*++p == '|') + str_append_c(ret, '/'); + else if (*p == '\0') + break; + else + str_append_c(ret, *p); + } + } + return str_c(ret); +} + +void dict_op_settings_dup(const struct dict_op_settings *source, + struct dict_op_settings_private *dest_r) +{ + i_zero(dest_r); + dest_r->username = i_strdup(source->username); + dest_r->home_dir = i_strdup(source->home_dir); + dest_r->hide_log_values = source->hide_log_values; +} + +void dict_op_settings_private_free(struct dict_op_settings_private *set) +{ + i_free(set->username); + i_free(set->home_dir); +} diff --git a/src/lib-dict/dict.h b/src/lib-dict/dict.h new file mode 100644 index 0000000..8cac9c7 --- /dev/null +++ b/src/lib-dict/dict.h @@ -0,0 +1,208 @@ +#ifndef DICT_H +#define DICT_H + +#define DICT_PATH_PRIVATE "priv/" +#define DICT_PATH_SHARED "shared/" + +struct timespec; +struct dict; +struct dict_iterate_context; + +enum dict_iterate_flags { + /* Recurse to all the sub-hierarchies (e.g. iterating "foo/" will + return "foo/a", but should it return "foo/a/b"?) */ + DICT_ITERATE_FLAG_RECURSE = 0x01, + /* Sort returned results by key */ + DICT_ITERATE_FLAG_SORT_BY_KEY = 0x02, + /* Sort returned results by value */ + DICT_ITERATE_FLAG_SORT_BY_VALUE = 0x04, + /* Don't return values, only keys */ + DICT_ITERATE_FLAG_NO_VALUE = 0x08, + /* Don't recurse at all. This is basically the same as dict_lookup(), + but it'll return all the rows instead of only the first one. */ + DICT_ITERATE_FLAG_EXACT_KEY = 0x10, + /* Perform iteration asynchronously. */ + DICT_ITERATE_FLAG_ASYNC = 0x20 +}; + +enum dict_data_type { + DICT_DATA_TYPE_STRING = 0, + DICT_DATA_TYPE_UINT32, + DICT_DATA_TYPE_LAST +}; + +struct dict_settings { + const char *base_dir; + /* set to parent event, if exists */ + struct event *event_parent; +}; + +struct dict_op_settings { + const char *username; + /* home directory for the user, if known */ + const char *home_dir; + + /* Hide values when logging about this transaction. */ + bool hide_log_values; +}; + +struct dict_lookup_result { + int ret; + + /* First returned value (ret > 0) */ + const char *value; + /* NULL-terminated list of all returned values (ret > 0) */ + const char *const *values; + + /* Error message for a failed lookup (ret < 0) */ + const char *error; +}; + +enum dict_commit_ret { + DICT_COMMIT_RET_OK = 1, + DICT_COMMIT_RET_NOTFOUND = 0, + DICT_COMMIT_RET_FAILED = -1, + /* write may or may not have succeeded (e.g. write timeout or + disconnected from server) */ + DICT_COMMIT_RET_WRITE_UNCERTAIN = -2, +}; + +struct dict_commit_result { + enum dict_commit_ret ret; + const char *error; +}; + +typedef void dict_lookup_callback_t(const struct dict_lookup_result *result, + void *context); +typedef void dict_iterate_callback_t(void *context); +typedef void +dict_transaction_commit_callback_t(const struct dict_commit_result *result, + void *context); + +void dict_driver_register(struct dict *driver); +void dict_driver_unregister(struct dict *driver); + +void dict_drivers_register_builtin(void); +void dict_drivers_unregister_builtin(void); + +void dict_drivers_register_all(void); +void dict_drivers_unregister_all(void); + +/* Open dictionary with given URI (type:data). + Returns 0 if ok, -1 if URI is invalid. */ +int dict_init(const char *uri, const struct dict_settings *set, + struct dict **dict_r, const char **error_r); +/* Close dictionary. */ +void dict_deinit(struct dict **dict); +/* Wait for all pending asynchronous operations to finish. */ +void dict_wait(struct dict *dict); +/* Switch the dict to the current ioloop. This can be used to do dict_wait() + among other IO work. Returns TRUE if there is actually some work that can + be waited on. */ +bool dict_switch_ioloop(struct dict *dict) ATTR_NOWARN_UNUSED_RESULT; + +/* Lookup value for key. Set it to NULL if it's not found. + Returns 1 if found, 0 if not found and -1 if lookup failed. */ +int dict_lookup(struct dict *dict, const struct dict_op_settings *set, pool_t pool, + const char *key, const char **value_r, const char **error_r); +void dict_lookup_async(struct dict *dict, const struct dict_op_settings *set, + const char *key, dict_lookup_callback_t *callback, + void *context); +#define dict_lookup_async(dict, set, key, callback, context) \ + dict_lookup_async(dict, set, key, (dict_lookup_callback_t *)(callback), \ + 1 ? (context) : \ + CALLBACK_TYPECHECK(callback, \ + void (*)(const struct dict_lookup_result *, typeof(context)))) + +/* Iterate through all values in a path. flag indicates how iteration + is carried out */ +struct dict_iterate_context * +dict_iterate_init(struct dict *dict, const struct dict_op_settings *set, + const char *path, enum dict_iterate_flags flags); +/* Set async callback. Note that if dict_iterate_init() already did all the + work, this callback may never be called. So after dict_iterate_init() you + should call dict_iterate() in any case to see if all the results are + already available. */ +void dict_iterate_set_async_callback(struct dict_iterate_context *ctx, + dict_iterate_callback_t *callback, + void *context); +#define dict_iterate_set_async_callback(ctx, callback, context) \ + dict_iterate_set_async_callback(ctx, (dict_iterate_callback_t *)(callback), \ + 1 ? (context) : \ + CALLBACK_TYPECHECK(callback, void (*)(typeof(context)))) +/* Limit how many rows will be returned by the iteration (0 = unlimited). + This allows backends to optimize the query (e.g. use LIMIT 1 with SQL). */ +void dict_iterate_set_limit(struct dict_iterate_context *ctx, + uint64_t max_rows); +/* If dict_iterate() returns FALSE, the iteration may be finished or if this + is an async iteration it may be waiting for more data. If this function + returns TRUE, the dict callback is called again with more data. If dict + supports multiple values, dict_iterate_values() can be used to return all + of them. dict_iterate() returns only the first value and ignores the rest. */ +bool dict_iterate_has_more(struct dict_iterate_context *ctx); +bool dict_iterate(struct dict_iterate_context *ctx, + const char **key_r, const char **value_r); +bool dict_iterate_values(struct dict_iterate_context *ctx, + const char **key_r, const char *const **values_r); +/* Returns 0 = ok, -1 = iteration failed */ +int dict_iterate_deinit(struct dict_iterate_context **ctx, const char **error_r); + +/* Start a new dictionary transaction. */ +struct dict_transaction_context * +dict_transaction_begin(struct dict *dict, const struct dict_op_settings *set); +/* Don't log a warning if the transaction commit took a long time. + This is needed if there are no guarantees that an asynchronous commit will + finish up anytime soon. Mainly useful for transactions which aren't + especially important whether they finish or not. */ +void dict_transaction_no_slowness_warning(struct dict_transaction_context *ctx); +/* Set write timestamp for the entire transaction. This must be set before + any changes are done and can't be changed afterwards. Currently only + dict-sql with Cassandra backend does anything with this. */ +void dict_transaction_set_timestamp(struct dict_transaction_context *ctx, + const struct timespec *ts); + +/* Set hide_log_values for the transaction. Currently only + dict-sql with Cassandra backend does anything with this. */ +void dict_transaction_set_hide_log_values(struct dict_transaction_context *ctx, + bool hide_log_values); +/* Commit the transaction. Returns 1 if ok, 0 if dict_atomic_inc() was used + on a nonexistent key, -1 if failed. */ +int dict_transaction_commit(struct dict_transaction_context **ctx, + const char **error_r); +/* Commit the transaction, but don't wait to see if it finishes successfully. + The callback is called when the transaction is finished. If it's not called + by the time you want to deinitialize dict, call dict_flush() to wait for the + result. */ +void dict_transaction_commit_async(struct dict_transaction_context **ctx, + dict_transaction_commit_callback_t *callback, + void *context) ATTR_NULL(2, 3); +#define dict_transaction_commit_async(ctx, callback, context) \ + dict_transaction_commit_async(ctx, (dict_transaction_commit_callback_t *)(callback), \ + 1 ? (context) : \ + CALLBACK_TYPECHECK(callback, \ + void (*)(const struct dict_commit_result *, typeof(context)))) +/* Same as dict_transaction_commit_async(), but don't call a callback. */ +void dict_transaction_commit_async_nocallback( + struct dict_transaction_context **ctx); +/* Rollback all changes made in transaction. */ +void dict_transaction_rollback(struct dict_transaction_context **ctx); + +/* Set key=value in dictionary. */ +void dict_set(struct dict_transaction_context *ctx, + const char *key, const char *value); +/* Unset a record in dictionary, identified by key*/ +void dict_unset(struct dict_transaction_context *ctx, + const char *key); +/* Increase/decrease a numeric value in dictionary. Note that the value is + changed when transaction is being committed, so you can't know beforehand + what the value will become. The value is updated only if it already exists, + otherwise commit() will return 0. */ +void dict_atomic_inc(struct dict_transaction_context *ctx, + const char *key, long long diff); + +/* Escape/unescape '/' characters in a string, so that it can be safely added + into path components in dict keys. */ +const char *dict_escape_string(const char *str); +const char *dict_unescape_string(const char *str); + +#endif diff --git a/src/lib-dict/test-dict-client.c b/src/lib-dict/test-dict-client.c new file mode 100644 index 0000000..4d4ca68 --- /dev/null +++ b/src/lib-dict/test-dict-client.c @@ -0,0 +1,117 @@ +/* Copyright (c) 2016-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "lib-signals.h" +#include "ioloop.h" +#include "dict-private.h" + +#include <stdio.h> + +static int pending = 0; +static volatile bool stop = FALSE; + +static void sig_die(const siginfo_t *si ATTR_UNUSED, void *context ATTR_UNUSED) +{ + stop = TRUE; +} + +static void lookup_callback(const struct dict_lookup_result *result, + void *context ATTR_UNUSED) +{ + if (result->error != NULL) + i_error("%s", result->error); + /*else if (result->ret == 0) + i_info("not found"); + else + i_info("%s", result->value);*/ + pending--; +} + +static void commit_callback(const struct dict_commit_result *result, + void *context ATTR_UNUSED) +{ + if (result->ret < 0) + i_error("commit %d", result->ret); + pending--; +} + +int main(int argc, char *argv[]) +{ + const char *prefix, *uri; + struct dict *dict; + struct dict_settings set; + struct dict_op_settings opset; + struct ioloop *ioloop; + const char *error; + char key[1000], value[100]; + + lib_init(); + lib_signals_init(); + ioloop = io_loop_create(); + lib_signals_set_handler(SIGINT, LIBSIG_FLAG_RESTART, sig_die, NULL); + dict_driver_register(&dict_driver_client); + + if (argc < 3) + i_fatal("Usage: <prefix> <uri>"); + prefix = argv[1]; + uri = argv[2]; + + i_zero(&set); + i_zero(&opset); + set.base_dir = "/var/run/dovecot"; + opset.username = "testuser"; + + if (dict_init(uri, &set, &dict, &error) < 0) + i_fatal("dict_init(%s) failed: %s", argv[1], error); + + while (!stop) { + i_snprintf(key, sizeof(key), "%s/%02x", prefix, + i_rand_limit(0xff)); + i_snprintf(value, sizeof(value), "%04x", i_rand_limit(0xffff)); + switch (i_rand_limit(4)) { + case 0: + pending++; + dict_lookup_async(dict, &opset, key, lookup_callback, NULL); + break; + case 1: { + struct dict_transaction_context *trans; + + pending++; + trans = dict_transaction_begin(dict, &opset); + dict_set(trans, key, value); + dict_transaction_commit_async(&trans, commit_callback, NULL); + break; + } + case 2: { + struct dict_transaction_context *trans; + + pending++; + trans = dict_transaction_begin(dict, &opset); + dict_unset(trans, key); + dict_transaction_commit_async(&trans, commit_callback, NULL); + break; + } + case 3: { + struct dict_iterate_context *iter; + const char *k, *v; + + iter = dict_iterate_init(dict, &opset, prefix, DICT_ITERATE_FLAG_EXACT_KEY); + while (dict_iterate(iter, &k, &v)) ; + if (dict_iterate_deinit(&iter, &error) < 0) + i_error("iter failed: %s", error); + break; + } + } + while (pending > 100) { + dict_wait(dict); + printf("%d\n", pending); fflush(stdout); + } + } + dict_wait(dict); + dict_deinit(&dict); + dict_driver_unregister(&dict_driver_client); + + io_loop_destroy(&ioloop); + lib_signals_deinit(); + lib_deinit(); +} diff --git a/src/lib-dict/test-dict.c b/src/lib-dict/test-dict.c new file mode 100644 index 0000000..04864ff --- /dev/null +++ b/src/lib-dict/test-dict.c @@ -0,0 +1,46 @@ +/* Copyright (c) 2010-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "dict-private.h" +#include "test-common.h" + +struct dict dict_driver_client; +struct dict dict_driver_file; +struct dict dict_driver_memcached; +struct dict dict_driver_memcached_ascii; +struct dict dict_driver_redis; + +static void test_dict_escape(void) +{ + static const char *input[] = { + "", "", + "foo", "foo", + "foo\\", "foo\\\\", + "foo\\bar", "foo\\\\bar", + "\\bar", "\\\\bar", + "foo/", "foo\\|", + "foo/bar", "foo\\|bar", + "/bar", "\\|bar", + "////", "\\|\\|\\|\\|", + "/", "\\|" + }; + unsigned int i; + + test_begin("dict escape"); + for (i = 0; i < N_ELEMENTS(input); i += 2) { + test_assert(strcmp(dict_escape_string(input[i]), input[i+1]) == 0); + test_assert(strcmp(dict_unescape_string(input[i+1]), input[i]) == 0); + } + test_assert(strcmp(dict_unescape_string("x\\"), "x") == 0); + test_assert(strcmp(dict_unescape_string("\\"), "") == 0); + test_end(); +} + +int main(void) +{ + static void (*const test_functions[])(void) = { + test_dict_escape, + NULL + }; + return test_run(test_functions); +} |