diff options
author | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-04-28 09:51:24 +0000 |
---|---|---|
committer | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-04-28 09:51:24 +0000 |
commit | f7548d6d28c313cf80e6f3ef89aed16a19815df1 (patch) | |
tree | a3f6f2a3f247293bee59ecd28e8cd8ceb6ca064a /src/stats | |
parent | Initial commit. (diff) | |
download | dovecot-f7548d6d28c313cf80e6f3ef89aed16a19815df1.tar.xz dovecot-f7548d6d28c313cf80e6f3ef89aed16a19815df1.zip |
Adding upstream version 1:2.3.19.1+dfsg1.upstream/1%2.3.19.1+dfsg1upstream
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to '')
33 files changed, 6199 insertions, 0 deletions
diff --git a/src/stats/Makefile.am b/src/stats/Makefile.am new file mode 100644 index 0000000..0d6f598 --- /dev/null +++ b/src/stats/Makefile.am @@ -0,0 +1,102 @@ +pkglibexecdir = $(libexecdir)/dovecot + +pkglibexec_PROGRAMS = stats + +noinst_LTLIBRARIES = libstats_local.la + +AM_CPPFLAGS = \ + -I$(top_srcdir)/src/lib \ + -I$(top_srcdir)/src/lib-settings \ + -I$(top_srcdir)/src/lib-master \ + -I$(top_srcdir)/src/lib-http \ + -I$(top_srcdir)/src/lib-ssl-iostream \ + -I$(top_srcdir)/src/lib-test \ + $(BINARY_CFLAGS) + +stats_LDADD = \ + $(noinst_LTLIBRARIES) \ + $(LIBDOVECOT) \ + $(DOVECOT_SSL_LIBS) \ + $(BINARY_LDFLAGS) \ + -lm + +stats_DEPENDENCIES = \ + $(noinst_LTLIBRARIES) \ + $(DOVECOT_SSL_LIBS) \ + $(LIBDOVECOT_DEPS) + +stats_services = \ + stats-service-openmetrics.c + +stats_SOURCES = \ + main.c + +libstats_local_la_SOURCES = \ + client-reader.c \ + client-writer.c \ + client-http.c \ + event-exporter-fmt.c \ + event-exporter-fmt-json.c \ + event-exporter-fmt-none.c \ + event-exporter-fmt-tab-text.c \ + event-exporter-transport-drop.c \ + event-exporter-transport-http-post.c \ + event-exporter-transport-log.c \ + $(stats_services) \ + stats-service.c \ + stats-event-category.c \ + stats-metrics.c \ + stats-settings.c + +noinst_HEADERS = \ + stats-common.h \ + client-reader.h \ + client-writer.h \ + client-http.h\ + event-exporter.h \ + stats-service.h \ + stats-service-private.h \ + stats-event-category.h \ + stats-metrics.h \ + stats-settings.h \ + test-stats-common.h + +test_libs = \ + $(noinst_LTLIBRARIES) \ + $(DOVECOT_SSL_LIBS) \ + $(LIBDOVECOT) \ + $(BINARY_LDFLAGS) \ + -lm + +test_deps = \ + $(noinst_LTLIBRARIES) \ + $(DOVECOT_SSL_LIBS) \ + $(LIBDOVECOT_DEPS) + +test_stats_metrics_SOURCES = test-stats-metrics.c test-stats-common.c +test_stats_metrics_LDADD = $(test_libs) +test_stats_metrics_DEPENDENCIES = $(test_deps) + +test_client_writer_SOURCES = test-client-writer.c test-stats-common.c +test_client_writer_LDADD = $(test_libs) +test_client_writer_DEPENDENCIES = $(test_deps) + +test_client_reader_SOURCES = test-client-reader.c test-stats-common.c +test_client_reader_LDADD = $(test_libs) +test_client_reader_DEPENDENCIES = $(test_deps) + +test_programs = test-stats-metrics test-client-writer test-client-reader +noinst_PROGRAMS = $(test_programs) + +check-local: + for bin in $(test_programs); do \ + if ! $(RUN_TEST) ./$$bin; then exit 1; fi; \ + done + +LIBDOVECOT_TEST_DEPS = \ + ../lib-ssl-iostream/libssl_iostream.la \ + ../lib-test/libtest.la \ + ../lib/liblib.la +LIBDOVECOT_TEST = \ + $(LIBDOVECOT_TEST_DEPS) \ + $(MODULE_LIBS) diff --git a/src/stats/Makefile.in b/src/stats/Makefile.in new file mode 100644 index 0000000..9a8da4a --- /dev/null +++ b/src/stats/Makefile.in @@ -0,0 +1,1024 @@ +# Makefile.in generated by automake 1.16.1 from Makefile.am. +# @configure_input@ + +# Copyright (C) 1994-2018 Free Software Foundation, Inc. + +# This Makefile.in is free software; the Free Software Foundation +# gives unlimited permission to copy and/or distribute it, +# with or without modifications, as long as this notice is preserved. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY, to the extent permitted by law; without +# even the implied warranty of MERCHANTABILITY or FITNESS FOR A +# PARTICULAR PURPOSE. + +@SET_MAKE@ + + + +VPATH = @srcdir@ +am__is_gnu_make = { \ + if test -z '$(MAKELEVEL)'; then \ + false; \ + elif test -n '$(MAKE_HOST)'; then \ + true; \ + elif test -n '$(MAKE_VERSION)' && test -n '$(CURDIR)'; then \ + true; \ + else \ + false; \ + fi; \ +} +am__make_running_with_option = \ + case $${target_option-} in \ + ?) ;; \ + *) echo "am__make_running_with_option: internal error: invalid" \ + "target option '$${target_option-}' specified" >&2; \ + exit 1;; \ + esac; \ + has_opt=no; \ + sane_makeflags=$$MAKEFLAGS; \ + if $(am__is_gnu_make); then \ + sane_makeflags=$$MFLAGS; \ + else \ + case $$MAKEFLAGS in \ + *\\[\ \ ]*) \ + bs=\\; \ + sane_makeflags=`printf '%s\n' "$$MAKEFLAGS" \ + | sed "s/$$bs$$bs[$$bs $$bs ]*//g"`;; \ + esac; \ + fi; \ + skip_next=no; \ + strip_trailopt () \ + { \ + flg=`printf '%s\n' "$$flg" | sed "s/$$1.*$$//"`; \ + }; \ + for flg in $$sane_makeflags; do \ + test $$skip_next = yes && { skip_next=no; continue; }; \ + case $$flg in \ + *=*|--*) continue;; \ + -*I) strip_trailopt 'I'; skip_next=yes;; \ + -*I?*) strip_trailopt 'I';; \ + -*O) strip_trailopt 'O'; skip_next=yes;; \ + -*O?*) strip_trailopt 'O';; \ + -*l) strip_trailopt 'l'; skip_next=yes;; \ + -*l?*) strip_trailopt 'l';; \ + -[dEDm]) skip_next=yes;; \ + -[JT]) skip_next=yes;; \ + esac; \ + case $$flg in \ + *$$target_option*) has_opt=yes; break;; \ + esac; \ + done; \ + test $$has_opt = yes +am__make_dryrun = (target_option=n; $(am__make_running_with_option)) +am__make_keepgoing = (target_option=k; $(am__make_running_with_option)) +pkgdatadir = $(datadir)/@PACKAGE@ +pkgincludedir = $(includedir)/@PACKAGE@ +pkglibdir = $(libdir)/@PACKAGE@ +am__cd = CDPATH="$${ZSH_VERSION+.}$(PATH_SEPARATOR)" && cd +install_sh_DATA = $(install_sh) -c -m 644 +install_sh_PROGRAM = $(install_sh) -c +install_sh_SCRIPT = $(install_sh) -c +INSTALL_HEADER = $(INSTALL_DATA) +transform = $(program_transform_name) +NORMAL_INSTALL = : +PRE_INSTALL = : +POST_INSTALL = : +NORMAL_UNINSTALL = : +PRE_UNINSTALL = : +POST_UNINSTALL = : +build_triplet = @build@ +host_triplet = @host@ +pkglibexec_PROGRAMS = stats$(EXEEXT) +noinst_PROGRAMS = $(am__EXEEXT_1) +subdir = src/stats +ACLOCAL_M4 = $(top_srcdir)/aclocal.m4 +am__aclocal_m4_deps = $(top_srcdir)/m4/ac_checktype2.m4 \ + $(top_srcdir)/m4/ac_typeof.m4 $(top_srcdir)/m4/arc4random.m4 \ + $(top_srcdir)/m4/blockdev.m4 $(top_srcdir)/m4/c99_vsnprintf.m4 \ + $(top_srcdir)/m4/clock_gettime.m4 $(top_srcdir)/m4/crypt.m4 \ + $(top_srcdir)/m4/crypt_xpg6.m4 $(top_srcdir)/m4/dbqlk.m4 \ + $(top_srcdir)/m4/dirent_dtype.m4 $(top_srcdir)/m4/dovecot.m4 \ + $(top_srcdir)/m4/fd_passing.m4 $(top_srcdir)/m4/fdatasync.m4 \ + $(top_srcdir)/m4/flexible_array_member.m4 \ + $(top_srcdir)/m4/glibc.m4 $(top_srcdir)/m4/gmtime_max.m4 \ + $(top_srcdir)/m4/gmtime_tm_gmtoff.m4 \ + $(top_srcdir)/m4/ioloop.m4 $(top_srcdir)/m4/iovec.m4 \ + $(top_srcdir)/m4/ipv6.m4 $(top_srcdir)/m4/libcap.m4 \ + $(top_srcdir)/m4/libtool.m4 $(top_srcdir)/m4/libwrap.m4 \ + $(top_srcdir)/m4/linux_mremap.m4 $(top_srcdir)/m4/ltoptions.m4 \ + $(top_srcdir)/m4/ltsugar.m4 $(top_srcdir)/m4/ltversion.m4 \ + $(top_srcdir)/m4/lt~obsolete.m4 $(top_srcdir)/m4/mmap_write.m4 \ + $(top_srcdir)/m4/mntctl.m4 $(top_srcdir)/m4/modules.m4 \ + $(top_srcdir)/m4/notify.m4 $(top_srcdir)/m4/nsl.m4 \ + $(top_srcdir)/m4/off_t_max.m4 $(top_srcdir)/m4/pkg.m4 \ + $(top_srcdir)/m4/pr_set_dumpable.m4 \ + $(top_srcdir)/m4/q_quotactl.m4 $(top_srcdir)/m4/quota.m4 \ + $(top_srcdir)/m4/random.m4 $(top_srcdir)/m4/rlimit.m4 \ + $(top_srcdir)/m4/sendfile.m4 $(top_srcdir)/m4/size_t_signed.m4 \ + $(top_srcdir)/m4/sockpeercred.m4 $(top_srcdir)/m4/sql.m4 \ + $(top_srcdir)/m4/ssl.m4 $(top_srcdir)/m4/st_tim.m4 \ + $(top_srcdir)/m4/static_array.m4 $(top_srcdir)/m4/test_with.m4 \ + $(top_srcdir)/m4/time_t.m4 $(top_srcdir)/m4/typeof.m4 \ + $(top_srcdir)/m4/typeof_dev_t.m4 \ + $(top_srcdir)/m4/uoff_t_max.m4 $(top_srcdir)/m4/vararg.m4 \ + $(top_srcdir)/m4/want_apparmor.m4 \ + $(top_srcdir)/m4/want_bsdauth.m4 \ + $(top_srcdir)/m4/want_bzlib.m4 \ + $(top_srcdir)/m4/want_cassandra.m4 \ + $(top_srcdir)/m4/want_cdb.m4 \ + $(top_srcdir)/m4/want_checkpassword.m4 \ + $(top_srcdir)/m4/want_clucene.m4 $(top_srcdir)/m4/want_db.m4 \ + $(top_srcdir)/m4/want_gssapi.m4 $(top_srcdir)/m4/want_icu.m4 \ + $(top_srcdir)/m4/want_ldap.m4 $(top_srcdir)/m4/want_lua.m4 \ + $(top_srcdir)/m4/want_lz4.m4 $(top_srcdir)/m4/want_lzma.m4 \ + $(top_srcdir)/m4/want_mysql.m4 $(top_srcdir)/m4/want_pam.m4 \ + $(top_srcdir)/m4/want_passwd.m4 $(top_srcdir)/m4/want_pgsql.m4 \ + $(top_srcdir)/m4/want_prefetch.m4 \ + $(top_srcdir)/m4/want_shadow.m4 \ + $(top_srcdir)/m4/want_sodium.m4 $(top_srcdir)/m4/want_solr.m4 \ + $(top_srcdir)/m4/want_sqlite.m4 \ + $(top_srcdir)/m4/want_stemmer.m4 \ + $(top_srcdir)/m4/want_systemd.m4 \ + $(top_srcdir)/m4/want_textcat.m4 \ + $(top_srcdir)/m4/want_unwind.m4 $(top_srcdir)/m4/want_zlib.m4 \ + $(top_srcdir)/m4/want_zstd.m4 $(top_srcdir)/configure.ac +am__configure_deps = $(am__aclocal_m4_deps) $(CONFIGURE_DEPENDENCIES) \ + $(ACLOCAL_M4) +DIST_COMMON = $(srcdir)/Makefile.am $(noinst_HEADERS) \ + $(am__DIST_COMMON) +mkinstalldirs = $(install_sh) -d +CONFIG_HEADER = $(top_builddir)/config.h +CONFIG_CLEAN_FILES = +CONFIG_CLEAN_VPATH_FILES = +am__EXEEXT_1 = test-stats-metrics$(EXEEXT) test-client-writer$(EXEEXT) \ + test-client-reader$(EXEEXT) +am__installdirs = "$(DESTDIR)$(pkglibexecdir)" +PROGRAMS = $(noinst_PROGRAMS) $(pkglibexec_PROGRAMS) +LTLIBRARIES = $(noinst_LTLIBRARIES) +libstats_local_la_LIBADD = +am__objects_1 = stats-service-openmetrics.lo +am_libstats_local_la_OBJECTS = client-reader.lo client-writer.lo \ + client-http.lo event-exporter-fmt.lo \ + event-exporter-fmt-json.lo event-exporter-fmt-none.lo \ + event-exporter-fmt-tab-text.lo \ + event-exporter-transport-drop.lo \ + event-exporter-transport-http-post.lo \ + event-exporter-transport-log.lo $(am__objects_1) \ + stats-service.lo stats-event-category.lo stats-metrics.lo \ + stats-settings.lo +libstats_local_la_OBJECTS = $(am_libstats_local_la_OBJECTS) +AM_V_lt = $(am__v_lt_@AM_V@) +am__v_lt_ = $(am__v_lt_@AM_DEFAULT_V@) +am__v_lt_0 = --silent +am__v_lt_1 = +am_stats_OBJECTS = main.$(OBJEXT) +stats_OBJECTS = $(am_stats_OBJECTS) +am__DEPENDENCIES_1 = +am_test_client_reader_OBJECTS = test-client-reader.$(OBJEXT) \ + test-stats-common.$(OBJEXT) +test_client_reader_OBJECTS = $(am_test_client_reader_OBJECTS) +am__DEPENDENCIES_2 = $(noinst_LTLIBRARIES) $(am__DEPENDENCIES_1) \ + $(am__DEPENDENCIES_1) +am_test_client_writer_OBJECTS = test-client-writer.$(OBJEXT) \ + test-stats-common.$(OBJEXT) +test_client_writer_OBJECTS = $(am_test_client_writer_OBJECTS) +am_test_stats_metrics_OBJECTS = test-stats-metrics.$(OBJEXT) \ + test-stats-common.$(OBJEXT) +test_stats_metrics_OBJECTS = $(am_test_stats_metrics_OBJECTS) +AM_V_P = $(am__v_P_@AM_V@) +am__v_P_ = $(am__v_P_@AM_DEFAULT_V@) +am__v_P_0 = false +am__v_P_1 = : +AM_V_GEN = $(am__v_GEN_@AM_V@) +am__v_GEN_ = $(am__v_GEN_@AM_DEFAULT_V@) +am__v_GEN_0 = @echo " GEN " $@; +am__v_GEN_1 = +AM_V_at = $(am__v_at_@AM_V@) +am__v_at_ = $(am__v_at_@AM_DEFAULT_V@) +am__v_at_0 = @ +am__v_at_1 = +DEFAULT_INCLUDES = -I.@am__isrc@ -I$(top_builddir) +depcomp = $(SHELL) $(top_srcdir)/depcomp +am__maybe_remake_depfiles = depfiles +am__depfiles_remade = ./$(DEPDIR)/client-http.Plo \ + ./$(DEPDIR)/client-reader.Plo ./$(DEPDIR)/client-writer.Plo \ + ./$(DEPDIR)/event-exporter-fmt-json.Plo \ + ./$(DEPDIR)/event-exporter-fmt-none.Plo \ + ./$(DEPDIR)/event-exporter-fmt-tab-text.Plo \ + ./$(DEPDIR)/event-exporter-fmt.Plo \ + ./$(DEPDIR)/event-exporter-transport-drop.Plo \ + ./$(DEPDIR)/event-exporter-transport-http-post.Plo \ + ./$(DEPDIR)/event-exporter-transport-log.Plo \ + ./$(DEPDIR)/main.Po ./$(DEPDIR)/stats-event-category.Plo \ + ./$(DEPDIR)/stats-metrics.Plo \ + ./$(DEPDIR)/stats-service-openmetrics.Plo \ + ./$(DEPDIR)/stats-service.Plo ./$(DEPDIR)/stats-settings.Plo \ + ./$(DEPDIR)/test-client-reader.Po \ + ./$(DEPDIR)/test-client-writer.Po \ + ./$(DEPDIR)/test-stats-common.Po \ + ./$(DEPDIR)/test-stats-metrics.Po +am__mv = mv -f +COMPILE = $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) \ + $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) +LTCOMPILE = $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) \ + $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) \ + $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) \ + $(AM_CFLAGS) $(CFLAGS) +AM_V_CC = $(am__v_CC_@AM_V@) +am__v_CC_ = $(am__v_CC_@AM_DEFAULT_V@) +am__v_CC_0 = @echo " CC " $@; +am__v_CC_1 = +CCLD = $(CC) +LINK = $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) \ + $(LIBTOOLFLAGS) --mode=link $(CCLD) $(AM_CFLAGS) $(CFLAGS) \ + $(AM_LDFLAGS) $(LDFLAGS) -o $@ +AM_V_CCLD = $(am__v_CCLD_@AM_V@) +am__v_CCLD_ = $(am__v_CCLD_@AM_DEFAULT_V@) +am__v_CCLD_0 = @echo " CCLD " $@; +am__v_CCLD_1 = +SOURCES = $(libstats_local_la_SOURCES) $(stats_SOURCES) \ + $(test_client_reader_SOURCES) $(test_client_writer_SOURCES) \ + $(test_stats_metrics_SOURCES) +DIST_SOURCES = $(libstats_local_la_SOURCES) $(stats_SOURCES) \ + $(test_client_reader_SOURCES) $(test_client_writer_SOURCES) \ + $(test_stats_metrics_SOURCES) +am__can_run_installinfo = \ + case $$AM_UPDATE_INFO_DIR in \ + n|no|NO) false;; \ + *) (install-info --version) >/dev/null 2>&1;; \ + esac +HEADERS = $(noinst_HEADERS) +am__tagged_files = $(HEADERS) $(SOURCES) $(TAGS_FILES) $(LISP) +# Read a list of newline-separated strings from the standard input, +# and print each of them once, without duplicates. Input order is +# *not* preserved. +am__uniquify_input = $(AWK) '\ + BEGIN { nonempty = 0; } \ + { items[$$0] = 1; nonempty = 1; } \ + END { if (nonempty) { for (i in items) print i; }; } \ +' +# Make sure the list of sources is unique. This is necessary because, +# e.g., the same source file might be shared among _SOURCES variables +# for different programs/libraries. +am__define_uniq_tagged_files = \ + list='$(am__tagged_files)'; \ + unique=`for i in $$list; do \ + if test -f "$$i"; then echo $$i; else echo $(srcdir)/$$i; fi; \ + done | $(am__uniquify_input)` +ETAGS = etags +CTAGS = ctags +am__DIST_COMMON = $(srcdir)/Makefile.in $(top_srcdir)/depcomp +DISTFILES = $(DIST_COMMON) $(DIST_SOURCES) $(TEXINFOS) $(EXTRA_DIST) +pkglibexecdir = $(libexecdir)/dovecot +ACLOCAL = @ACLOCAL@ +ACLOCAL_AMFLAGS = @ACLOCAL_AMFLAGS@ +AMTAR = @AMTAR@ +AM_DEFAULT_VERBOSITY = @AM_DEFAULT_VERBOSITY@ +APPARMOR_LIBS = @APPARMOR_LIBS@ +AR = @AR@ +AUTH_CFLAGS = @AUTH_CFLAGS@ +AUTH_LIBS = @AUTH_LIBS@ +AUTOCONF = @AUTOCONF@ +AUTOHEADER = @AUTOHEADER@ +AUTOMAKE = @AUTOMAKE@ +AWK = @AWK@ +BINARY_CFLAGS = @BINARY_CFLAGS@ +BINARY_LDFLAGS = @BINARY_LDFLAGS@ +BISON = @BISON@ +CASSANDRA_CFLAGS = @CASSANDRA_CFLAGS@ +CASSANDRA_LIBS = @CASSANDRA_LIBS@ +CC = @CC@ +CCDEPMODE = @CCDEPMODE@ +CDB_LIBS = @CDB_LIBS@ +CFLAGS = @CFLAGS@ +CLUCENE_CFLAGS = @CLUCENE_CFLAGS@ +CLUCENE_LIBS = @CLUCENE_LIBS@ +COMPRESS_LIBS = @COMPRESS_LIBS@ +CPP = @CPP@ +CPPFLAGS = @CPPFLAGS@ +CRYPT_LIBS = @CRYPT_LIBS@ +CXX = @CXX@ +CXXCPP = @CXXCPP@ +CXXDEPMODE = @CXXDEPMODE@ +CXXFLAGS = @CXXFLAGS@ +CYGPATH_W = @CYGPATH_W@ +DEFS = @DEFS@ +DEPDIR = @DEPDIR@ +DICT_LIBS = @DICT_LIBS@ +DLLIB = @DLLIB@ +DLLTOOL = @DLLTOOL@ +DSYMUTIL = @DSYMUTIL@ +DUMPBIN = @DUMPBIN@ +ECHO_C = @ECHO_C@ +ECHO_N = @ECHO_N@ +ECHO_T = @ECHO_T@ +EGREP = @EGREP@ +EXEEXT = @EXEEXT@ +FGREP = @FGREP@ +FLEX = @FLEX@ +FUZZER_CPPFLAGS = @FUZZER_CPPFLAGS@ +FUZZER_LDFLAGS = @FUZZER_LDFLAGS@ +GREP = @GREP@ +INSTALL = @INSTALL@ +INSTALL_DATA = @INSTALL_DATA@ +INSTALL_PROGRAM = @INSTALL_PROGRAM@ +INSTALL_SCRIPT = @INSTALL_SCRIPT@ +INSTALL_STRIP_PROGRAM = @INSTALL_STRIP_PROGRAM@ +KRB5CONFIG = @KRB5CONFIG@ +KRB5_CFLAGS = @KRB5_CFLAGS@ +KRB5_LIBS = @KRB5_LIBS@ +LD = @LD@ +LDAP_LIBS = @LDAP_LIBS@ +LDFLAGS = @LDFLAGS@ +LD_NO_WHOLE_ARCHIVE = @LD_NO_WHOLE_ARCHIVE@ +LD_WHOLE_ARCHIVE = @LD_WHOLE_ARCHIVE@ +LIBCAP = @LIBCAP@ +LIBDOVECOT = @LIBDOVECOT@ +LIBDOVECOT_COMPRESS = @LIBDOVECOT_COMPRESS@ +LIBDOVECOT_DEPS = @LIBDOVECOT_DEPS@ +LIBDOVECOT_DSYNC = @LIBDOVECOT_DSYNC@ +LIBDOVECOT_LA_LIBS = @LIBDOVECOT_LA_LIBS@ +LIBDOVECOT_LDA = @LIBDOVECOT_LDA@ +LIBDOVECOT_LDAP = @LIBDOVECOT_LDAP@ +LIBDOVECOT_LIBFTS = @LIBDOVECOT_LIBFTS@ +LIBDOVECOT_LIBFTS_DEPS = @LIBDOVECOT_LIBFTS_DEPS@ +LIBDOVECOT_LOGIN = @LIBDOVECOT_LOGIN@ +LIBDOVECOT_LUA = @LIBDOVECOT_LUA@ +LIBDOVECOT_LUA_DEPS = @LIBDOVECOT_LUA_DEPS@ +LIBDOVECOT_SQL = @LIBDOVECOT_SQL@ +LIBDOVECOT_STORAGE = @LIBDOVECOT_STORAGE@ +LIBDOVECOT_STORAGE_DEPS = @LIBDOVECOT_STORAGE_DEPS@ +LIBEXTTEXTCAT_CFLAGS = @LIBEXTTEXTCAT_CFLAGS@ +LIBEXTTEXTCAT_LIBS = @LIBEXTTEXTCAT_LIBS@ +LIBICONV = @LIBICONV@ +LIBICU_CFLAGS = @LIBICU_CFLAGS@ +LIBICU_LIBS = @LIBICU_LIBS@ +LIBOBJS = @LIBOBJS@ +LIBS = @LIBS@ +LIBSODIUM_CFLAGS = @LIBSODIUM_CFLAGS@ +LIBSODIUM_LIBS = @LIBSODIUM_LIBS@ +LIBTIRPC_CFLAGS = @LIBTIRPC_CFLAGS@ +LIBTIRPC_LIBS = @LIBTIRPC_LIBS@ +LIBTOOL = @LIBTOOL@ +LIBUNWIND_CFLAGS = @LIBUNWIND_CFLAGS@ +LIBUNWIND_LIBS = @LIBUNWIND_LIBS@ +LIBWRAP_LIBS = @LIBWRAP_LIBS@ +LINKED_STORAGE_LDADD = @LINKED_STORAGE_LDADD@ +LIPO = @LIPO@ +LN_S = @LN_S@ +LTLIBICONV = @LTLIBICONV@ +LTLIBOBJS = @LTLIBOBJS@ +LT_SYS_LIBRARY_PATH = @LT_SYS_LIBRARY_PATH@ +LUA_CFLAGS = @LUA_CFLAGS@ +LUA_LIBS = @LUA_LIBS@ +MAINT = @MAINT@ +MAKEINFO = @MAKEINFO@ +MANIFEST_TOOL = @MANIFEST_TOOL@ +MKDIR_P = @MKDIR_P@ +MODULE_LIBS = @MODULE_LIBS@ +MODULE_SUFFIX = @MODULE_SUFFIX@ +MYSQL_CFLAGS = @MYSQL_CFLAGS@ +MYSQL_CONFIG = @MYSQL_CONFIG@ +MYSQL_LIBS = @MYSQL_LIBS@ +NM = @NM@ +NMEDIT = @NMEDIT@ +NOPLUGIN_LDFLAGS = @NOPLUGIN_LDFLAGS@ +OBJDUMP = @OBJDUMP@ +OBJEXT = @OBJEXT@ +OTOOL = @OTOOL@ +OTOOL64 = @OTOOL64@ +PACKAGE = @PACKAGE@ +PACKAGE_BUGREPORT = @PACKAGE_BUGREPORT@ +PACKAGE_NAME = @PACKAGE_NAME@ +PACKAGE_STRING = @PACKAGE_STRING@ +PACKAGE_TARNAME = @PACKAGE_TARNAME@ +PACKAGE_URL = @PACKAGE_URL@ +PACKAGE_VERSION = @PACKAGE_VERSION@ +PANDOC = @PANDOC@ +PATH_SEPARATOR = @PATH_SEPARATOR@ +PGSQL_CFLAGS = @PGSQL_CFLAGS@ +PGSQL_LIBS = @PGSQL_LIBS@ +PG_CONFIG = @PG_CONFIG@ +PIE_CFLAGS = @PIE_CFLAGS@ +PIE_LDFLAGS = @PIE_LDFLAGS@ +PKG_CONFIG = @PKG_CONFIG@ +PKG_CONFIG_LIBDIR = @PKG_CONFIG_LIBDIR@ +PKG_CONFIG_PATH = @PKG_CONFIG_PATH@ +QUOTA_LIBS = @QUOTA_LIBS@ +RANLIB = @RANLIB@ +RELRO_LDFLAGS = @RELRO_LDFLAGS@ +RPCGEN = @RPCGEN@ +RUN_TEST = @RUN_TEST@ +SED = @SED@ +SETTING_FILES = @SETTING_FILES@ +SET_MAKE = @SET_MAKE@ +SHELL = @SHELL@ +SQLITE_CFLAGS = @SQLITE_CFLAGS@ +SQLITE_LIBS = @SQLITE_LIBS@ +SQL_CFLAGS = @SQL_CFLAGS@ +SQL_LIBS = @SQL_LIBS@ +SSL_CFLAGS = @SSL_CFLAGS@ +SSL_LIBS = @SSL_LIBS@ +STRIP = @STRIP@ +SYSTEMD_CFLAGS = @SYSTEMD_CFLAGS@ +SYSTEMD_LIBS = @SYSTEMD_LIBS@ +VALGRIND = @VALGRIND@ +VERSION = @VERSION@ +ZSTD_CFLAGS = @ZSTD_CFLAGS@ +ZSTD_LIBS = @ZSTD_LIBS@ +abs_builddir = @abs_builddir@ +abs_srcdir = @abs_srcdir@ +abs_top_builddir = @abs_top_builddir@ +abs_top_srcdir = @abs_top_srcdir@ +ac_ct_AR = @ac_ct_AR@ +ac_ct_CC = @ac_ct_CC@ +ac_ct_CXX = @ac_ct_CXX@ +ac_ct_DUMPBIN = @ac_ct_DUMPBIN@ +am__include = @am__include@ +am__leading_dot = @am__leading_dot@ +am__quote = @am__quote@ +am__tar = @am__tar@ +am__untar = @am__untar@ +bindir = @bindir@ +build = @build@ +build_alias = @build_alias@ +build_cpu = @build_cpu@ +build_os = @build_os@ +build_vendor = @build_vendor@ +builddir = @builddir@ +datadir = @datadir@ +datarootdir = @datarootdir@ +dict_drivers = @dict_drivers@ +docdir = @docdir@ +dvidir = @dvidir@ +exec_prefix = @exec_prefix@ +host = @host@ +host_alias = @host_alias@ +host_cpu = @host_cpu@ +host_os = @host_os@ +host_vendor = @host_vendor@ +htmldir = @htmldir@ +includedir = @includedir@ +infodir = @infodir@ +install_sh = @install_sh@ +libdir = @libdir@ +libexecdir = @libexecdir@ +localedir = @localedir@ +localstatedir = @localstatedir@ +mandir = @mandir@ +mkdir_p = @mkdir_p@ +moduledir = @moduledir@ +oldincludedir = @oldincludedir@ +pdfdir = @pdfdir@ +prefix = @prefix@ +program_transform_name = @program_transform_name@ +psdir = @psdir@ +rundir = @rundir@ +runstatedir = @runstatedir@ +sbindir = @sbindir@ +sharedstatedir = @sharedstatedir@ +sql_drivers = @sql_drivers@ +srcdir = @srcdir@ +ssldir = @ssldir@ +statedir = @statedir@ +sysconfdir = @sysconfdir@ +systemdservicetype = @systemdservicetype@ +systemdsystemunitdir = @systemdsystemunitdir@ +target_alias = @target_alias@ +top_build_prefix = @top_build_prefix@ +top_builddir = @top_builddir@ +top_srcdir = @top_srcdir@ +noinst_LTLIBRARIES = libstats_local.la +AM_CPPFLAGS = \ + -I$(top_srcdir)/src/lib \ + -I$(top_srcdir)/src/lib-settings \ + -I$(top_srcdir)/src/lib-master \ + -I$(top_srcdir)/src/lib-http \ + -I$(top_srcdir)/src/lib-ssl-iostream \ + -I$(top_srcdir)/src/lib-test \ + $(BINARY_CFLAGS) + +stats_LDADD = \ + $(noinst_LTLIBRARIES) \ + $(LIBDOVECOT) \ + $(DOVECOT_SSL_LIBS) \ + $(BINARY_LDFLAGS) \ + -lm + +stats_DEPENDENCIES = \ + $(noinst_LTLIBRARIES) \ + $(DOVECOT_SSL_LIBS) \ + $(LIBDOVECOT_DEPS) + +stats_services = \ + stats-service-openmetrics.c + +stats_SOURCES = \ + main.c + +libstats_local_la_SOURCES = \ + client-reader.c \ + client-writer.c \ + client-http.c \ + event-exporter-fmt.c \ + event-exporter-fmt-json.c \ + event-exporter-fmt-none.c \ + event-exporter-fmt-tab-text.c \ + event-exporter-transport-drop.c \ + event-exporter-transport-http-post.c \ + event-exporter-transport-log.c \ + $(stats_services) \ + stats-service.c \ + stats-event-category.c \ + stats-metrics.c \ + stats-settings.c + +noinst_HEADERS = \ + stats-common.h \ + client-reader.h \ + client-writer.h \ + client-http.h\ + event-exporter.h \ + stats-service.h \ + stats-service-private.h \ + stats-event-category.h \ + stats-metrics.h \ + stats-settings.h \ + test-stats-common.h + +test_libs = \ + $(noinst_LTLIBRARIES) \ + $(DOVECOT_SSL_LIBS) \ + $(LIBDOVECOT) \ + $(BINARY_LDFLAGS) \ + -lm + +test_deps = \ + $(noinst_LTLIBRARIES) \ + $(DOVECOT_SSL_LIBS) \ + $(LIBDOVECOT_DEPS) + +test_stats_metrics_SOURCES = test-stats-metrics.c test-stats-common.c +test_stats_metrics_LDADD = $(test_libs) +test_stats_metrics_DEPENDENCIES = $(test_deps) +test_client_writer_SOURCES = test-client-writer.c test-stats-common.c +test_client_writer_LDADD = $(test_libs) +test_client_writer_DEPENDENCIES = $(test_deps) +test_client_reader_SOURCES = test-client-reader.c test-stats-common.c +test_client_reader_LDADD = $(test_libs) +test_client_reader_DEPENDENCIES = $(test_deps) +test_programs = test-stats-metrics test-client-writer test-client-reader +LIBDOVECOT_TEST_DEPS = \ + ../lib-ssl-iostream/libssl_iostream.la \ + ../lib-test/libtest.la \ + ../lib/liblib.la + +LIBDOVECOT_TEST = \ + $(LIBDOVECOT_TEST_DEPS) \ + $(MODULE_LIBS) + +all: all-am + +.SUFFIXES: +.SUFFIXES: .c .lo .o .obj +$(srcdir)/Makefile.in: @MAINTAINER_MODE_TRUE@ $(srcdir)/Makefile.am $(am__configure_deps) + @for dep in $?; do \ + case '$(am__configure_deps)' in \ + *$$dep*) \ + ( cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh ) \ + && { if test -f $@; then exit 0; else break; fi; }; \ + exit 1;; \ + esac; \ + done; \ + echo ' cd $(top_srcdir) && $(AUTOMAKE) --foreign src/stats/Makefile'; \ + $(am__cd) $(top_srcdir) && \ + $(AUTOMAKE) --foreign src/stats/Makefile +Makefile: $(srcdir)/Makefile.in $(top_builddir)/config.status + @case '$?' in \ + *config.status*) \ + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh;; \ + *) \ + echo ' cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__maybe_remake_depfiles)'; \ + cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__maybe_remake_depfiles);; \ + esac; + +$(top_builddir)/config.status: $(top_srcdir)/configure $(CONFIG_STATUS_DEPENDENCIES) + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh + +$(top_srcdir)/configure: @MAINTAINER_MODE_TRUE@ $(am__configure_deps) + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh +$(ACLOCAL_M4): @MAINTAINER_MODE_TRUE@ $(am__aclocal_m4_deps) + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh +$(am__aclocal_m4_deps): + +clean-noinstPROGRAMS: + @list='$(noinst_PROGRAMS)'; test -n "$$list" || exit 0; \ + echo " rm -f" $$list; \ + rm -f $$list || exit $$?; \ + test -n "$(EXEEXT)" || exit 0; \ + list=`for p in $$list; do echo "$$p"; done | sed 's/$(EXEEXT)$$//'`; \ + echo " rm -f" $$list; \ + rm -f $$list +install-pkglibexecPROGRAMS: $(pkglibexec_PROGRAMS) + @$(NORMAL_INSTALL) + @list='$(pkglibexec_PROGRAMS)'; test -n "$(pkglibexecdir)" || list=; \ + if test -n "$$list"; then \ + echo " $(MKDIR_P) '$(DESTDIR)$(pkglibexecdir)'"; \ + $(MKDIR_P) "$(DESTDIR)$(pkglibexecdir)" || exit 1; \ + fi; \ + for p in $$list; do echo "$$p $$p"; done | \ + sed 's/$(EXEEXT)$$//' | \ + while read p p1; do if test -f $$p \ + || test -f $$p1 \ + ; then echo "$$p"; echo "$$p"; else :; fi; \ + done | \ + sed -e 'p;s,.*/,,;n;h' \ + -e 's|.*|.|' \ + -e 'p;x;s,.*/,,;s/$(EXEEXT)$$//;$(transform);s/$$/$(EXEEXT)/' | \ + sed 'N;N;N;s,\n, ,g' | \ + $(AWK) 'BEGIN { files["."] = ""; dirs["."] = 1 } \ + { d=$$3; if (dirs[d] != 1) { print "d", d; dirs[d] = 1 } \ + if ($$2 == $$4) files[d] = files[d] " " $$1; \ + else { print "f", $$3 "/" $$4, $$1; } } \ + END { for (d in files) print "f", d, files[d] }' | \ + while read type dir files; do \ + if test "$$dir" = .; then dir=; else dir=/$$dir; fi; \ + test -z "$$files" || { \ + echo " $(INSTALL_PROGRAM_ENV) $(LIBTOOL) $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=install $(INSTALL_PROGRAM) $$files '$(DESTDIR)$(pkglibexecdir)$$dir'"; \ + $(INSTALL_PROGRAM_ENV) $(LIBTOOL) $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=install $(INSTALL_PROGRAM) $$files "$(DESTDIR)$(pkglibexecdir)$$dir" || exit $$?; \ + } \ + ; done + +uninstall-pkglibexecPROGRAMS: + @$(NORMAL_UNINSTALL) + @list='$(pkglibexec_PROGRAMS)'; test -n "$(pkglibexecdir)" || list=; \ + files=`for p in $$list; do echo "$$p"; done | \ + sed -e 'h;s,^.*/,,;s/$(EXEEXT)$$//;$(transform)' \ + -e 's/$$/$(EXEEXT)/' \ + `; \ + test -n "$$list" || exit 0; \ + echo " ( cd '$(DESTDIR)$(pkglibexecdir)' && rm -f" $$files ")"; \ + cd "$(DESTDIR)$(pkglibexecdir)" && rm -f $$files + +clean-pkglibexecPROGRAMS: + @list='$(pkglibexec_PROGRAMS)'; test -n "$$list" || exit 0; \ + echo " rm -f" $$list; \ + rm -f $$list || exit $$?; \ + test -n "$(EXEEXT)" || exit 0; \ + list=`for p in $$list; do echo "$$p"; done | sed 's/$(EXEEXT)$$//'`; \ + echo " rm -f" $$list; \ + rm -f $$list + +clean-noinstLTLIBRARIES: + -test -z "$(noinst_LTLIBRARIES)" || rm -f $(noinst_LTLIBRARIES) + @list='$(noinst_LTLIBRARIES)'; \ + locs=`for p in $$list; do echo $$p; done | \ + sed 's|^[^/]*$$|.|; s|/[^/]*$$||; s|$$|/so_locations|' | \ + sort -u`; \ + test -z "$$locs" || { \ + echo rm -f $${locs}; \ + rm -f $${locs}; \ + } + +libstats_local.la: $(libstats_local_la_OBJECTS) $(libstats_local_la_DEPENDENCIES) $(EXTRA_libstats_local_la_DEPENDENCIES) + $(AM_V_CCLD)$(LINK) $(libstats_local_la_OBJECTS) $(libstats_local_la_LIBADD) $(LIBS) + +stats$(EXEEXT): $(stats_OBJECTS) $(stats_DEPENDENCIES) $(EXTRA_stats_DEPENDENCIES) + @rm -f stats$(EXEEXT) + $(AM_V_CCLD)$(LINK) $(stats_OBJECTS) $(stats_LDADD) $(LIBS) + +test-client-reader$(EXEEXT): $(test_client_reader_OBJECTS) $(test_client_reader_DEPENDENCIES) $(EXTRA_test_client_reader_DEPENDENCIES) + @rm -f test-client-reader$(EXEEXT) + $(AM_V_CCLD)$(LINK) $(test_client_reader_OBJECTS) $(test_client_reader_LDADD) $(LIBS) + +test-client-writer$(EXEEXT): $(test_client_writer_OBJECTS) $(test_client_writer_DEPENDENCIES) $(EXTRA_test_client_writer_DEPENDENCIES) + @rm -f test-client-writer$(EXEEXT) + $(AM_V_CCLD)$(LINK) $(test_client_writer_OBJECTS) $(test_client_writer_LDADD) $(LIBS) + +test-stats-metrics$(EXEEXT): $(test_stats_metrics_OBJECTS) $(test_stats_metrics_DEPENDENCIES) $(EXTRA_test_stats_metrics_DEPENDENCIES) + @rm -f test-stats-metrics$(EXEEXT) + $(AM_V_CCLD)$(LINK) $(test_stats_metrics_OBJECTS) $(test_stats_metrics_LDADD) $(LIBS) + +mostlyclean-compile: + -rm -f *.$(OBJEXT) + +distclean-compile: + -rm -f *.tab.c + +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/client-http.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/client-reader.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/client-writer.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/event-exporter-fmt-json.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/event-exporter-fmt-none.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/event-exporter-fmt-tab-text.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/event-exporter-fmt.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/event-exporter-transport-drop.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/event-exporter-transport-http-post.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/event-exporter-transport-log.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/main.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/stats-event-category.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/stats-metrics.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/stats-service-openmetrics.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/stats-service.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/stats-settings.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/test-client-reader.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/test-client-writer.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/test-stats-common.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/test-stats-metrics.Po@am__quote@ # am--include-marker + +$(am__depfiles_remade): + @$(MKDIR_P) $(@D) + @echo '# dummy' >$@-t && $(am__mv) $@-t $@ + +am--depfiles: $(am__depfiles_remade) + +.c.o: +@am__fastdepCC_TRUE@ $(AM_V_CC)$(COMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ $< +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='$<' object='$@' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(COMPILE) -c -o $@ $< + +.c.obj: +@am__fastdepCC_TRUE@ $(AM_V_CC)$(COMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ `$(CYGPATH_W) '$<'` +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='$<' object='$@' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(COMPILE) -c -o $@ `$(CYGPATH_W) '$<'` + +.c.lo: +@am__fastdepCC_TRUE@ $(AM_V_CC)$(LTCOMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ $< +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Plo +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='$<' object='$@' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(LTCOMPILE) -c -o $@ $< + +mostlyclean-libtool: + -rm -f *.lo + +clean-libtool: + -rm -rf .libs _libs + +ID: $(am__tagged_files) + $(am__define_uniq_tagged_files); mkid -fID $$unique +tags: tags-am +TAGS: tags + +tags-am: $(TAGS_DEPENDENCIES) $(am__tagged_files) + set x; \ + here=`pwd`; \ + $(am__define_uniq_tagged_files); \ + shift; \ + if test -z "$(ETAGS_ARGS)$$*$$unique"; then :; else \ + test -n "$$unique" || unique=$$empty_fix; \ + if test $$# -gt 0; then \ + $(ETAGS) $(ETAGSFLAGS) $(AM_ETAGSFLAGS) $(ETAGS_ARGS) \ + "$$@" $$unique; \ + else \ + $(ETAGS) $(ETAGSFLAGS) $(AM_ETAGSFLAGS) $(ETAGS_ARGS) \ + $$unique; \ + fi; \ + fi +ctags: ctags-am + +CTAGS: ctags +ctags-am: $(TAGS_DEPENDENCIES) $(am__tagged_files) + $(am__define_uniq_tagged_files); \ + test -z "$(CTAGS_ARGS)$$unique" \ + || $(CTAGS) $(CTAGSFLAGS) $(AM_CTAGSFLAGS) $(CTAGS_ARGS) \ + $$unique + +GTAGS: + here=`$(am__cd) $(top_builddir) && pwd` \ + && $(am__cd) $(top_srcdir) \ + && gtags -i $(GTAGS_ARGS) "$$here" +cscopelist: cscopelist-am + +cscopelist-am: $(am__tagged_files) + list='$(am__tagged_files)'; \ + case "$(srcdir)" in \ + [\\/]* | ?:[\\/]*) sdir="$(srcdir)" ;; \ + *) sdir=$(subdir)/$(srcdir) ;; \ + esac; \ + for i in $$list; do \ + if test -f "$$i"; then \ + echo "$(subdir)/$$i"; \ + else \ + echo "$$sdir/$$i"; \ + fi; \ + done >> $(top_builddir)/cscope.files + +distclean-tags: + -rm -f TAGS ID GTAGS GRTAGS GSYMS GPATH tags + +distdir: $(BUILT_SOURCES) + $(MAKE) $(AM_MAKEFLAGS) distdir-am + +distdir-am: $(DISTFILES) + @srcdirstrip=`echo "$(srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \ + topsrcdirstrip=`echo "$(top_srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \ + list='$(DISTFILES)'; \ + dist_files=`for file in $$list; do echo $$file; done | \ + sed -e "s|^$$srcdirstrip/||;t" \ + -e "s|^$$topsrcdirstrip/|$(top_builddir)/|;t"`; \ + case $$dist_files in \ + */*) $(MKDIR_P) `echo "$$dist_files" | \ + sed '/\//!d;s|^|$(distdir)/|;s,/[^/]*$$,,' | \ + sort -u` ;; \ + esac; \ + for file in $$dist_files; do \ + if test -f $$file || test -d $$file; then d=.; else d=$(srcdir); fi; \ + if test -d $$d/$$file; then \ + dir=`echo "/$$file" | sed -e 's,/[^/]*$$,,'`; \ + if test -d "$(distdir)/$$file"; then \ + find "$(distdir)/$$file" -type d ! -perm -700 -exec chmod u+rwx {} \;; \ + fi; \ + if test -d $(srcdir)/$$file && test $$d != $(srcdir); then \ + cp -fpR $(srcdir)/$$file "$(distdir)$$dir" || exit 1; \ + find "$(distdir)/$$file" -type d ! -perm -700 -exec chmod u+rwx {} \;; \ + fi; \ + cp -fpR $$d/$$file "$(distdir)$$dir" || exit 1; \ + else \ + test -f "$(distdir)/$$file" \ + || cp -p $$d/$$file "$(distdir)/$$file" \ + || exit 1; \ + fi; \ + done +check-am: all-am + $(MAKE) $(AM_MAKEFLAGS) check-local +check: check-am +all-am: Makefile $(PROGRAMS) $(LTLIBRARIES) $(HEADERS) +installdirs: + for dir in "$(DESTDIR)$(pkglibexecdir)"; do \ + test -z "$$dir" || $(MKDIR_P) "$$dir"; \ + done +install: install-am +install-exec: install-exec-am +install-data: install-data-am +uninstall: uninstall-am + +install-am: all-am + @$(MAKE) $(AM_MAKEFLAGS) install-exec-am install-data-am + +installcheck: installcheck-am +install-strip: + if test -z '$(STRIP)'; then \ + $(MAKE) $(AM_MAKEFLAGS) INSTALL_PROGRAM="$(INSTALL_STRIP_PROGRAM)" \ + install_sh_PROGRAM="$(INSTALL_STRIP_PROGRAM)" INSTALL_STRIP_FLAG=-s \ + install; \ + else \ + $(MAKE) $(AM_MAKEFLAGS) INSTALL_PROGRAM="$(INSTALL_STRIP_PROGRAM)" \ + install_sh_PROGRAM="$(INSTALL_STRIP_PROGRAM)" INSTALL_STRIP_FLAG=-s \ + "INSTALL_PROGRAM_ENV=STRIPPROG='$(STRIP)'" install; \ + fi +mostlyclean-generic: + +clean-generic: + +distclean-generic: + -test -z "$(CONFIG_CLEAN_FILES)" || rm -f $(CONFIG_CLEAN_FILES) + -test . = "$(srcdir)" || test -z "$(CONFIG_CLEAN_VPATH_FILES)" || rm -f $(CONFIG_CLEAN_VPATH_FILES) + +maintainer-clean-generic: + @echo "This command is intended for maintainers to use" + @echo "it deletes files that may require special tools to rebuild." +clean: clean-am + +clean-am: clean-generic clean-libtool clean-noinstLTLIBRARIES \ + clean-noinstPROGRAMS clean-pkglibexecPROGRAMS mostlyclean-am + +distclean: distclean-am + -rm -f ./$(DEPDIR)/client-http.Plo + -rm -f ./$(DEPDIR)/client-reader.Plo + -rm -f ./$(DEPDIR)/client-writer.Plo + -rm -f ./$(DEPDIR)/event-exporter-fmt-json.Plo + -rm -f ./$(DEPDIR)/event-exporter-fmt-none.Plo + -rm -f ./$(DEPDIR)/event-exporter-fmt-tab-text.Plo + -rm -f ./$(DEPDIR)/event-exporter-fmt.Plo + -rm -f ./$(DEPDIR)/event-exporter-transport-drop.Plo + -rm -f ./$(DEPDIR)/event-exporter-transport-http-post.Plo + -rm -f ./$(DEPDIR)/event-exporter-transport-log.Plo + -rm -f ./$(DEPDIR)/main.Po + -rm -f ./$(DEPDIR)/stats-event-category.Plo + -rm -f ./$(DEPDIR)/stats-metrics.Plo + -rm -f ./$(DEPDIR)/stats-service-openmetrics.Plo + -rm -f ./$(DEPDIR)/stats-service.Plo + -rm -f ./$(DEPDIR)/stats-settings.Plo + -rm -f ./$(DEPDIR)/test-client-reader.Po + -rm -f ./$(DEPDIR)/test-client-writer.Po + -rm -f ./$(DEPDIR)/test-stats-common.Po + -rm -f ./$(DEPDIR)/test-stats-metrics.Po + -rm -f Makefile +distclean-am: clean-am distclean-compile distclean-generic \ + distclean-tags + +dvi: dvi-am + +dvi-am: + +html: html-am + +html-am: + +info: info-am + +info-am: + +install-data-am: + +install-dvi: install-dvi-am + +install-dvi-am: + +install-exec-am: install-pkglibexecPROGRAMS + +install-html: install-html-am + +install-html-am: + +install-info: install-info-am + +install-info-am: + +install-man: + +install-pdf: install-pdf-am + +install-pdf-am: + +install-ps: install-ps-am + +install-ps-am: + +installcheck-am: + +maintainer-clean: maintainer-clean-am + -rm -f ./$(DEPDIR)/client-http.Plo + -rm -f ./$(DEPDIR)/client-reader.Plo + -rm -f ./$(DEPDIR)/client-writer.Plo + -rm -f ./$(DEPDIR)/event-exporter-fmt-json.Plo + -rm -f ./$(DEPDIR)/event-exporter-fmt-none.Plo + -rm -f ./$(DEPDIR)/event-exporter-fmt-tab-text.Plo + -rm -f ./$(DEPDIR)/event-exporter-fmt.Plo + -rm -f ./$(DEPDIR)/event-exporter-transport-drop.Plo + -rm -f ./$(DEPDIR)/event-exporter-transport-http-post.Plo + -rm -f ./$(DEPDIR)/event-exporter-transport-log.Plo + -rm -f ./$(DEPDIR)/main.Po + -rm -f ./$(DEPDIR)/stats-event-category.Plo + -rm -f ./$(DEPDIR)/stats-metrics.Plo + -rm -f ./$(DEPDIR)/stats-service-openmetrics.Plo + -rm -f ./$(DEPDIR)/stats-service.Plo + -rm -f ./$(DEPDIR)/stats-settings.Plo + -rm -f ./$(DEPDIR)/test-client-reader.Po + -rm -f ./$(DEPDIR)/test-client-writer.Po + -rm -f ./$(DEPDIR)/test-stats-common.Po + -rm -f ./$(DEPDIR)/test-stats-metrics.Po + -rm -f Makefile +maintainer-clean-am: distclean-am maintainer-clean-generic + +mostlyclean: mostlyclean-am + +mostlyclean-am: mostlyclean-compile mostlyclean-generic \ + mostlyclean-libtool + +pdf: pdf-am + +pdf-am: + +ps: ps-am + +ps-am: + +uninstall-am: uninstall-pkglibexecPROGRAMS + +.MAKE: check-am install-am install-strip + +.PHONY: CTAGS GTAGS TAGS all all-am am--depfiles check check-am \ + check-local clean clean-generic clean-libtool \ + clean-noinstLTLIBRARIES clean-noinstPROGRAMS \ + clean-pkglibexecPROGRAMS cscopelist-am ctags ctags-am \ + distclean distclean-compile distclean-generic \ + distclean-libtool distclean-tags distdir dvi dvi-am html \ + html-am info info-am install install-am install-data \ + install-data-am install-dvi install-dvi-am install-exec \ + install-exec-am install-html install-html-am install-info \ + install-info-am install-man install-pdf install-pdf-am \ + install-pkglibexecPROGRAMS install-ps install-ps-am \ + install-strip installcheck installcheck-am installdirs \ + maintainer-clean maintainer-clean-generic mostlyclean \ + mostlyclean-compile mostlyclean-generic mostlyclean-libtool \ + pdf pdf-am ps ps-am tags tags-am uninstall uninstall-am \ + uninstall-pkglibexecPROGRAMS + +.PRECIOUS: Makefile + + +check-local: + for bin in $(test_programs); do \ + if ! $(RUN_TEST) ./$$bin; then exit 1; fi; \ + done + +# Tell versions [3.59,3.63) of GNU make to not export all variables. +# Otherwise a system limit (for SysV at least) may be exceeded. +.NOEXPORT: diff --git a/src/stats/client-http.c b/src/stats/client-http.c new file mode 100644 index 0000000..1811917 --- /dev/null +++ b/src/stats/client-http.c @@ -0,0 +1,233 @@ +/* Copyright (c) 2019 Dovecot authors, see the included COPYING file */ + +#include "stats-common.h" +#include "str.h" +#include "array.h" +#include "strescape.h" +#include "connection.h" +#include "ostream.h" +#include "master-service.h" +#include "http-server.h" +#include "http-url.h" +#include "stats-metrics.h" +#include "stats-service.h" +#include "client-http.h" + +struct stats_http_client; + +struct stats_http_client { + struct http_server_connection *http_conn; +}; + +struct stats_http_resource { + pool_t pool; + const char *title; + struct http_server_resource *resource; + + stats_http_resource_callback_t *callback; + void *context; +}; + +static struct http_server *stats_http_server; +static ARRAY(struct stats_http_resource *) stats_http_resources; + +/* + * Request + */ + +static void +stats_http_server_handle_request(void *context ATTR_UNUSED, + struct http_server_request *http_sreq) +{ + http_server_request_fail(http_sreq, 404, "Path Not Found"); +} + +/* + * Connection + */ + +static void +stats_http_server_connection_destroy(void *context, const char *reason); + +static const struct http_server_callbacks stats_http_callbacks = { + .connection_destroy = stats_http_server_connection_destroy, + .handle_request = stats_http_server_handle_request +}; + +void client_http_create(struct master_service_connection *conn) +{ + struct stats_http_client *client; + + client = i_new(struct stats_http_client, 1); + + client->http_conn = http_server_connection_create( + stats_http_server, conn->fd, conn->fd, conn->ssl, + &stats_http_callbacks, client); +} + +static void stats_http_client_destroy(struct stats_http_client *client) +{ + i_free(client); + + master_service_client_connection_destroyed(master_service); +} + +static void +stats_http_server_connection_destroy(void *context, + const char *reason ATTR_UNUSED) +{ + struct stats_http_client *client = context; + + if (client->http_conn == NULL) { + /* Already destroying client directly */ + return; + } + + /* HTTP connection is destroyed already now */ + client->http_conn = NULL; + + /* Destroy the connection itself */ + stats_http_client_destroy(client); +} + +/* + * Resources + */ + +/* Registry */ + +static void +stats_http_resource_callback(struct stats_http_resource *res, + struct http_server_request *req, + const char *sub_path) +{ + res->callback(res->context, req, sub_path); +} + +#undef stats_http_resource_add +void stats_http_resource_add(const char *path, const char *title, + stats_http_resource_callback_t *callback, + void *context) +{ + struct stats_http_resource *res; + pool_t pool; + + pool = pool_alloconly_create("stats http resource", 2048); + res = p_new(pool, struct stats_http_resource, 1); + res->pool = pool; + res->title = p_strdup(pool, title); + res->callback = callback; + res->context = context; + + res->resource = http_server_resource_create( + stats_http_server, pool, stats_http_resource_callback, res); + http_server_resource_add_location(res->resource, path); + + pool_unref(&pool); + array_append(&stats_http_resources, &res, 1); +} + +/* Root */ + +static void +stats_http_resource_root_make_response(struct http_server_response *resp, + const struct http_request *hreq) +{ + struct stats_http_resource *res; + struct http_url url; + string_t *msg; + + http_url_init_authority_from(&url, hreq->target.url); + + msg = t_str_new(1024); + + str_append(msg, "<!DOCTYPE html>\n"); + str_append(msg, "<html lang=\"en\">\n"); + str_append(msg, "\n"); + str_append(msg, "<head>\n"); + str_append(msg, "<meta charset=\"utf-8\">\n"); + str_append(msg, "<title>Dovecot Stats</title>\n"); + str_append(msg, "</head>\n"); + str_append(msg, "\n"); + str_append(msg, "<body>\n"); + + str_append(msg, "<h1>Dovecot Stats:</h1>\n"); + str_append(msg, "<p><ul>\n"); + + array_foreach_elem(&stats_http_resources, res) { + if (res->title == NULL) + continue; + + /* List the resource at its primary location. */ + url.path = http_server_resource_get_path(res->resource); + + str_append(msg, "<li><a href=\""); + str_append(msg, http_url_create(&url)); + str_append(msg, "\">"); + str_append(msg, res->title); + str_append(msg, "</a></li>\n"); + } + + str_append(msg, "</ul></p>\n"); + str_append(msg, "</body>\n"); + str_append(msg, "\n"); + str_append(msg, "</html>\n"); + + http_server_response_set_payload_data( + resp, str_data(msg), str_len(msg)); +} + +static void +stats_http_resource_root_request(void *context ATTR_UNUSED, + struct http_server_request *req, + const char *sub_path) +{ + const struct http_request *hreq = http_server_request_get(req); + struct http_server_response *resp; + + if (strcmp(hreq->method, "OPTIONS") == 0) { + resp = http_server_response_create(req, 200, "OK"); + http_server_response_add_header(resp, "Allow", "GET"); + http_server_response_submit(resp); + return; + } + if (strcmp(hreq->method, "GET") != 0) { + http_server_request_fail_bad_method(req, "GET"); + return; + } + if (*sub_path != '\0') { + http_server_request_fail(req, 404, "Not Found"); + return; + } + + resp = http_server_response_create(req, 200, "OK"); + http_server_response_add_header(resp, "Content-Type", + "text/html; charset=utf-8"); + + stats_http_resource_root_make_response(resp, hreq); + + http_server_response_submit(resp); +} + +/* + * Server + */ + +void client_http_init(const struct stats_settings *set) +{ + struct http_server_settings http_set = { + .rawlog_dir = set->stats_http_rawlog_dir, + }; + + i_array_init(&stats_http_resources, 8); + + stats_http_server = http_server_init(&http_set); + stats_http_resource_add("/", NULL, + stats_http_resource_root_request, NULL); +} + +void client_http_deinit(void) +{ + http_server_deinit(&stats_http_server); + array_free(&stats_http_resources); +} diff --git a/src/stats/client-http.h b/src/stats/client-http.h new file mode 100644 index 0000000..4251ce3 --- /dev/null +++ b/src/stats/client-http.h @@ -0,0 +1,28 @@ +#ifndef CLIENT_HTTP_H +#define CLIENT_HTTP_H + +struct master_service_connection; +struct http_server_request; + +typedef void +(stats_http_resource_callback_t)(void *context, + struct http_server_request *req, + const char *sub_path); + +void client_http_create(struct master_service_connection *conn); + +void stats_http_resource_add(const char *path, const char *title, + stats_http_resource_callback_t *callback, + void *context); +#define stats_http_resource_add(path, title, callback, context) \ + stats_http_resource_add(path, title, \ + (stats_http_resource_callback_t *)callback, \ + (TRUE ? context : \ + CALLBACK_TYPECHECK(callback, void (*)( \ + typeof(context), struct http_server_request *req, \ + const char *sub_path)))) + +void client_http_init(const struct stats_settings *set); +void client_http_deinit(void); + +#endif diff --git a/src/stats/client-reader.c b/src/stats/client-reader.c new file mode 100644 index 0000000..e944001 --- /dev/null +++ b/src/stats/client-reader.c @@ -0,0 +1,253 @@ +/* Copyright (c) 2017-2018 Dovecot authors, see the included COPYING file */ + +#include "stats-common.h" +#include "array.h" +#include "str.h" +#include "stats-dist.h" +#include "strescape.h" +#include "connection.h" +#include "ostream.h" +#include "master-service.h" +#include "stats-metrics.h" +#include "stats-settings.h" +#include "client-reader.h" +#include "client-writer.h" + +struct reader_client { + struct connection conn; +}; + +static struct connection_list *reader_clients = NULL; + +void client_reader_create(int fd) +{ + struct reader_client *client; + + client = i_new(struct reader_client, 1); + connection_init_server(reader_clients, &client->conn, + "stats-reader", fd, fd); +} + +static void reader_client_destroy(struct connection *conn) +{ + connection_deinit(conn); + i_free(conn); + + master_service_client_connection_destroyed(master_service); +} + +static void reader_client_dump_stats(string_t *str, struct stats_dist *stats, + const char *const *fields) +{ + for (unsigned int i = 0; fields[i] != NULL; i++) { + const char *field = fields[i]; + + str_append_c(str, '\t'); + if (strcmp(field, "count") == 0) + str_printfa(str, "%u", stats_dist_get_count(stats)); + else if (strcmp(field, "sum") == 0) + str_printfa(str, "%"PRIu64, stats_dist_get_sum(stats)); + else if (strcmp(field, "min") == 0) + str_printfa(str, "%"PRIu64, stats_dist_get_min(stats)); + else if (strcmp(field, "max") == 0) + str_printfa(str, "%"PRIu64, stats_dist_get_max(stats)); + else if (strcmp(field, "avg") == 0) + str_printfa(str, "%.02f", stats_dist_get_avg(stats)); + else if (strcmp(field, "median") == 0) + str_printfa(str, "%"PRIu64, stats_dist_get_median(stats)); + else if (strcmp(field, "variance") == 0) + str_printfa(str, "%.02f", stats_dist_get_variance(stats)); + else if (field[0] == '%') { + str_printfa(str, "%"PRIu64, + stats_dist_get_percentile(stats, strtod(field+1, NULL)/100.0)); + } else { + /* return unknown fields as empty */ + } + } +} + +static void reader_client_dump_metric(string_t *str, const struct metric *metric, + const char *const *fields) +{ + reader_client_dump_stats(str, metric->duration_stats, fields); + for (unsigned int i = 0; i < metric->fields_count; i++) { + str_append_c(str, '\t'); + str_append_tabescaped(str, metric->fields[i].field_key); + reader_client_dump_stats(str, metric->fields[i].stats, fields); + } + str_append_c(str, '\n'); +} + +static void +reader_client_append_sub_name(string_t *str, const char *sub_name) +{ + for (; *sub_name != '\0'; sub_name++) { + switch (*sub_name) { + case '\t': + case '\n': + case '\r': + case ' ': + str_append_c(str, '_'); + break; + default: + str_append_c(str, *sub_name); + } + } +} + +static void +reader_client_dump_sub_metrics(struct ostream *output, const struct metric *metric, + const char *sub_name, const char *const *fields) +{ + size_t root_pos, name_pos; + struct metric *const *sub_metrics; + if (!array_is_created(&metric->sub_metrics)) + return; + string_t *str = t_str_new(128); + reader_client_append_sub_name(str, sub_name); + str_append_c(str, '_'); + root_pos = str->used; + + array_foreach(&metric->sub_metrics, sub_metrics) { + str_truncate(str, root_pos); + reader_client_append_sub_name(str, (*sub_metrics)->sub_name); + name_pos = str->used; + reader_client_dump_metric(str, *sub_metrics, fields); + o_stream_nsend(output, str_data(str), str_len(str)); + str_truncate(str, name_pos); + reader_client_dump_sub_metrics(output, *sub_metrics, + str_c(str), fields); + } +} + +static int +reader_client_input_dump(struct reader_client *client, const char *const *args) +{ + struct stats_metrics_iter *iter; + const struct metric *metric; + + o_stream_cork(client->conn.output); + iter = stats_metrics_iterate_init(stats_metrics); + while ((metric = stats_metrics_iterate(iter)) != NULL) T_BEGIN { + string_t *str = t_str_new(128); + str_append_tabescaped(str, metric->name); + reader_client_dump_metric(str, metric, args); + o_stream_nsend(client->conn.output, str_data(str), str_len(str)); + reader_client_dump_sub_metrics(client->conn.output, metric, + metric->name, args); + } T_END; + o_stream_nsend(client->conn.output, "\n", 1); + stats_metrics_iterate_deinit(&iter); + o_stream_uncork(client->conn.output); + return 1; +} + +static int +reader_client_input_dump_reset(struct reader_client *client, + const char *const *args) +{ + (void)reader_client_input_dump(client, args); + stats_metrics_reset(stats_metrics); + return 1; +} + +static int +reader_client_input_metrics_add(struct reader_client *client, + const char *const *args) +{ + const char *error; + + if (str_array_length(args) < 7) { + e_error(client->conn.event, "METRICS-ADD: Not enough parameters"); + return -1; + } + + struct stats_metric_settings set = { + .metric_name = args[0], + .description = args[1], + .fields = args[2], + .group_by = args[3], + .filter = args[4], + .exporter = args[5], + .exporter_include = args[6], + }; + o_stream_cork(client->conn.output); + if (stats_metrics_add_dynamic(stats_metrics, &set, &error)) { + client_writer_update_connections(); + o_stream_nsend(client->conn.output, "+", 1); + } else { + o_stream_nsend(client->conn.output, "-", 1); + o_stream_nsend_str(client->conn.output, "METRICS-ADD: "); + o_stream_nsend_str(client->conn.output, error); + } + o_stream_nsend(client->conn.output, "\n", 1); + o_stream_uncork(client->conn.output); + return 1; +} + +static int +reader_client_input_metrics_remove(struct reader_client *client, + const char *const *args) +{ + if (str_array_length(args) < 1) { + e_error(client->conn.event, "METRICS-REMOVE: Not enough parameters"); + return -1; + } + + if (stats_metrics_remove_dynamic(stats_metrics, args[0])) { + client_writer_update_connections(); + o_stream_nsend(client->conn.output, "+\n", 2); + } else { + o_stream_nsend_str(client->conn.output, + t_strdup_printf("-metrics '%s' not found\n", args[0])); + } + return 1; +} + +static int +reader_client_input_args(struct connection *conn, const char *const *args) +{ + struct reader_client *client = (struct reader_client *)conn; + const char *cmd = args[0]; + + if (cmd == NULL) { + i_error("Client sent empty line"); + return 1; + } + args++; + if (strcmp(cmd, "DUMP") == 0) + return reader_client_input_dump(client, args); + else if (strcmp(cmd, "METRICS-ADD") == 0) + return reader_client_input_metrics_add(client, args); + else if (strcmp(cmd, "METRICS-REMOVE") == 0) + return reader_client_input_metrics_remove(client, args); + else if (strcmp(cmd, "DUMP-RESET") == 0) + return reader_client_input_dump_reset(client, args); + return 1; +} + +static struct connection_settings client_set = { + .service_name_in = "stats-reader-client", + .service_name_out = "stats-reader-server", + .major_version = 2, + .minor_version = 0, + + .input_max_size = 1024, + .output_max_size = SIZE_MAX, + .client = FALSE, +}; + +static const struct connection_vfuncs client_vfuncs = { + .destroy = reader_client_destroy, + .input_args = reader_client_input_args, +}; + +void client_readers_init(void) +{ + reader_clients = connection_list_init(&client_set, &client_vfuncs); +} + +void client_readers_deinit(void) +{ + connection_list_deinit(&reader_clients); +} diff --git a/src/stats/client-reader.h b/src/stats/client-reader.h new file mode 100644 index 0000000..2acab19 --- /dev/null +++ b/src/stats/client-reader.h @@ -0,0 +1,11 @@ +#ifndef CLIENT_READER_H +#define CLIENT_READER_H + +struct stats_metrics; + +void client_reader_create(int fd); + +void client_readers_init(void); +void client_readers_deinit(void); + +#endif diff --git a/src/stats/client-writer.c b/src/stats/client-writer.c new file mode 100644 index 0000000..40772cf --- /dev/null +++ b/src/stats/client-writer.c @@ -0,0 +1,373 @@ +/* Copyright (c) 2017-2018 Dovecot authors, see the included COPYING file */ + +#include "stats-common.h" +#include "array.h" +#include "llist.h" +#include "hash.h" +#include "str.h" +#include "strescape.h" +#include "lib-event-private.h" +#include "event-filter.h" +#include "ostream.h" +#include "connection.h" +#include "master-service.h" +#include "stats-event-category.h" +#include "stats-metrics.h" +#include "stats-settings.h" +#include "client-writer.h" + +#define STATS_UPDATE_CLIENTS_DELAY_MSECS 1000 + +struct stats_event { + struct stats_event *prev, *next; + + uint64_t id; + struct event *event; +}; + +struct writer_client { + struct connection conn; + + struct stats_event *events; + HASH_TABLE(struct stats_event *, struct stats_event *) events_hash; +}; + +static struct timeout *to_update_clients; +static struct connection_list *writer_clients = NULL; + +static void client_writer_send_handshake(struct writer_client *client) +{ + string_t *filter = t_str_new(128); + string_t *str = t_str_new(128); + + event_filter_export(stats_metrics_get_event_filter(stats_metrics), filter); + + str_append(str, "FILTER\t"); + str_append_tabescaped(str, str_c(filter)); + str_append_c(str, '\n'); + o_stream_nsend(client->conn.output, str_data(str), str_len(str)); +} + +static unsigned int stats_event_hash(const struct stats_event *event) +{ + return (unsigned int)event->id; +} + +static int stats_event_cmp(const struct stats_event *event1, + const struct stats_event *event2) +{ + return event1->id == event2->id ? 0 : 1; +} + +void client_writer_create(int fd) +{ + struct writer_client *client; + + client = i_new(struct writer_client, 1); + hash_table_create(&client->events_hash, default_pool, 0, + stats_event_hash, stats_event_cmp); + + connection_init_server(writer_clients, &client->conn, + "stats", fd, fd); + client_writer_send_handshake(client); +} + +static void writer_client_destroy(struct connection *conn) +{ + struct writer_client *client = (struct writer_client *)conn; + struct stats_event *event, *next; + + for (event = client->events; event != NULL; event = next) { + next = event->next; + event_unref(&event->event); + i_free(event); + } + hash_table_destroy(&client->events_hash); + + connection_deinit(conn); + i_free(conn); + + master_service_client_connection_destroyed(master_service); +} + +static struct stats_event * +writer_client_find_event(struct writer_client *client, uint64_t event_id) +{ + struct stats_event lookup_event = { .id = event_id }; + return hash_table_lookup(client->events_hash, &lookup_event); +} + +static bool +writer_client_run_event(struct writer_client *client, + uint64_t parent_event_id, const char *const *args, + struct event **event_r, const char **error_r) +{ + struct event *parent_event; + unsigned int log_type; + + if (parent_event_id == 0) + parent_event = NULL; + else { + struct stats_event *stats_parent_event = + writer_client_find_event(client, parent_event_id); + if (stats_parent_event == NULL) { + *error_r = "Unknown parent event ID"; + return FALSE; + } + parent_event = stats_parent_event->event; + } + if (args[0] == NULL || str_to_uint(args[0], &log_type) < 0 || + log_type >= LOG_TYPE_COUNT) { + *error_r = "Invalid log type"; + return FALSE; + } + const struct failure_context ctx = { + .type = (enum log_type)log_type + }; + args++; + + struct event *event = event_create(parent_event); + if (!event_import_unescaped(event, args, error_r)) { + event_unref(&event); + return FALSE; + } + stats_metrics_event(stats_metrics, event, &ctx); + *event_r = event; + return TRUE; +} + +static bool +writer_client_input_event(struct writer_client *client, + const char *const *args, const char **error_r) +{ + struct event *event, *global_event = NULL; + uint64_t parent_event_id, global_event_id; + bool ret; + + if (args[1] == NULL || str_to_uint64(args[0], &global_event_id) < 0) { + *error_r = "Invalid global event ID"; + return FALSE; + } + if (args[1] == NULL || str_to_uint64(args[1], &parent_event_id) < 0) { + *error_r = "Invalid parent ID"; + return FALSE; + } + + if (global_event_id != 0) { + struct stats_event *stats_global_event = + writer_client_find_event(client, global_event_id); + if (stats_global_event == NULL) { + *error_r = "Unknown global event ID"; + return FALSE; + } + global_event = stats_global_event->event; + event_push_global(global_event); + } + + ret = writer_client_run_event(client, parent_event_id, args+2, + &event, error_r); + if (global_event != NULL) + event_pop_global(global_event); + if (!ret) + return FALSE; + event_unref(&event); + return TRUE; +} + +static bool +writer_client_input_event_begin(struct writer_client *client, + const char *const *args, const char **error_r) +{ + struct event *event; + struct stats_event *stats_event; + uint64_t event_id, parent_event_id; + + if (args[0] == NULL || args[1] == NULL || + str_to_uint64(args[0], &event_id) < 0 || + str_to_uint64(args[1], &parent_event_id) < 0) { + *error_r = "Invalid event IDs"; + return FALSE; + } + if (writer_client_find_event(client, event_id) != NULL) { + *error_r = "Duplicate event ID"; + return FALSE; + } + if (!writer_client_run_event(client, parent_event_id, args+2, &event, error_r)) + return FALSE; + + stats_event = i_new(struct stats_event, 1); + stats_event->id = event_id; + stats_event->event = event; + DLLIST_PREPEND(&client->events, stats_event); + hash_table_insert(client->events_hash, stats_event, stats_event); + return TRUE; +} + +static bool +writer_client_input_event_update(struct writer_client *client, + const char *const *args, const char **error_r) +{ + struct stats_event *stats_event, *parent_stats_event; + struct event *parent_event; + uint64_t event_id, parent_event_id; + + if (args[0] == NULL || args[1] == NULL || + str_to_uint64(args[0], &event_id) < 0 || + str_to_uint64(args[1], &parent_event_id) < 0) { + *error_r = "Invalid event IDs"; + return FALSE; + } + stats_event = writer_client_find_event(client, event_id); + if (stats_event == NULL) { + *error_r = "Unknown event ID"; + return FALSE; + } + parent_stats_event = parent_event_id == 0 ? NULL : + writer_client_find_event(client, parent_event_id); + parent_event = parent_stats_event == NULL ? NULL : + parent_stats_event->event; + if (stats_event->event->parent != parent_event) { + *error_r = "Event unexpectedly changed parent"; + return FALSE; + } + return event_import_unescaped(stats_event->event, args+2, error_r); +} + +static bool +writer_client_input_event_end(struct writer_client *client, + const char *const *args, const char **error_r) +{ + struct stats_event *stats_event; + uint64_t event_id; + + if (args[0] == NULL || str_to_uint64(args[0], &event_id) < 0) { + *error_r = "Invalid event ID"; + return FALSE; + } + stats_event = writer_client_find_event(client, event_id); + if (stats_event == NULL) { + *error_r = "Unknown event ID"; + return FALSE; + } + + DLLIST_REMOVE(&client->events, stats_event); + hash_table_remove(client->events_hash, stats_event); + event_unref(&stats_event->event); + i_free(stats_event); + return TRUE; +} + +static bool +writer_client_input_category(struct writer_client *client ATTR_UNUSED, + const char *const *args, const char **error_r) +{ + struct event_category *category, *parent; + + if (args[0] == NULL) { + *error_r = "Missing category name"; + return FALSE; + } + if (args[1] == NULL) + parent = NULL; + else if ((parent = event_category_find_registered(args[1])) == NULL) { + *error_r = "Unknown parent category"; + return FALSE; + } + + category = event_category_find_registered(args[0]); + if (category == NULL) { + /* new category - create */ + stats_event_category_register(args[0], parent); + } else if (category->parent != parent) { + *error_r = t_strdup_printf( + "Category parent '%s' changed to '%s'", + category->parent == NULL ? "" : category->parent->name, + parent == NULL ? "" : parent->name); + return FALSE; + } else { + /* duplicate - ignore */ + return TRUE; + } + return TRUE; +} + +static int +writer_client_input_args(struct connection *conn, const char *const *args) +{ + struct writer_client *client = (struct writer_client *)conn; + const char *error, *cmd = args[0]; + bool ret; + + if (cmd == NULL) { + i_error("Client sent empty line"); + return 1; + } + if (strcmp(cmd, "EVENT") == 0) + ret = writer_client_input_event(client, args+1, &error); + else if (strcmp(cmd, "BEGIN") == 0) + ret = writer_client_input_event_begin(client, args+1, &error); + else if (strcmp(cmd, "UPDATE") == 0) + ret = writer_client_input_event_update(client, args+1, &error); + else if (strcmp(cmd, "END") == 0) + ret = writer_client_input_event_end(client, args+1, &error); + else if (strcmp(cmd, "CATEGORY") == 0) + ret = writer_client_input_category(client, args+1, &error); + else { + error = "Unknown command"; + ret = FALSE; + } + if (!ret) { + i_error("Client sent invalid input for %s: %s (input: %s)", + cmd, error, t_strarray_join(args, "\t")); + return -1; + } + return 1; +} + +static struct connection_settings client_set = { + .service_name_in = "stats-client", + .service_name_out = "stats-server", + .major_version = 4, + .minor_version = 0, + + .input_max_size = 1024*128, /* "big enough" */ + .output_max_size = SIZE_MAX, + .client = FALSE, +}; + +static const struct connection_vfuncs client_vfuncs = { + .destroy = writer_client_destroy, + .input_args = writer_client_input_args, +}; + +static void +client_writer_update_connections_internal(void *context ATTR_UNUSED) +{ + struct connection *conn; + for (conn = writer_clients->connections; conn != NULL; conn = conn->next) { + struct writer_client *client = + container_of(conn, struct writer_client, conn); + client_writer_send_handshake(client); + } + timeout_remove(&to_update_clients); +} + +void client_writer_update_connections(void) +{ + if (to_update_clients != NULL) + return; + to_update_clients = timeout_add(STATS_UPDATE_CLIENTS_DELAY_MSECS, + client_writer_update_connections_internal, + NULL); +} + +void client_writers_init(void) +{ + writer_clients = connection_list_init(&client_set, &client_vfuncs); +} + +void client_writers_deinit(void) +{ + timeout_remove(&to_update_clients); + connection_list_deinit(&writer_clients); +} diff --git a/src/stats/client-writer.h b/src/stats/client-writer.h new file mode 100644 index 0000000..a88de38 --- /dev/null +++ b/src/stats/client-writer.h @@ -0,0 +1,13 @@ +#ifndef CLIENT_WRITER_H +#define CLIENT_WRITER_H + +struct stats_metrics; + +void client_writer_create(int fd); + +void client_writer_update_connections(void); + +void client_writers_init(void); +void client_writers_deinit(void); + +#endif diff --git a/src/stats/event-exporter-fmt-json.c b/src/stats/event-exporter-fmt-json.c new file mode 100644 index 0000000..25c6116 --- /dev/null +++ b/src/stats/event-exporter-fmt-json.c @@ -0,0 +1,249 @@ +/* Copyright (c) 2019 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "ioloop.h" +#include "array.h" +#include "lib-event-private.h" +#include "event-exporter.h" +#include "str.h" +#include "json-parser.h" +#include "hostpid.h" + +static void append_str(string_t *dest, const char *str) +{ + str_append_c(dest, '"'); + json_append_escaped(dest, str); + str_append_c(dest, '"'); +} + +static void append_str_max_len(string_t *dest, const char *str, + const struct metric_export_info *info) +{ + str_append_c(dest, '"'); + if (info->exporter->format_max_field_len == 0) + json_append_escaped(dest, str); + else { + size_t len = strlen(str); + json_append_escaped_data(dest, (const unsigned char *)str, + I_MIN(len, info->exporter->format_max_field_len)); + if (len > info->exporter->format_max_field_len) + str_append(dest, "..."); + } + str_append_c(dest, '"'); +} + +static void +append_strlist(string_t *dest, const ARRAY_TYPE(const_string) *strlist, + const struct metric_export_info *info) +{ + const char *value; + bool first = TRUE; + + str_append_c(dest, '['); + array_foreach_elem(strlist, value) { + if (first) + first = FALSE; + else + str_append_c(dest, ','); + append_str_max_len(dest, value, info); + } + str_append_c(dest, ']'); +} + +static void append_int(string_t *dest, intmax_t val) +{ + str_printfa(dest, "%jd", val); +} + +static void append_time(string_t *dest, const struct timeval *time, + enum event_exporter_time_fmt fmt) +{ + switch (fmt) { + case EVENT_EXPORTER_TIME_FMT_NATIVE: + i_panic("JSON does not have a native date/time type"); + case EVENT_EXPORTER_TIME_FMT_UNIX: + event_export_helper_fmt_unix_time(dest, time); + break; + case EVENT_EXPORTER_TIME_FMT_RFC3339: + str_append_c(dest, '"'); + event_export_helper_fmt_rfc3339_time(dest, time); + str_append_c(dest, '"'); + break; + } +} + +static void append_field_value(string_t *dest, const struct event_field *field, + const struct metric_export_info *info) +{ + switch (field->value_type) { + case EVENT_FIELD_VALUE_TYPE_STR: + append_str_max_len(dest, field->value.str, info); + break; + case EVENT_FIELD_VALUE_TYPE_INTMAX: + append_int(dest, field->value.intmax); + break; + case EVENT_FIELD_VALUE_TYPE_TIMEVAL: + append_time(dest, &field->value.timeval, + info->exporter->time_format); + break; + case EVENT_FIELD_VALUE_TYPE_STRLIST: + append_strlist(dest, &field->value.strlist, info); + break; + } +} + +static void json_export_name(string_t *dest, struct event *event, + const struct metric_export_info *info) +{ + if ((info->include & EVENT_EXPORTER_INCL_NAME) == 0) + return; + + append_str(dest, "event"); + str_append_c(dest, ':'); + append_str(dest, event->sending_name); + str_append_c(dest, ','); +} + +static void json_export_hostname(string_t *dest, + const struct metric_export_info *info) +{ + if ((info->include & EVENT_EXPORTER_INCL_HOSTNAME) == 0) + return; + + append_str(dest, "hostname"); + str_append_c(dest, ':'); + append_str(dest, my_hostname); + str_append_c(dest, ','); +} + +static void json_export_timestamps(string_t *dest, struct event *event, + const struct metric_export_info *info) +{ + if ((info->include & EVENT_EXPORTER_INCL_TIMESTAMPS) == 0) + return; + + append_str(dest, "start_time"); + str_append_c(dest, ':'); + append_time(dest, &event->tv_created, info->exporter->time_format); + str_append_c(dest, ','); + + append_str(dest, "end_time"); + str_append_c(dest, ':'); + append_time(dest, &ioloop_timeval, info->exporter->time_format); + str_append_c(dest, ','); +} + +static void json_export_categories(string_t *dest, struct event *event, + const struct metric_export_info *info) +{ + struct event_category *const *cats; + unsigned int count; + + if ((info->include & EVENT_EXPORTER_INCL_CATEGORIES) == 0) + return; + + append_str(dest, "categories"); + str_append(dest, ":["); + + cats = event_get_categories(event, &count); + event_export_helper_fmt_categories(dest, cats, count, + append_str, ","); + + str_append(dest, "],"); +} + +static void json_export_fields(string_t *dest, struct event *event, + const struct metric_export_info *info, + const unsigned int fields_count, + const struct metric_field *fields) +{ + bool appended = FALSE; + + if ((info->include & EVENT_EXPORTER_INCL_FIELDS) == 0) + return; + + append_str(dest, "fields"); + str_append(dest, ":{"); + + if (fields_count == 0) { + /* include all fields */ + const struct event_field *fields; + unsigned int count; + + fields = event_get_fields(event, &count); + + for (unsigned int i = 0; i < count; i++) { + const struct event_field *field = &fields[i]; + + append_str(dest, field->key); + str_append_c(dest, ':'); + append_field_value(dest, field, info); + str_append_c(dest, ','); + + appended = TRUE; + } + } else { + for (unsigned int i = 0; i < fields_count; i++) { + const char *name = fields[i].field_key; + const struct event_field *field; + + field = event_find_field_recursive(event, name); + if (field == NULL) + continue; /* doesn't exist, skip it */ + + append_str(dest, name); + str_append_c(dest, ':'); + append_field_value(dest, field, info); + str_append_c(dest, ','); + + appended = TRUE; + } + } + + /* remove trailing comma */ + if (appended) + str_truncate(dest, str_len(dest) - 1); + + str_append(dest, "},"); +} + +/* + * Serialize the event as: + * + * { + * "name": <event name>, + * "hostname": <hostname>, + * "start_time": <event creation timestamp>, + * "end_time": <event export timestamp>, + * "categories": [ <categories>, ... ], + * "fields": { + * <name>: <value>, + * ... + * } + * } + * + */ +void event_export_fmt_json(const struct metric *metric, + struct event *event, buffer_t *dest) +{ + const struct metric_export_info *info = &metric->export_info; + + if (info->include == EVENT_EXPORTER_INCL_NONE) { + str_append(dest, "{}"); + return; + } + + str_append_c(dest, '{'); + + json_export_name(dest, event, info); + json_export_hostname(dest, info); + json_export_timestamps(dest, event, info); + json_export_categories(dest, event, info); + json_export_fields(dest, event, info, metric->fields_count, + metric->fields); + + /* remove trailing comma */ + str_truncate(dest, str_len(dest) - 1); + + str_append_c(dest, '}'); +} diff --git a/src/stats/event-exporter-fmt-none.c b/src/stats/event-exporter-fmt-none.c new file mode 100644 index 0000000..cd052c2 --- /dev/null +++ b/src/stats/event-exporter-fmt-none.c @@ -0,0 +1,12 @@ +/* Copyright (c) 2019 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "ioloop.h" +#include "event-exporter.h" + +void event_export_fmt_none(const struct metric *metric ATTR_UNUSED, + struct event *event ATTR_UNUSED, + buffer_t *dest ATTR_UNUSED) +{ + /* nothing to do */ +} diff --git a/src/stats/event-exporter-fmt-tab-text.c b/src/stats/event-exporter-fmt-tab-text.c new file mode 100644 index 0000000..49a065b --- /dev/null +++ b/src/stats/event-exporter-fmt-tab-text.c @@ -0,0 +1,212 @@ +/* Copyright (c) 2019 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "ioloop.h" +#include "array.h" +#include "lib-event-private.h" +#include "event-exporter.h" +#include "str.h" +#include "strescape.h" +#include "hostpid.h" + +static void append_strlist(string_t *dest, const ARRAY_TYPE(const_string) *strlist) +{ + string_t *str = t_str_new(64); + const char *value; + bool first = TRUE; + + /* append the strings first escaped into a temporary string */ + array_foreach_elem(strlist, value) { + if (first) + first = FALSE; + else + str_append_c(str, '\t'); + str_append_tabescaped(str, value); + } + /* append the temporary string (double-)escaped as the value */ + str_append_tabescaped(dest, str_c(str)); +} + +static void append_int(string_t *dest, intmax_t val) +{ + str_printfa(dest, "%jd", val); +} + +static void append_time(string_t *dest, const struct timeval *time, + enum event_exporter_time_fmt fmt) +{ + switch (fmt) { + case EVENT_EXPORTER_TIME_FMT_NATIVE: + i_panic("tab-text format does not have a native date/time type"); + case EVENT_EXPORTER_TIME_FMT_UNIX: + event_export_helper_fmt_unix_time(dest, time); + break; + case EVENT_EXPORTER_TIME_FMT_RFC3339: + event_export_helper_fmt_rfc3339_time(dest, time); + break; + } +} + +static void append_field_str(string_t *dest, const char *str, + const struct metric_export_info *info) +{ + if (info->exporter->format_max_field_len == 0) + str_append_tabescaped(dest, str); + else { + size_t len = strlen(str); + str_append_tabescaped_n(dest, (const unsigned char *)str, + I_MIN(len, info->exporter->format_max_field_len)); + if (len > info->exporter->format_max_field_len) + str_append(dest, "..."); + } +} + +static void append_field_value(string_t *dest, const struct event_field *field, + const struct metric_export_info *info) +{ + switch (field->value_type) { + case EVENT_FIELD_VALUE_TYPE_STR: + append_field_str(dest, field->value.str, info); + break; + case EVENT_FIELD_VALUE_TYPE_INTMAX: + append_int(dest, field->value.intmax); + break; + case EVENT_FIELD_VALUE_TYPE_TIMEVAL: + append_time(dest, &field->value.timeval, + info->exporter->time_format); + break; + case EVENT_FIELD_VALUE_TYPE_STRLIST: + append_strlist(dest, &field->value.strlist); + break; + } +} + +static void tabtext_export_name(string_t *dest, struct event *event, + const struct metric_export_info *info) +{ + if ((info->include & EVENT_EXPORTER_INCL_NAME) == 0) + return; + + str_append(dest, "event:"); + str_append_tabescaped(dest, event->sending_name); + str_append_c(dest, '\t'); +} + +static void tabtext_export_hostname(string_t *dest, + const struct metric_export_info *info) +{ + if ((info->include & EVENT_EXPORTER_INCL_HOSTNAME) == 0) + return; + + str_append(dest, "hostname:"); + str_append_tabescaped(dest, my_hostname); + str_append_c(dest, '\t'); +} + +static void tabtext_export_timestamps(string_t *dest, struct event *event, + const struct metric_export_info *info) +{ + if ((info->include & EVENT_EXPORTER_INCL_TIMESTAMPS) == 0) + return; + + str_append(dest, "start_time:"); + append_time(dest, &event->tv_created, info->exporter->time_format); + str_append(dest, "\tend_time:"); + append_time(dest, &ioloop_timeval, info->exporter->time_format); + str_append_c(dest, '\t'); +} + +static void append_category(string_t *dest, const char *cat) +{ + str_append(dest, "category:"); + str_append_tabescaped(dest, cat); +} + +static void tabtext_export_categories(string_t *dest, struct event *event, + const struct metric_export_info *info) +{ + struct event_category *const *cats; + unsigned int count; + + if ((info->include & EVENT_EXPORTER_INCL_CATEGORIES) == 0) + return; + + cats = event_get_categories(event, &count); + event_export_helper_fmt_categories(dest, cats, count, + append_category, "\t"); + + str_append_c(dest, '\t'); /* extra \t to have something to remove later */ +} + +static void tabtext_export_fields(string_t *dest, struct event *event, + const struct metric_export_info *info, + const unsigned int fields_count, + const struct metric_field *fields) +{ + if ((info->include & EVENT_EXPORTER_INCL_FIELDS) == 0) + return; + + if (fields_count == 0) { + /* include all fields */ + const struct event_field *fields; + unsigned int count; + + fields = event_get_fields(event, &count); + + for (unsigned int i = 0; i < count; i++) { + const struct event_field *field = &fields[i]; + + str_append(dest, "field:"); + str_append_tabescaped(dest, field->key); + str_append_c(dest, '='); + append_field_value(dest, field, info); + str_append_c(dest, '\t'); + } + } else { + for (unsigned int i = 0; i < fields_count; i++) { + const char *name = fields[i].field_key; + const struct event_field *field; + + field = event_find_field_recursive(event, name); + if (field == NULL) + continue; /* doesn't exist, skip it */ + + str_append(dest, "field:"); + str_append_tabescaped(dest, name); + str_append_c(dest, '='); + append_field_value(dest, field, info); + str_append_c(dest, '\t'); + } + } +} + +/* + * Serialize the event as tab delimited collection of the following: + * + * event:<event name> + * hostname:<tab escaped hostname> + * start_time:<event creation timestamp> + * end_time:<event export timestamp> + * category:<category> + * field:<name>=<tab escaped value> + * + * Note: cat and field can occur multiple times. + */ +void event_export_fmt_tabescaped_text(const struct metric *metric, + struct event *event, buffer_t *dest) +{ + const struct metric_export_info *info = &metric->export_info; + + if (info->include == EVENT_EXPORTER_INCL_NONE) + return; + + tabtext_export_name(dest, event, info); + tabtext_export_hostname(dest, info); + tabtext_export_timestamps(dest, event, info); + tabtext_export_categories(dest, event, info); + tabtext_export_fields(dest, event, info, metric->fields_count, + metric->fields); + + /* remove trailing tab */ + str_truncate(dest, str_len(dest) - 1); +} diff --git a/src/stats/event-exporter-fmt.c b/src/stats/event-exporter-fmt.c new file mode 100644 index 0000000..a4ec5a8 --- /dev/null +++ b/src/stats/event-exporter-fmt.c @@ -0,0 +1,78 @@ +/* Copyright (c) 2019 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "str.h" +#include "hash.h" +#include "ioloop.h" +#include "event-exporter.h" + +void event_export_helper_fmt_unix_time(string_t *dest, + const struct timeval *time) +{ + str_printfa(dest, "%"PRIdTIME_T".%06u", time->tv_sec, + (unsigned int) time->tv_usec); +} + +void event_export_helper_fmt_rfc3339_time(string_t *dest, + const struct timeval *time) +{ + const struct tm *tm; + + tm = gmtime(&time->tv_sec); + + str_printfa(dest, "%04d-%02d-%02dT%02d:%02d:%02d.%06luZ", + tm->tm_year + 1900, tm->tm_mon + 1, tm->tm_mday, + tm->tm_hour, tm->tm_min, tm->tm_sec, + time->tv_usec); +} + +HASH_TABLE_DEFINE_TYPE(category_set, void *, const struct event_category *); + +static void insert_category(HASH_TABLE_TYPE(category_set) hash, + const struct event_category * const cat) +{ + /* insert this category (key == the unique internal pointer) */ + hash_table_update(hash, cat->internal, cat); + + /* insert parent's categories */ + if (cat->parent != NULL) + insert_category(hash, cat->parent); +} + +void event_export_helper_fmt_categories(string_t *dest, + struct event_category * const *cats, + unsigned int count, + void (*append)(string_t *, const char *), + const char *separator) +{ + HASH_TABLE_TYPE(category_set) hash; + struct hash_iterate_context *iter; + const struct event_category *cat; + void *key ATTR_UNUSED; + unsigned int i; + bool first = TRUE; + + if (count == 0) + return; + + hash_table_create_direct(&hash, pool_datastack_create(), + 3 * count /* estimate */); + + /* insert all the categories into the hash table */ + for (i = 0; i < count; i++) + insert_category(hash, cats[i]); + + /* output each category from hash table */ + iter = hash_table_iterate_init(hash); + while (hash_table_iterate(iter, hash, &key, &cat)) { + if (!first) + str_append(dest, separator); + + append(dest, cat->name); + + first = FALSE; + } + hash_table_iterate_deinit(&iter); + + hash_table_destroy(&hash); +} diff --git a/src/stats/event-exporter-transport-drop.c b/src/stats/event-exporter-transport-drop.c new file mode 100644 index 0000000..943f305 --- /dev/null +++ b/src/stats/event-exporter-transport-drop.c @@ -0,0 +1,9 @@ +/* Copyright (c) 2019 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "event-exporter.h" + +void event_export_transport_drop(const struct exporter *exporter ATTR_UNUSED, + const buffer_t *buf ATTR_UNUSED) +{ +} diff --git a/src/stats/event-exporter-transport-http-post.c b/src/stats/event-exporter-transport-http-post.c new file mode 100644 index 0000000..6d328cd --- /dev/null +++ b/src/stats/event-exporter-transport-http-post.c @@ -0,0 +1,76 @@ +/* Copyright (c) 2019 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "ioloop.h" +#include "str.h" +#include "event-exporter.h" +#include "http-client.h" +#include "iostream-ssl.h" +#include "master-service.h" +#include "master-service-ssl-settings.h" + +/* the http client used to export all events with exporter=http-post */ +static struct http_client *exporter_http_client; + +void event_export_transport_http_post_deinit(void) +{ + if (exporter_http_client != NULL) + http_client_deinit(&exporter_http_client); +} + +static void response_fxn(const struct http_response *response, + void *context ATTR_UNUSED) +{ + static time_t last_log; + static unsigned suppressed; + + if (http_response_is_success(response)) + return; + + if (last_log == ioloop_time) { + suppressed++; + return; /* don't spam the log */ + } + + if (suppressed == 0) + i_error("Failed to export event via HTTP POST: %d %s", + response->status, response->reason); + else + i_error("Failed to export event via HTTP POST: %d %s (%u more errors suppressed)", + response->status, response->reason, suppressed); + + last_log = ioloop_time; + suppressed = 0; +} + +void event_export_transport_http_post(const struct exporter *exporter, + const buffer_t *buf) +{ + struct http_client_request *req; + + if (exporter_http_client == NULL) { + const struct master_service_ssl_settings *master_ssl_set = + master_service_ssl_settings_get(master_service); + struct ssl_iostream_settings ssl_set; + + struct http_client_settings set = { + .dns_client_socket_path = "dns-client", + }; + if (master_ssl_set != NULL) { + master_service_ssl_client_settings_to_iostream_set( + master_ssl_set, pool_datastack_create(), + &ssl_set); + set.ssl = &ssl_set; + } + exporter_http_client = http_client_init(&set); + } + + req = http_client_request_url_str(exporter_http_client, "POST", + exporter->transport_args, + response_fxn, NULL); + http_client_request_add_header(req, "Content-Type", exporter->format_mime_type); + http_client_request_set_payload_data(req, buf->data, buf->used); + + http_client_request_set_timeout_msecs(req, exporter->transport_timeout); + http_client_request_submit(req); +} diff --git a/src/stats/event-exporter-transport-log.c b/src/stats/event-exporter-transport-log.c new file mode 100644 index 0000000..a0cf67f --- /dev/null +++ b/src/stats/event-exporter-transport-log.c @@ -0,0 +1,12 @@ +/* Copyright (c) 2019 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "ioloop.h" +#include "str.h" +#include "event-exporter.h" + +void event_export_transport_log(const struct exporter *exporter ATTR_UNUSED, + const buffer_t *buf) +{ + i_info("%.*s", (int)buf->used, (const char *)buf->data); +} diff --git a/src/stats/event-exporter.h b/src/stats/event-exporter.h new file mode 100644 index 0000000..64bb708 --- /dev/null +++ b/src/stats/event-exporter.h @@ -0,0 +1,31 @@ +#ifndef EVENT_EXPORTER_H +#define EVENT_EXPORTER_H + +#include "stats-metrics.h" + +/* fmt functions */ +void event_export_fmt_json(const struct metric *metric, struct event *event, buffer_t *dest); +void event_export_fmt_none(const struct metric *metric, struct event *event, buffer_t *dest); +void event_export_fmt_tabescaped_text(const struct metric *metric, struct event *event, buffer_t *dest); + +/* transport functions */ +void event_export_transport_drop(const struct exporter *exporter, const buffer_t *buf); +void event_export_transport_http_post(const struct exporter *exporter, const buffer_t *buf); +void event_export_transport_http_post_deinit(void); +void event_export_transport_log(const struct exporter *exporter, const buffer_t *buf); + +/* append a microsecond resolution RFC3339 UTC timestamp */ +void event_export_helper_fmt_rfc3339_time(string_t *dest, const struct timeval *time); +/* append a microsecond resolution unix timestamp in seconds (i.e., %u.%06u) */ +void event_export_helper_fmt_unix_time(string_t *dest, const struct timeval *time); +/* append category names using 'append' function pointer, separated by 'separator' arg + + The result has no duplicates regardless of if the array has any or if any + of the categories' ancestors are implictly or explicitly duplicated. */ +void event_export_helper_fmt_categories(string_t *dest, + struct event_category *const *cats, + unsigned int count, + void (*append)(string_t *, const char *), + const char *separator); + +#endif diff --git a/src/stats/main.c b/src/stats/main.c new file mode 100644 index 0000000..15e0733 --- /dev/null +++ b/src/stats/main.c @@ -0,0 +1,117 @@ +/* Copyright (c) 2017-2018 Dovecot authors, see the included COPYING file */ + +#include "stats-common.h" +#include "restrict-access.h" +#include "ioloop.h" +#include "master-service.h" +#include "master-service-settings.h" +#include "stats-settings.h" +#include "stats-event-category.h" +#include "stats-metrics.h" +#include "stats-service.h" +#include "client-writer.h" +#include "client-reader.h" +#include "client-http.h" + +struct stats_metrics *stats_metrics; +time_t stats_startup_time; + +static const struct stats_settings *stats_settings; + +static bool client_is_writer(const char *path) +{ + const char *name, *suffix; + + name = strrchr(path, '/'); + if (name == NULL) + name = path; + else + name++; + + suffix = strrchr(name, '-'); + if (suffix == NULL) + suffix = name; + else + suffix++; + + return strcmp(suffix, "writer") == 0; +} + +static void client_connected(struct master_service_connection *conn) +{ + if (strcmp(conn->name, "http") == 0) + client_http_create(conn); + else if (client_is_writer(conn->name)) + client_writer_create(conn->fd); + else + client_reader_create(conn->fd); + master_service_client_connection_accept(conn); +} + +static void stats_die(void) +{ + /* just wait for existing stats clients to disconnect from us */ +} + +static void main_preinit(void) +{ + restrict_access_by_env(RESTRICT_ACCESS_FLAG_ALLOW_ROOT, NULL); + restrict_access_allow_coredumps(TRUE); +} + +static void main_init(void) +{ + void **sets = master_service_settings_get_others(master_service); + stats_settings = sets[0]; + + stats_startup_time = ioloop_time; + stats_metrics = stats_metrics_init(stats_settings); + stats_event_categories_init(); + client_readers_init(); + client_writers_init(); + client_http_init(stats_settings); + stats_services_init(); +} + +static void main_deinit(void) +{ + stats_services_deinit(); + client_readers_deinit(); + client_writers_deinit(); + client_http_deinit(); + stats_event_categories_deinit(); + stats_metrics_deinit(&stats_metrics); +} + +int main(int argc, char *argv[]) +{ + const struct setting_parser_info *set_roots[] = { + &stats_setting_parser_info, + NULL + }; + const enum master_service_flags service_flags = + MASTER_SERVICE_FLAG_NO_SSL_INIT | + MASTER_SERVICE_FLAG_DONT_SEND_STATS | + MASTER_SERVICE_FLAG_NO_IDLE_DIE | + MASTER_SERVICE_FLAG_UPDATE_PROCTITLE; + const char *error; + + master_service = master_service_init("stats", service_flags, + &argc, &argv, ""); + if (master_getopt(master_service) > 0) + return FATAL_DEFAULT; + if (master_service_settings_read_simple(master_service, set_roots, + &error) < 0) + i_fatal("Error reading configuration: %s", error); + master_service_init_log(master_service); + master_service_set_die_callback(master_service, stats_die); + + main_preinit(); + + main_init(); + master_service_init_finish(master_service); + master_service_run(master_service, client_connected); + main_deinit(); + master_service_deinit(&master_service); + return 0; +} diff --git a/src/stats/stats-common.h b/src/stats/stats-common.h new file mode 100644 index 0000000..09af424 --- /dev/null +++ b/src/stats/stats-common.h @@ -0,0 +1,10 @@ +#ifndef STATS_COMMON_H +#define STATS_COMMON_H + +#include "lib.h" +#include "stats-settings.h" + +extern struct stats_metrics *stats_metrics; +extern time_t stats_startup_time; + +#endif diff --git a/src/stats/stats-event-category.c b/src/stats/stats-event-category.c new file mode 100644 index 0000000..cb01e8b --- /dev/null +++ b/src/stats/stats-event-category.c @@ -0,0 +1,32 @@ +/* Copyright (c) 2017-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "stats-event-category.h" + +static pool_t categories_pool; + +void stats_event_category_register(const char *name, + struct event_category *parent) +{ + struct event_category *category = + p_new(categories_pool, struct event_category, 1); + category->parent = parent; + category->name = p_strdup(categories_pool, name); + + /* Create a temporary event to register the category. A bit slower + than necessary, but this code won't be called often. */ + struct event *event = event_create(NULL); + struct event_category *categories[] = { category, NULL }; + event_add_categories(event, categories); + event_unref(&event); +} + +void stats_event_categories_init(void) +{ + categories_pool = pool_alloconly_create("categories", 1024); +} + +void stats_event_categories_deinit(void) +{ + pool_unref(&categories_pool); +} diff --git a/src/stats/stats-event-category.h b/src/stats/stats-event-category.h new file mode 100644 index 0000000..b56fc91 --- /dev/null +++ b/src/stats/stats-event-category.h @@ -0,0 +1,12 @@ +#ifndef STATS_EVENT_CATEGORY_H +#define STATS_EVENT_CATEGORY_H + +/* Register a new event category if it doesn't already exist. + parent may be NULL. */ +void stats_event_category_register(const char *name, + struct event_category *parent); + +void stats_event_categories_init(void); +void stats_event_categories_deinit(void); + +#endif diff --git a/src/stats/stats-metrics.c b/src/stats/stats-metrics.c new file mode 100644 index 0000000..632c486 --- /dev/null +++ b/src/stats/stats-metrics.c @@ -0,0 +1,735 @@ +/* Copyright (c) 2017-2018 Dovecot authors, see the included COPYING file */ + +#include "stats-common.h" +#include "array.h" +#include "str.h" +#include "str-sanitize.h" +#include "stats-dist.h" +#include "time-util.h" +#include "event-filter.h" +#include "event-exporter.h" +#include "stats-settings.h" +#include "stats-metrics.h" +#include "settings-parser.h" + +#include <ctype.h> + +#define LOG_EXPORTER_LONG_FIELD_TRUNCATE_LEN 1000 + +struct stats_metrics { + pool_t pool; + struct event_filter *filter; /* stats & export */ + ARRAY(struct exporter *) exporters; + ARRAY(struct metric *) metrics; +}; + +static void +stats_metric_event(struct metric *metric, struct event *event, pool_t pool); +static struct metric * +stats_metric_sub_metric_alloc(struct metric *metric, const char *name, pool_t pool); +static void stats_metric_free(struct metric *metric); + +static void stats_exporters_add_set(struct stats_metrics *metrics, + const struct stats_exporter_settings *set) +{ + struct exporter *exporter; + + exporter = p_new(metrics->pool, struct exporter, 1); + exporter->name = p_strdup(metrics->pool, set->name); + exporter->transport_args = p_strdup(metrics->pool, set->transport_args); + exporter->transport_timeout = set->transport_timeout; + exporter->time_format = set->parsed_time_format; + + /* TODO: The following should be plugable. + * + * Note: Make sure to mirror any changes to the below code in + * stats_exporter_settings_check(). + */ + if (strcmp(set->format, "none") == 0) { + exporter->format = event_export_fmt_none; + exporter->format_mime_type = "application/octet-stream"; + } else if (strcmp(set->format, "json") == 0) { + exporter->format = event_export_fmt_json; + exporter->format_mime_type = "application/json"; + } else if (strcmp(set->format, "tab-text") == 0) { + exporter->format = event_export_fmt_tabescaped_text; + exporter->format_mime_type = "text/plain"; + } else { + i_unreached(); + } + + /* TODO: The following should be plugable. + * + * Note: Make sure to mirror any changes to the below code in + * stats_exporter_settings_check(). + */ + if (strcmp(set->transport, "drop") == 0) { + exporter->transport = event_export_transport_drop; + } else if (strcmp(set->transport, "http-post") == 0) { + exporter->transport = event_export_transport_http_post; + } else if (strcmp(set->transport, "log") == 0) { + exporter->transport = event_export_transport_log; + exporter->format_max_field_len = + LOG_EXPORTER_LONG_FIELD_TRUNCATE_LEN; + } else { + i_unreached(); + } + + exporter->transport_args = set->transport_args; + + array_push_back(&metrics->exporters, &exporter); +} + +static struct metric * +stats_metric_alloc(pool_t pool, const char *name, + const struct stats_metric_settings *set, + const char *const *fields) +{ + struct metric *metric = p_new(pool, struct metric, 1); + metric->name = p_strdup(pool, name); + metric->set = set; + metric->duration_stats = stats_dist_init(); + metric->fields_count = str_array_length(fields); + if (metric->fields_count > 0) { + metric->fields = p_new(pool, struct metric_field, + metric->fields_count); + for (unsigned int i = 0; i < metric->fields_count; i++) { + metric->fields[i].field_key = p_strdup(pool, fields[i]); + metric->fields[i].stats = stats_dist_init(); + } + } + return metric; +} + +static void stats_metrics_add_set(struct stats_metrics *metrics, + const struct stats_metric_settings *set) +{ + struct exporter *exporter; + struct metric *metric; + const char *const *fields; + const char *const *tmp; + + fields = t_strsplit_spaces(set->fields, " "); + metric = stats_metric_alloc(metrics->pool, set->metric_name, set, fields); + + if (array_is_created(&set->parsed_group_by)) + metric->group_by = array_get(&set->parsed_group_by, + &metric->group_by_count); + + array_push_back(&metrics->metrics, &metric); + + event_filter_merge_with_context(metrics->filter, set->parsed_filter, metric); + + /* + * Metrics may also be exported - make sure exporter info is set + */ + + if (set->exporter[0] == '\0') + return; /* not exported */ + + array_foreach_elem(&metrics->exporters, exporter) { + if (strcmp(set->exporter, exporter->name) == 0) { + metric->export_info.exporter = exporter; + break; + } + } + + if (metric->export_info.exporter == NULL) + i_panic("Could not find exporter (%s) for metric (%s)", + set->exporter, set->metric_name); + + /* Defaults */ + metric->export_info.include = EVENT_EXPORTER_INCL_NONE; + + tmp = t_strsplit_spaces(set->exporter_include, " "); + for (; *tmp != NULL; tmp++) { + if (strcmp(*tmp, "name") == 0) + metric->export_info.include |= EVENT_EXPORTER_INCL_NAME; + else if (strcmp(*tmp, "hostname") == 0) + metric->export_info.include |= EVENT_EXPORTER_INCL_HOSTNAME; + else if (strcmp(*tmp, "timestamps") == 0) + metric->export_info.include |= EVENT_EXPORTER_INCL_TIMESTAMPS; + else if (strcmp(*tmp, "categories") == 0) + metric->export_info.include |= EVENT_EXPORTER_INCL_CATEGORIES; + else if (strcmp(*tmp, "fields") == 0) + metric->export_info.include |= EVENT_EXPORTER_INCL_FIELDS; + else + i_warning("Ignoring unknown exporter include '%s'", *tmp); + } +} + +static struct stats_metric_settings * +stats_metric_settings_dup(pool_t pool, const struct stats_metric_settings *src) +{ + struct stats_metric_settings *set = p_new(pool, struct stats_metric_settings, 1); + + set->metric_name = p_strdup(pool, src->metric_name); + set->description = p_strdup(pool, src->description); + set->fields = p_strdup(pool, src->fields); + set->group_by = p_strdup(pool, src->group_by); + set->filter = p_strdup(pool, src->filter); + set->exporter = p_strdup(pool, src->exporter); + set->exporter_include = p_strdup(pool, src->exporter_include); + + return set; +} + +static struct metric * +stats_metrics_find(struct stats_metrics *metrics, + const char *name, unsigned int *idx_r) +{ + struct metric *const *m; + array_foreach(&metrics->metrics, m) { + if (strcmp((*m)->name, name) == 0) { + *idx_r = array_foreach_idx(&metrics->metrics, m); + return *m; + } + } + return NULL; +} + +bool stats_metrics_add_dynamic(struct stats_metrics *metrics, + struct stats_metric_settings *set, + const char **error_r) +{ + unsigned int existing_idx ATTR_UNUSED; + if (stats_metrics_find(metrics, set->metric_name, &existing_idx) != NULL) { + *error_r = "Metric already exists"; + return FALSE; + } + + struct stats_metric_settings *_set = + stats_metric_settings_dup(metrics->pool, set); + if (!stats_metric_setting_parser_info.check_func(_set, metrics->pool, error_r)) + return FALSE; + stats_metrics_add_set(metrics, _set); + return TRUE; +} + +bool stats_metrics_remove_dynamic(struct stats_metrics *metrics, + const char *name) +{ + unsigned int m_idx; + bool ret = FALSE; + struct metric *m = stats_metrics_find(metrics, name, &m_idx); + if (m != NULL) { + array_delete(&metrics->metrics, m_idx, 1); + ret = event_filter_remove_queries_with_context(metrics->filter, m); + stats_metric_free(m); + } + return ret; +} + +static void +stats_metrics_add_from_settings(struct stats_metrics *metrics, + const struct stats_settings *set) +{ + /* add all the exporters first */ + if (!array_is_created(&set->exporters)) { + p_array_init(&metrics->exporters, metrics->pool, 0); + } else { + struct stats_exporter_settings *exporter_set; + + p_array_init(&metrics->exporters, metrics->pool, + array_count(&set->exporters)); + array_foreach_elem(&set->exporters, exporter_set) + stats_exporters_add_set(metrics, exporter_set); + } + + /* then add all the metrics */ + if (!array_is_created(&set->metrics)) { + p_array_init(&metrics->metrics, metrics->pool, 0); + } else { + struct stats_metric_settings *metric_set; + + p_array_init(&metrics->metrics, metrics->pool, + array_count(&set->metrics)); + array_foreach_elem(&set->metrics, metric_set) T_BEGIN { + stats_metrics_add_set(metrics, metric_set); + } T_END; + } +} + +struct stats_metrics *stats_metrics_init(const struct stats_settings *set) +{ + struct stats_metrics *metrics; + pool_t pool = pool_alloconly_create("stats metrics", 1024); + + metrics = p_new(pool, struct stats_metrics, 1); + metrics->pool = pool; + metrics->filter = event_filter_create(); + stats_metrics_add_from_settings(metrics, set); + return metrics; +} + +static void stats_metric_free(struct metric *metric) +{ + struct metric *sub_metric; + stats_dist_deinit(&metric->duration_stats); + for (unsigned int i = 0; i < metric->fields_count; i++) + stats_dist_deinit(&metric->fields[i].stats); + if (!array_is_created(&metric->sub_metrics)) + return; + array_foreach_elem(&metric->sub_metrics, sub_metric) + stats_metric_free(sub_metric); +} + +static void stats_export_deinit(void) +{ + /* no need for event_export_transport_drop_deinit() - no-op */ + event_export_transport_http_post_deinit(); + /* no need for event_export_transport_log_deinit() - no-op */ +} + +void stats_metrics_deinit(struct stats_metrics **_metrics) +{ + struct stats_metrics *metrics = *_metrics; + struct metric *metric; + + *_metrics = NULL; + + stats_export_deinit(); + + array_foreach_elem(&metrics->metrics, metric) + stats_metric_free(metric); + event_filter_unref(&metrics->filter); + pool_unref(&metrics->pool); +} + +static void stats_metric_reset(struct metric *metric) +{ + struct metric *sub_metric; + stats_dist_reset(metric->duration_stats); + for (unsigned int i = 0; i < metric->fields_count; i++) + stats_dist_reset(metric->fields[i].stats); + if (!array_is_created(&metric->sub_metrics)) + return; + array_foreach_elem(&metric->sub_metrics, sub_metric) + stats_metric_reset(sub_metric); +} + +void stats_metrics_reset(struct stats_metrics *metrics) +{ + struct metric *metric; + + array_foreach_elem(&metrics->metrics, metric) + stats_metric_reset(metric); +} + +struct event_filter * +stats_metrics_get_event_filter(struct stats_metrics *metrics) +{ + return metrics->filter; +} + +static struct metric * +stats_metric_find_sub_metric(struct metric *metric, + const struct metric_value *value) +{ + struct metric *sub_metrics; + + /* lookup sub-metric */ + array_foreach_elem(&metric->sub_metrics, sub_metrics) { + switch (sub_metrics->group_value.type) { + case METRIC_VALUE_TYPE_STR: + if (memcmp(sub_metrics->group_value.hash, value->hash, + SHA1_RESULTLEN) == 0) + return sub_metrics; + break; + case METRIC_VALUE_TYPE_INT: + if (sub_metrics->group_value.intmax == value->intmax) + return sub_metrics; + break; + case METRIC_VALUE_TYPE_BUCKET_INDEX: + if (sub_metrics->group_value.intmax == value->intmax) + return sub_metrics; + break; + } + } + return NULL; +} + +static struct metric * +stats_metric_sub_metric_alloc(struct metric *metric, const char *name, pool_t pool) +{ + struct metric *sub_metric; + ARRAY_TYPE(const_string) fields; + t_array_init(&fields, metric->fields_count); + for (unsigned int i = 0; i < metric->fields_count; i++) + array_append(&fields, &metric->fields[i].field_key, 1); + array_append_zero(&fields); + sub_metric = stats_metric_alloc(pool, metric->name, metric->set, + array_idx(&fields, 0)); + sub_metric->sub_name = p_strdup(pool, str_sanitize_utf8(name, 32)); + array_append(&metric->sub_metrics, &sub_metric, 1); + return sub_metric; +} + +static bool +stats_metric_group_by_discrete(const struct event_field *field, + struct metric_value *value_r) +{ + switch (field->value_type) { + case EVENT_FIELD_VALUE_TYPE_STR: + value_r->type = METRIC_VALUE_TYPE_STR; + /* use sha1 of value to avoid excessive memory usage in case the + actual value is quite long */ + sha1_get_digest(field->value.str, strlen(field->value.str), + value_r->hash); + return TRUE; + case EVENT_FIELD_VALUE_TYPE_INTMAX: + value_r->type = METRIC_VALUE_TYPE_INT; + value_r->intmax = field->value.intmax; + return TRUE; + case EVENT_FIELD_VALUE_TYPE_TIMEVAL: + return FALSE; + case EVENT_FIELD_VALUE_TYPE_STRLIST: + return FALSE; + } + + i_unreached(); +} + +/* convert the value to a bucket index */ +static bool +stats_metric_group_by_quantized(const struct event_field *field, + struct metric_value *value_r, + const struct stats_metric_settings_group_by *group_by) +{ + switch (field->value_type) { + case EVENT_FIELD_VALUE_TYPE_STR: + case EVENT_FIELD_VALUE_TYPE_TIMEVAL: + case EVENT_FIELD_VALUE_TYPE_STRLIST: + return FALSE; + case EVENT_FIELD_VALUE_TYPE_INTMAX: + break; + } + + value_r->type = METRIC_VALUE_TYPE_BUCKET_INDEX; + + for (unsigned int i = 0; i < group_by->num_ranges; i++) { + if ((field->value.intmax <= group_by->ranges[i].min) || + (field->value.intmax > group_by->ranges[i].max)) + continue; + + value_r->intmax = i; + return TRUE; + } + + i_panic("failed to find a matching bucket for '%s'=%jd", + group_by->field, field->value.intmax); +} + +/* convert value to a bucket label */ +static const char * +stats_metric_group_by_quantized_label(const struct event_field *field, + const struct stats_metric_settings_group_by *group_by, + const size_t bucket_index) +{ + const struct stats_metric_settings_bucket_range *range = &group_by->ranges[bucket_index]; + const char *name = group_by->field; + const char *label; + + switch (field->value_type) { + case EVENT_FIELD_VALUE_TYPE_STR: + case EVENT_FIELD_VALUE_TYPE_TIMEVAL: + case EVENT_FIELD_VALUE_TYPE_STRLIST: + i_unreached(); + case EVENT_FIELD_VALUE_TYPE_INTMAX: + break; + } + + if (range->min == INTMAX_MIN) + label = t_strdup_printf("%s_ninf_%jd", name, range->max); + else if (range->max == INTMAX_MAX) + label = t_strdup_printf("%s_%jd_inf", name, range->min + 1); + else + label = t_strdup_printf("%s_%jd_%jd", name, + range->min + 1, range->max); + + return label; +} + +static bool +stats_metric_group_by_get_value(const struct event_field *field, + const struct stats_metric_settings_group_by *group_by, + struct metric_value *value_r) +{ + switch (group_by->func) { + case STATS_METRIC_GROUPBY_DISCRETE: + if (!stats_metric_group_by_discrete(field, value_r)) + return FALSE; + return TRUE; + case STATS_METRIC_GROUPBY_QUANTIZED: + if (!stats_metric_group_by_quantized(field, value_r, group_by)) + return FALSE; + return TRUE; + } + + i_panic("unknown group-by function %d", group_by->func); +} + +static const char * +stats_metric_group_by_get_label(const struct event_field *field, + const struct stats_metric_settings_group_by *group_by, + const struct metric_value *value) +{ + switch (group_by->func) { + case STATS_METRIC_GROUPBY_DISCRETE: + i_unreached(); + case STATS_METRIC_GROUPBY_QUANTIZED: + return stats_metric_group_by_quantized_label(field, group_by, + value->intmax); + } + + i_panic("unknown group-by function %d", group_by->func); +} + +static const char * +stats_metric_group_by_value_label(const struct event_field *field, + const struct stats_metric_settings_group_by *group_by, + const struct metric_value *value) +{ + switch (value->type) { + case METRIC_VALUE_TYPE_STR: + return field->value.str; + case METRIC_VALUE_TYPE_INT: + return dec2str(field->value.intmax); + case METRIC_VALUE_TYPE_BUCKET_INDEX: + return stats_metric_group_by_get_label(field, group_by, value); + } + i_unreached(); +} + +static struct metric * +stats_metric_get_sub_metric(struct metric *metric, + const struct event_field *field, + const struct metric_value *value, + pool_t pool) +{ + struct metric *sub_metric; + + sub_metric = stats_metric_find_sub_metric(metric, value); + if (sub_metric != NULL) + return sub_metric; + + T_BEGIN { + const char *value_label = + stats_metric_group_by_value_label(field, + &metric->group_by[0], value); + sub_metric = stats_metric_sub_metric_alloc(metric, value_label, + pool); + } T_END; + if (metric->group_by_count > 1) { + sub_metric->group_by_count = metric->group_by_count - 1; + sub_metric->group_by = &metric->group_by[1]; + } + sub_metric->group_value.type = value->type; + sub_metric->group_value.intmax = value->intmax; + memcpy(sub_metric->group_value.hash, value->hash, SHA1_RESULTLEN); + return sub_metric; +} + +static void +stats_metric_group_by_field(struct metric *metric, struct event *event, + const struct event_field *field, pool_t pool) +{ + struct metric *sub_metric; + struct metric_value value; + + if (!stats_metric_group_by_get_value(field, &metric->group_by[0], &value)) + return; + + if (!array_is_created(&metric->sub_metrics)) + p_array_init(&metric->sub_metrics, pool, 8); + sub_metric = stats_metric_get_sub_metric(metric, field, &value, pool); + + /* sub-metrics are recursive, so each sub-metric can have additional + sub-metrics. */ + stats_metric_event(sub_metric, event, pool); +} + +static void +stats_event_get_strlist(struct event *event, const char *name, + ARRAY_TYPE(const_string) *strings) +{ + if (event == NULL) + return; + + const struct event_field *field = + event_find_field_nonrecursive(event, name); + if (field != NULL) { + const char *str; + array_foreach_elem(&field->value.strlist, str) + array_push_back(strings, &str); + } + stats_event_get_strlist(event_get_parent(event), name, strings); +} + +static void +stats_metric_group_by(struct metric *metric, struct event *event, pool_t pool) +{ + const struct event_field *field = + event_find_field_recursive(event, metric->group_by[0].field); + + /* ignore missing field */ + if (field == NULL) + return; + + if (field->value_type != EVENT_FIELD_VALUE_TYPE_STRLIST) + stats_metric_group_by_field(metric, event, field, pool); + else { + /* Handle each string in strlist separately. The strlist needs + to be combined from the event and its parents, as well as + the global event and its parents. */ + ARRAY_TYPE(const_string) strings; + + t_array_init(&strings, 8); + stats_event_get_strlist(event, metric->group_by[0].field, + &strings); + stats_event_get_strlist(event_get_global(), + metric->group_by[0].field, &strings); + + struct event_field str_field = { + .value_type = EVENT_FIELD_VALUE_TYPE_STR, + }; + const char *str; + + /* sort strings so duplicates can be easily skipped */ + array_sort(&strings, i_strcmp_p); + array_foreach_elem(&strings, str) { + if (str_field.value.str == NULL || + strcmp(str_field.value.str, str) != 0) { + str_field.value.str = str; + stats_metric_group_by_field(metric, event, + &str_field, pool); + } + } + } +} + +static void +stats_metric_event_field(struct event *event, const char *fieldname, + struct stats_dist *stats) +{ + const struct event_field *field = + event_find_field_recursive(event, fieldname); + intmax_t num = 0; + + if (field == NULL) + return; + + switch (field->value_type) { + case EVENT_FIELD_VALUE_TYPE_STR: + case EVENT_FIELD_VALUE_TYPE_STRLIST: + break; + case EVENT_FIELD_VALUE_TYPE_INTMAX: + num = field->value.intmax; + break; + case EVENT_FIELD_VALUE_TYPE_TIMEVAL: + num = field->value.timeval.tv_sec * 1000000ULL + + field->value.timeval.tv_usec; + break; + } + + stats_dist_add(stats, num); +} + +static void +stats_metric_event(struct metric *metric, struct event *event, pool_t pool) +{ + /* duration is special - we always add it */ + stats_metric_event_field(event, STATS_EVENT_FIELD_NAME_DURATION, + metric->duration_stats); + + for (unsigned int i = 0; i < metric->fields_count; i++) + stats_metric_event_field(event, + metric->fields[i].field_key, + metric->fields[i].stats); + + if (metric->group_by != NULL) + stats_metric_group_by(metric, event, pool); +} + +static void +stats_export_event(struct metric *metric, struct event *oldevent) +{ + const struct metric_export_info *info = &metric->export_info; + const struct exporter *exporter = info->exporter; + struct event *event; + + i_assert(exporter != NULL); + + event = event_flatten(oldevent); + + T_BEGIN { + buffer_t *buf; + + buf = t_buffer_create(128); + + exporter->format(metric, event, buf); + exporter->transport(exporter, buf); + } T_END; + + event_unref(&event); +} + +void stats_metrics_event(struct stats_metrics *metrics, struct event *event, + const struct failure_context *ctx) +{ + struct event_filter_match_iter *iter; + struct metric *metric; + uintmax_t duration; + + /* Note: Adding the field here means that it will get exported + below. This is necessary to allow group-by functions to quantize + based on the event duration. */ + event_get_last_duration(event, &duration); + event_add_int(event, STATS_EVENT_FIELD_NAME_DURATION, duration); + + /* process stats & exports */ + iter = event_filter_match_iter_init(metrics->filter, event, ctx); + while ((metric = event_filter_match_iter_next(iter)) != NULL) T_BEGIN { + /* every metric is fed into stats */ + stats_metric_event(metric, event, metrics->pool); + + /* some metrics are exported */ + if (metric->export_info.exporter != NULL) + stats_export_event(metric, event); + } T_END; + event_filter_match_iter_deinit(&iter); +} + +struct stats_metrics_iter { + struct stats_metrics *metrics; + unsigned int idx; +}; + +struct stats_metrics_iter * +stats_metrics_iterate_init(struct stats_metrics *metrics) +{ + struct stats_metrics_iter *iter; + + iter = i_new(struct stats_metrics_iter, 1); + iter->metrics = metrics; + return iter; +} + +const struct metric *stats_metrics_iterate(struct stats_metrics_iter *iter) +{ + struct metric *const *metrics; + unsigned int count; + + metrics = array_get(&iter->metrics->metrics, &count); + if (iter->idx >= count) + return NULL; + return metrics[iter->idx++]; +} + +void stats_metrics_iterate_deinit(struct stats_metrics_iter **_iter) +{ + struct stats_metrics_iter *iter = *_iter; + + *_iter = NULL; + i_free(iter); +} diff --git a/src/stats/stats-metrics.h b/src/stats/stats-metrics.h new file mode 100644 index 0000000..6d7d745 --- /dev/null +++ b/src/stats/stats-metrics.h @@ -0,0 +1,134 @@ +#ifndef STATS_METRICS_H +#define STATS_METRICS_H + +#include "stats-settings.h" +#include "sha1.h" + +#define STATS_EVENT_FIELD_NAME_DURATION "duration" + +struct metric; +struct stats_metrics; + +struct exporter { + const char *name; + + /* + * serialization format options + * + * the "how do we encode the event before sending it" knobs + */ + enum event_exporter_time_fmt time_format; + + /* Max length for string field values */ + size_t format_max_field_len; + + /* function to serialize the event */ + void (*format)(const struct metric *, struct event *, buffer_t *); + + /* mime type for the format */ + const char *format_mime_type; + + /* + * transport options + * + * the "how do we get the event to the external location" knobs + */ + const char *transport_args; + unsigned int transport_timeout; + + /* function to send the event */ + void (*transport)(const struct exporter *, const buffer_t *); +}; + +struct metric_export_info { + const struct exporter *exporter; + + enum event_exporter_includes { + EVENT_EXPORTER_INCL_NONE = 0, + EVENT_EXPORTER_INCL_NAME = 0x01, + EVENT_EXPORTER_INCL_HOSTNAME = 0x02, + EVENT_EXPORTER_INCL_TIMESTAMPS = 0x04, + EVENT_EXPORTER_INCL_CATEGORIES = 0x08, + EVENT_EXPORTER_INCL_FIELDS = 0x10, + } include; +}; + +struct metric_field { + const char *field_key; + struct stats_dist *stats; +}; + +enum metric_value_type { + METRIC_VALUE_TYPE_STR, + METRIC_VALUE_TYPE_INT, + METRIC_VALUE_TYPE_BUCKET_INDEX, +}; + +struct metric_value { + enum metric_value_type type; + unsigned char hash[SHA1_RESULTLEN]; + intmax_t intmax; +}; + +struct metric { + const struct stats_metric_settings *set; + const char *name; + /* When this metric is a sub-metric, then this is the + suffix for name and any sub_names before it. + + So if we have + + struct metric imap_command { + event_name = imap_command_finished + group_by = cmd_name + } + + The metric.name will always be imap_command and for each sub-metric + metric.sub_name will be whatever the cmd_name is, such as 'select'. + + This is a display name and does not guarantee uniqueness. + */ + const char *sub_name; + + /* Timing for how long the event existed */ + struct stats_dist *duration_stats; + + unsigned int fields_count; + struct metric_field *fields; + + unsigned int group_by_count; + const struct stats_metric_settings_group_by *group_by; + struct metric_value group_value; + ARRAY(struct metric *) sub_metrics; + + struct metric_export_info export_info; +}; + +bool stats_metrics_add_dynamic(struct stats_metrics *metrics, + struct stats_metric_settings *set, + const char **error_r); + +bool stats_metrics_remove_dynamic(struct stats_metrics *metrics, + const char *name); + +struct stats_metrics *stats_metrics_init(const struct stats_settings *set); +void stats_metrics_deinit(struct stats_metrics **metrics); + +/* Reset all metrics */ +void stats_metrics_reset(struct stats_metrics *metrics); + +/* Returns event filter created from the stats_settings. */ +struct event_filter * +stats_metrics_get_event_filter(struct stats_metrics *metrics); + +/* Update metrics with given event. */ +void stats_metrics_event(struct stats_metrics *metrics, struct event *event, + const struct failure_context *ctx); + +/* Iterate through all the tracked metrics. */ +struct stats_metrics_iter * +stats_metrics_iterate_init(struct stats_metrics *metrics); +const struct metric *stats_metrics_iterate(struct stats_metrics_iter *iter); +void stats_metrics_iterate_deinit(struct stats_metrics_iter **iter); + +#endif diff --git a/src/stats/stats-service-openmetrics.c b/src/stats/stats-service-openmetrics.c new file mode 100644 index 0000000..9c50592 --- /dev/null +++ b/src/stats/stats-service-openmetrics.c @@ -0,0 +1,779 @@ +/* Copyright (c) 2019 Dovecot authors, see the included COPYING file */ + +#include "stats-common.h" +#include "dovecot-version.h" +#include "str.h" +#include "array.h" +#include "json-parser.h" +#include "ioloop.h" +#include "ostream.h" +#include "stats-dist.h" +#include "http-server.h" +#include "client-http.h" +#include "stats-settings.h" +#include "stats-metrics.h" +#include "stats-service-private.h" + +#define OPENMETRICS_CONTENT_VERSION "0.0.1" + +#ifdef DOVECOT_REVISION +#define OPENMETRICS_BUILD_INFO \ + "version=\""DOVECOT_VERSION"\"," \ + "revision=\""DOVECOT_REVISION"\"" +#else +#define OPENMETRICS_BUILD_INFO \ + "version=\""DOVECOT_VERSION"\"" +#endif + +enum openmetrics_metric_type { + OPENMETRICS_METRIC_TYPE_COUNT, + OPENMETRICS_METRIC_TYPE_DURATION, + OPENMETRICS_METRIC_TYPE_HISTOGRAM, +}; + +enum openmetrics_request_state { + OPENMETRICS_REQUEST_STATE_INIT = 0, + OPENMETRICS_REQUEST_STATE_METRIC, + OPENMETRICS_REQUEST_STATE_METRIC_HEADER, + OPENMETRICS_REQUEST_STATE_SUB_METRICS, + OPENMETRICS_REQUEST_STATE_METRIC_BODY, + OPENMETRICS_REQUEST_STATE_FINISHED, +}; + +struct openmetrics_request_sub_metric { + size_t labels_pos; + const struct metric *metric; + unsigned int sub_index; +}; + +struct openmetrics_request { + struct ostream *output; + + enum openmetrics_request_state state; + struct stats_metrics_iter *stats_iter; + const struct metric *metric; + enum openmetrics_metric_type metric_type; + string_t *labels; + size_t labels_pos; + ARRAY(struct openmetrics_request_sub_metric) sub_metric_stack; + + bool has_submetric:1; +}; + +/* https://prometheus.io/docs/concepts/data_model/#metric-names-and-labels: + + Every time series is uniquely identified by its metric name and optional + key-value pairs called labels. + + The metric name specifies the general feature of a system that is measured + (e.g. http_requests_total - the total number of HTTP requests received). It + may contain ASCII letters and digits, as well as underscores and colons. It + must match the regex [a-zA-Z_:][a-zA-Z0-9_:]*. + */ + +static bool openmetrics_check_name(const char *name) +{ + const unsigned char *p, *pend; + + p = (const unsigned char *)name; + pend = p + strlen(name); + + if (p == pend) + return FALSE; + + if (!(*p >= 'a' && *p <= 'z') && !(*p >= 'A' && *p <= 'Z') && + *p != '_' && *p != ':') + return FALSE; + p++; + while (p < pend) { + if (!(*p >= 'a' && *p <= 'z') && !(*p >= 'A' && *p <= 'Z') && + !(*p >= '0' && *p <= '9') && *p != '_' && *p != ':') + return FALSE; + p++; + } + return TRUE; +} + +static void openmetrics_export_dovecot(string_t *out) +{ + i_assert(stats_startup_time <= ioloop_time); + str_append(out, "# HELP process_start_time_seconds " + "Timestamp of service start\n"); + str_append(out, "# TYPE process_start_time_seconds gauge\n"); + str_printfa(out, "process_start_time_seconds %"PRIdTIME_T"\n", + stats_startup_time); + + str_append(out, "# HELP dovecot_build " + "Dovecot build information\n"); + str_append(out, "# TYPE dovecot_build info\n"); + str_append(out, "dovecot_build_info{"OPENMETRICS_BUILD_INFO"} 1\n"); +} + +static void openmetrics_export_eof(string_t *out) +{ + str_append(out, "# EOF\n"); +} + +static void +openmetrics_export_metric_value(struct openmetrics_request *req, string_t *out, + const struct metric *metric) +{ + /* Metric name */ + str_append(out, "dovecot_"); + str_append(out, req->metric->name); + switch (req->metric_type) { + case OPENMETRICS_METRIC_TYPE_COUNT: + if (req->metric->group_by != NULL && str_len(req->labels) == 0) + str_append(out, "_count"); + else + str_append(out, "_total"); + break; + case OPENMETRICS_METRIC_TYPE_DURATION: + if (req->metric->group_by != NULL && str_len(req->labels) == 0) + str_append(out, "_duration_seconds_sum"); + else + str_append(out, "_duration_seconds_total"); + break; + case OPENMETRICS_METRIC_TYPE_HISTOGRAM: + i_unreached(); + } + /* Labels */ + if (str_len(req->labels) > 0) { + str_append_c(out, '{'); + str_append_str(out, req->labels); + str_append_c(out, '}'); + } + /* Value */ + switch (req->metric_type) { + case OPENMETRICS_METRIC_TYPE_COUNT: + str_printfa(out, " %u\n", + stats_dist_get_count(metric->duration_stats)); + break; + case OPENMETRICS_METRIC_TYPE_DURATION: + /* Convert from microseconds to seconds */ + str_printfa(out, " %.6f\n", + stats_dist_get_sum(metric->duration_stats)/1e6F); + break; + case OPENMETRICS_METRIC_TYPE_HISTOGRAM: + i_unreached(); + } +} + +static const struct metric * +openmetrics_find_histogram_bucket(const struct metric *metric, + unsigned int index) +{ + struct metric *sub_metric; + + if (!array_is_created(&metric->sub_metrics)) + return NULL; + + array_foreach_elem(&metric->sub_metrics, sub_metric) { + if (sub_metric->group_value.type != + METRIC_VALUE_TYPE_BUCKET_INDEX) + continue; + if (sub_metric->group_value.intmax == index) + return sub_metric; + } + + return NULL; +} + +static void +openmetrics_export_histogram_bucket(struct openmetrics_request *req, + string_t *out, const struct metric *metric, + intmax_t bucket_limit, int64_t count) +{ + /* Metric name */ + str_append(out, "dovecot_"); + str_append(out, metric->name); + str_append(out, "_bucket"); + /* Labels */ + str_append_c(out, '{'); + if (str_len(req->labels) > 0) { + str_append_str(out, req->labels); + str_append_c(out, ','); + } + if (bucket_limit == INTMAX_MAX) + str_append(out, "le=\"+Inf\""); + else if (strcmp(metric->group_by->field, + STATS_EVENT_FIELD_NAME_DURATION) == 0) { + /* Convert from microseconds to seconds */ + str_printfa(out, "le=\"%.6f\"", bucket_limit/1e6F); + } else { + str_printfa(out, "le=\"%jd\"", bucket_limit); + } + str_printfa(out, "} %"PRIu64"\n", count); +} + +static void +openmetrics_export_histogram(struct openmetrics_request *req, string_t *out, + const struct metric *metric) +{ + const struct stats_metric_settings_group_by *group_by = + metric->group_by; + float sum = 0; + uint64_t count = 0; + + /* Buckets */ + for (unsigned int i = 0; i < group_by->num_ranges; i++) { + const struct metric *sub_metric = + openmetrics_find_histogram_bucket(metric, i); + + if (sub_metric != NULL) { + sum += stats_dist_get_sum(sub_metric->duration_stats); + count += stats_dist_get_count( + sub_metric->duration_stats); + } + + openmetrics_export_histogram_bucket(req, out, metric, + group_by->ranges[i].max, + count); + } + + /* There is either no data in histogram, which adding the optional + sum and count metrics doesn't add any new information or + these have already been exported for submetrics. */ + if (count == 0) + return; + + /* Sum */ + str_append(out, "dovecot_"); + str_append(out, metric->name); + str_append(out, "_sum"); + /* Labels */ + if (str_len(req->labels) > 0) { + str_append_c(out, '{'); + str_append_str(out, req->labels); + str_append_c(out, '}'); + } + if (strcmp(metric->group_by->field, + STATS_EVENT_FIELD_NAME_DURATION) == 0) { + /* Convert from microseconds to seconds */ + sum /= 1e6F; + } + str_printfa(out, " %.6f\n", sum); + /* Count */ + str_append(out, "dovecot_"); + str_append(out, metric->name); + str_append(out, "_count"); + /* Labels */ + if (str_len(req->labels) > 0) { + str_append_c(out, '{'); + str_append_str(out, req->labels); + str_append_c(out, '}'); + } + str_printfa(out, " %"PRIu64"\n", count); +} + +static void +openmetrics_export_metric_header(struct openmetrics_request *req, string_t *out) +{ + const struct metric *metric = req->metric; + + /* Description */ + str_append(out, "# HELP dovecot_"); + str_append(out, metric->name); + switch (req->metric_type) { + case OPENMETRICS_METRIC_TYPE_COUNT: + str_append(out, " Total number of all events of this kind"); + break; + case OPENMETRICS_METRIC_TYPE_DURATION: + str_append(out, "_duration_seconds Total duration of all events of this kind"); + break; + case OPENMETRICS_METRIC_TYPE_HISTOGRAM: + str_append(out, " Histogram"); + break; + } + if (*metric->set->description != '\0') { + str_append(out, " of "); + str_append(out, metric->set->description); + } + str_append_c(out, '\n'); + /* Type */ + str_append(out, "# TYPE dovecot_"); + str_append(out, metric->name); + switch (req->metric_type) { + case OPENMETRICS_METRIC_TYPE_COUNT: + str_append(out, " counter\n"); + break; + case OPENMETRICS_METRIC_TYPE_DURATION: + str_append(out, "_duration_seconds counter\n"); + break; + case OPENMETRICS_METRIC_TYPE_HISTOGRAM: + str_append(out, " histogram\n"); + break; + } +} + +static void +openmetrics_export_submetric(struct openmetrics_request *req, string_t *out, + const struct metric *metric) +{ + /* This metric may be a submetric and therefore have a label + associated with it. */ + if (metric->sub_name != NULL) { + str_append_c(req->labels, '"'); + json_append_escaped(req->labels, metric->sub_name); + str_append_c(req->labels, '"'); + } + + if (req->metric_type == OPENMETRICS_METRIC_TYPE_HISTOGRAM) { + if (metric->group_by == NULL || + metric->group_by[0].func != STATS_METRIC_GROUPBY_QUANTIZED) + return; + + openmetrics_export_histogram(req, out, metric); + return; + } + + openmetrics_export_metric_value(req, out, metric); + + req->has_submetric = TRUE; +} + +static const struct metric * +openmetrics_export_sub_metric_get(struct openmetrics_request_sub_metric *reqsm) +{ + if (reqsm->sub_index >= array_count(&reqsm->metric->sub_metrics)) + return NULL; + + return array_idx_elem(&reqsm->metric->sub_metrics, reqsm->sub_index); +} + +static const struct metric * +openmetrics_export_sub_metric_get_next( + struct openmetrics_request_sub_metric *reqsm) +{ + /* Get the next valid sub-metric */ + reqsm->sub_index++; + return openmetrics_export_sub_metric_get(reqsm); +} + +static struct openmetrics_request_sub_metric * +openmetrics_export_sub_metric_down(struct openmetrics_request *req) +{ + struct openmetrics_request_sub_metric *reqsm = + array_back_modifiable(&req->sub_metric_stack); + const struct metric *sub_metric; + + /* Descend further into sub-metric tree */ + + if (reqsm->metric->group_by == NULL || + !openmetrics_check_name(reqsm->metric->group_by->field) || + !array_is_created(&reqsm->metric->sub_metrics) || + array_count(&reqsm->metric->sub_metrics) == 0) + return NULL; + if (reqsm->metric->group_by[0].func == STATS_METRIC_GROUPBY_QUANTIZED) { + /* Never descend into quantized group_by sub-metrics. + Histograms are exported as a single blob. */ + return NULL; + } + + /* Find sub-metric to descend into */ + sub_metric = openmetrics_export_sub_metric_get(reqsm); + if (sub_metric == NULL) { + /* None valid */ + return NULL; + } + + if (str_len(req->labels) > 0) + str_append_c(req->labels, ','); + str_append(req->labels, reqsm->metric->group_by->field); + str_append_c(req->labels, '='); + reqsm->labels_pos = str_len(req->labels); + + /* Descend */ + reqsm = array_append_space(&req->sub_metric_stack); + reqsm->metric = sub_metric; + + return reqsm; +} + +static struct openmetrics_request_sub_metric * +openmetrics_export_sub_metric_up_next(struct openmetrics_request *req) +{ + struct openmetrics_request_sub_metric *reqsm; + const struct metric *sub_metric = NULL; + + /* Ascend to next sub-metric of an ancestor */ + + while (array_count(&req->sub_metric_stack) > 1) { + /* Ascend */ + array_pop_back(&req->sub_metric_stack); + reqsm = array_back_modifiable(&req->sub_metric_stack); + str_truncate(req->labels, reqsm->labels_pos); + + /* Find next sub-metric */ + sub_metric = openmetrics_export_sub_metric_get_next(reqsm); + if (sub_metric != NULL) { + /* None valid */ + break; + } + } + if (sub_metric == NULL) { + /* End of sub-metric tree */ + return NULL; + } + + /* Descend */ + reqsm = array_append_space(&req->sub_metric_stack); + reqsm->metric = sub_metric; + return reqsm; +} + +static struct openmetrics_request_sub_metric * +openmetrics_export_sub_metric_current(struct openmetrics_request *req) +{ + struct openmetrics_request_sub_metric *reqsm; + + /* Get state for current sub-metric */ + + if (!array_is_created(&req->sub_metric_stack)) + i_array_init(&req->sub_metric_stack, 8); + if (array_count(&req->sub_metric_stack) >= 2) { + /* Already walking the sub-metric tree */ + return array_back_modifiable(&req->sub_metric_stack); + } + + /* Start tree walking */ + + reqsm = array_append_space(&req->sub_metric_stack); + reqsm->metric = req->metric; + reqsm->labels_pos = str_len(req->labels); + + return openmetrics_export_sub_metric_down(req); +} + +static bool +openmetrics_export_sub_metrics(struct openmetrics_request *req, string_t *out) +{ + struct openmetrics_request_sub_metric *reqsm = NULL; + + if (!array_is_created(&req->metric->sub_metrics)) + return TRUE; + + reqsm = openmetrics_export_sub_metric_current(req); + if (reqsm == NULL) { + /* No valid sub-metrics to export */ + return TRUE; + } + openmetrics_export_submetric(req, out, reqsm->metric); + + /* Try do descend into sub-metrics tree for next sub-metric to export. + */ + reqsm = openmetrics_export_sub_metric_down(req); + if (reqsm == NULL) { + /* Sub-metrics of this metric exhausted; ascend to the next + parent sub-metric. + */ + reqsm = openmetrics_export_sub_metric_up_next(req); + } + + if (reqsm == NULL) { + /* Finished */ + array_clear(&req->sub_metric_stack); + return TRUE; + } + return FALSE; +} + +static void +openmetrics_export_metric_body(struct openmetrics_request *req, string_t *out) +{ + openmetrics_export_metric_value(req, out, req->metric); +} + +static int +openmetrics_send_buffer(struct openmetrics_request *req, buffer_t *buffer) +{ + ssize_t sent; + + if (buffer->used == 0) + return 1; + + sent = o_stream_send(req->output, buffer->data, buffer->used); + if (sent < 0) + return -1; + + /* Max buffer size is enormous */ + i_assert((size_t)sent == buffer->used); + + if (o_stream_get_buffer_used_size(req->output) >= IO_BLOCK_SIZE) + return 0; + return 1; +} + +static bool openmetrics_export_has_histogram(struct openmetrics_request *req) +{ + const struct metric *metric = req->metric; + unsigned int i; + + if (metric->group_by_count == 0) { + /* No group_by */ + return FALSE; + } + + /* We can only support quantized group_by when it is the last group + item. */ + for (i = 0; i < (metric->group_by_count - 1); i++) { + if (metric->group_by[i].func == + STATS_METRIC_GROUPBY_QUANTIZED) + return FALSE; + } + + return (metric->group_by[metric->group_by_count - 1].func == + STATS_METRIC_GROUPBY_QUANTIZED); +} + +static void openmetrics_export_next(struct openmetrics_request *req) +{ + /* Determine what to export next. */ + switch (req->metric_type) { + case OPENMETRICS_METRIC_TYPE_COUNT: + /* Continue with duration output for this metric. */ + req->metric_type = OPENMETRICS_METRIC_TYPE_DURATION; + req->state = OPENMETRICS_REQUEST_STATE_METRIC_HEADER; + break; + case OPENMETRICS_METRIC_TYPE_DURATION: + if (openmetrics_export_has_histogram(req)) { + /* Continue with histogram output for this metric. */ + req->metric_type = OPENMETRICS_METRIC_TYPE_HISTOGRAM; + req->state = OPENMETRICS_REQUEST_STATE_METRIC_HEADER; + } else { + /* No histogram; continue with next metric */ + req->state = OPENMETRICS_REQUEST_STATE_METRIC; + } + break; + case OPENMETRICS_METRIC_TYPE_HISTOGRAM: + /* Continue with next metric */ + req->state = OPENMETRICS_REQUEST_STATE_METRIC; + break; + } +} + +static void +openmetrics_export_continue(struct openmetrics_request *req, string_t *out) +{ + switch (req->state) { + case OPENMETRICS_REQUEST_STATE_INIT: + /* Export the Dovecot base metrics. */ + i_assert(req->stats_iter == NULL); + req->stats_iter = stats_metrics_iterate_init(stats_metrics); + openmetrics_export_dovecot(out); + req->state = OPENMETRICS_REQUEST_STATE_METRIC; + break; + case OPENMETRICS_REQUEST_STATE_METRIC: + /* Export the next metric. */ + i_assert(req->stats_iter != NULL); + do { + req->metric = stats_metrics_iterate(req->stats_iter); + } while (req->metric != NULL && + !openmetrics_check_name(req->metric->name)); + if (req->metric == NULL) { + /* Finished exporting metrics. */ + req->state = OPENMETRICS_REQUEST_STATE_FINISHED; + break; + } + + if (req->labels == NULL) + req->labels = str_new(default_pool, 32); + else + str_truncate(req->labels, 0); + req->labels_pos = 0; + + /* Start with count output for this metric if the type + is not histogram. If the metric is of type histogram, + start with quantiles. */ + if (openmetrics_export_has_histogram(req)) + req->metric_type = OPENMETRICS_METRIC_TYPE_HISTOGRAM; + else + req->metric_type = OPENMETRICS_METRIC_TYPE_COUNT; + req->state = OPENMETRICS_REQUEST_STATE_METRIC_HEADER; + /* Fall through */ + case OPENMETRICS_REQUEST_STATE_METRIC_HEADER: + /* Export the HELP/TYPE header for the current metric */ + str_truncate(req->labels, req->labels_pos); + req->has_submetric = FALSE; + openmetrics_export_metric_header(req, out); + req->state = OPENMETRICS_REQUEST_STATE_SUB_METRICS; + break; + case OPENMETRICS_REQUEST_STATE_SUB_METRICS: + /* Export the sub-metrics for the current metric. This will + return for each sub-metric, so that the out string buffer + stays small. */ + if (!openmetrics_export_sub_metrics(req, out)) + break; + /* All sub-metrics written. */ + req->state = OPENMETRICS_REQUEST_STATE_METRIC_BODY; + break; + case OPENMETRICS_REQUEST_STATE_METRIC_BODY: + /* Export the body of the current metric. */ + str_truncate(req->labels, req->labels_pos); + if (req->metric_type == OPENMETRICS_METRIC_TYPE_HISTOGRAM) + openmetrics_export_histogram(req, out, req->metric); + else + openmetrics_export_metric_body(req, out); + openmetrics_export_next(req); + break; + case OPENMETRICS_REQUEST_STATE_FINISHED: + i_unreached(); + } +} + +static void openmetrics_handle_write_error(struct openmetrics_request *req) +{ + i_info("openmetrics: write(%s) failed: %s", + o_stream_get_name(req->output), + o_stream_get_error(req->output)); + o_stream_destroy(&req->output); +} + +static void openmetrics_request_deinit(struct openmetrics_request *req) +{ + stats_metrics_iterate_deinit(&req->stats_iter); + str_free(&req->labels); + array_free(&req->sub_metric_stack); +} + +static int openmetrics_export(struct openmetrics_request *req) +{ + string_t *out; + int ret; + + ret = o_stream_flush(req->output); + if (ret < 0) { + openmetrics_handle_write_error(req); + return -1; + } + if (ret == 0) { + /* Output stream buffer needs to be flushed further */ + return 0; + } + + if (req->state == OPENMETRICS_REQUEST_STATE_FINISHED) { + /* All metrics were exported already, so we can finish the + HTTP request now. */ + o_stream_destroy(&req->output); + return 1; + } + + /* Export metrics into a string buffer and write that buffer to the + output stream after each (sub-)metric, so that the string buffer + stays small. The output stream buffer can grow bigger, but writing is + stopped for later resumption when the output stream buffer has grown + beyond an optimal size. */ + out = t_str_new(1024); + for (;;) { + str_truncate(out, 0); + + openmetrics_export_continue(req, out); + + ret = openmetrics_send_buffer(req, out); + if (ret < 0) { + openmetrics_handle_write_error(req); + return -1; + } + if (req->state == OPENMETRICS_REQUEST_STATE_FINISHED) { + /* Finished export of metrics, but the output stream + buffer may still contain data. */ + break; + } + if (ret == 0) { + /* Output stream buffer is filled up beyond the optimal + size; wait until we can write more. */ + return ret; + } + } + + /* Send EOF */ + str_truncate(out, 0); + openmetrics_export_eof(out); + ret = openmetrics_send_buffer(req, out); + if (ret < 0) { + openmetrics_handle_write_error(req); + return -1; + } + + /* Cleanup everything except the output stream */ + openmetrics_request_deinit(req); + + /* Finished; flush output */ + ret = o_stream_finish(req->output); + if (ret < 0) { + openmetrics_handle_write_error(req); + return -1; + } + return ret; +} + +static void openmetrics_request_destroy(struct openmetrics_request *req) +{ + o_stream_destroy(&req->output); + openmetrics_request_deinit(req); +} + +static void +stats_service_openmetrics_request(void *context ATTR_UNUSED, + struct http_server_request *hsreq, + const char *sub_path) +{ + const struct http_request *hreq = http_server_request_get(hsreq); + struct http_server_response *hsresp; + struct openmetrics_request *req; + pool_t pool; + + if (strcmp(hreq->method, "OPTIONS") == 0) { + hsresp = http_server_response_create(hsreq, 200, "OK"); + http_server_response_add_header(hsresp, "Allow", "GET"); + http_server_response_submit(hsresp); + return; + } + if (strcmp(hreq->method, "GET") != 0) { + http_server_request_fail_bad_method(hsreq, "GET"); + return; + } + if (*sub_path != '\0') { + http_server_request_fail(hsreq, 404, "Not Found"); + return; + } + + pool = http_server_request_get_pool(hsreq); + req = p_new(pool, struct openmetrics_request, 1); + + http_server_request_set_destroy_callback( + hsreq, openmetrics_request_destroy, req); + + hsresp = http_server_response_create(hsreq, 200, "OK"); + http_server_response_add_header( + hsresp, "Content-Type", + "application/openmetrics-text; version="OPENMETRICS_CONTENT_VERSION"; " + "charset=utf-8"); + + req->output = http_server_response_get_payload_output( + hsresp, SIZE_MAX, FALSE); + + o_stream_set_flush_callback(req->output, openmetrics_export, req); + o_stream_set_flush_pending(req->output, TRUE); +} + +void stats_service_openmetrics_init(void) +{ + struct stats_metrics_iter *iter; + const struct metric *metric; + + iter = stats_metrics_iterate_init(stats_metrics); + while ((metric = stats_metrics_iterate(iter)) != NULL) { + if (!openmetrics_check_name(metric->name)) { + i_warning( + "stats: openmetrics: " + "Metric `%s' is not valid for OpenMetrics" + "(invalid metric name; skipped)", + metric->name); + } + } + stats_metrics_iterate_deinit(&iter); + + stats_http_resource_add("/metrics", "OpenMetrics", + stats_service_openmetrics_request, NULL); +} diff --git a/src/stats/stats-service-private.h b/src/stats/stats-service-private.h new file mode 100644 index 0000000..5f5ab20 --- /dev/null +++ b/src/stats/stats-service-private.h @@ -0,0 +1,8 @@ +#ifndef STATS_SERVICE_PRIVATE_H +#define STATS_SERVICE_PRIVATE_H + +#include "stats-service.h" + +void stats_service_openmetrics_init(void); + +#endif diff --git a/src/stats/stats-service.c b/src/stats/stats-service.c new file mode 100644 index 0000000..fa67864 --- /dev/null +++ b/src/stats/stats-service.c @@ -0,0 +1,15 @@ +/* Copyright (c) 2019 Dovecot authors, see the included COPYING file */ + +#include "stats-common.h" +#include "http-server.h" +#include "stats-service-private.h" + +void stats_services_init(void) +{ + stats_service_openmetrics_init(); +} + +void stats_services_deinit(void) +{ + /* Nothing yet */ +} diff --git a/src/stats/stats-service.h b/src/stats/stats-service.h new file mode 100644 index 0000000..fab477b --- /dev/null +++ b/src/stats/stats-service.h @@ -0,0 +1,7 @@ +#ifndef STATS_SERVICE_H +#define STATS_SERVICE_H + +void stats_services_init(void); +void stats_services_deinit(void); + +#endif diff --git a/src/stats/stats-settings.c b/src/stats/stats-settings.c new file mode 100644 index 0000000..f75a737 --- /dev/null +++ b/src/stats/stats-settings.c @@ -0,0 +1,538 @@ +/* Copyright (c) 2009-2018 Dovecot authors, see the included COPYING file */ + +#include "stats-common.h" +#include "buffer.h" +#include "settings-parser.h" +#include "service-settings.h" +#include "stats-settings.h" +#include "array.h" + +/* <settings checks> */ +#include "event-filter.h" +#include <math.h> +/* </settings checks> */ + +static bool stats_metric_settings_check(void *_set, pool_t pool, const char **error_r); +static bool stats_exporter_settings_check(void *_set, pool_t pool, const char **error_r); +static bool stats_settings_check(void *_set, pool_t pool, const char **error_r); + +/* <settings checks> */ +static struct file_listener_settings stats_unix_listeners_array[] = { + { "stats-reader", 0600, "", "" }, + { "stats-writer", 0660, "", "$default_internal_group" }, + { "login/stats-writer", 0600, "$default_login_user", "" }, +}; +static struct file_listener_settings *stats_unix_listeners[] = { + &stats_unix_listeners_array[0], + &stats_unix_listeners_array[1], + &stats_unix_listeners_array[2], +}; +static buffer_t stats_unix_listeners_buf = { + { { stats_unix_listeners, sizeof(stats_unix_listeners) } } +}; +/* </settings checks> */ + +struct service_settings stats_service_settings = { + .name = "stats", + .protocol = "", + .type = "", + .executable = "stats", + .user = "$default_internal_user", + .group = "", + .privileged_group = "", + .extra_groups = "", + .chroot = "", + + .drop_priv_before_exec = FALSE, + + .process_min_avail = 0, + .process_limit = 1, + .client_limit = 0, + .service_count = 0, + .idle_kill = UINT_MAX, + .vsz_limit = UOFF_T_MAX, + + .unix_listeners = { { &stats_unix_listeners_buf, + sizeof(stats_unix_listeners[0]) } }, + .inet_listeners = ARRAY_INIT, +}; + +/* + * event_exporter { } block settings + */ + +#undef DEF +#define DEF(type, name) \ + SETTING_DEFINE_STRUCT_##type(#name, name, struct stats_exporter_settings) + +static const struct setting_define stats_exporter_setting_defines[] = { + DEF(STR, name), + DEF(STR, transport), + DEF(STR, transport_args), + DEF(TIME_MSECS, transport_timeout), + DEF(STR, format), + DEF(STR, format_args), + SETTING_DEFINE_LIST_END +}; + +static const struct stats_exporter_settings stats_exporter_default_settings = { + .name = "", + .transport = "", + .transport_args = "", + .transport_timeout = 250, /* ms */ + .format = "", + .format_args = "", +}; + +const struct setting_parser_info stats_exporter_setting_parser_info = { + .defines = stats_exporter_setting_defines, + .defaults = &stats_exporter_default_settings, + + .type_offset = offsetof(struct stats_exporter_settings, name), + .struct_size = sizeof(struct stats_exporter_settings), + + .parent_offset = SIZE_MAX, + .check_func = stats_exporter_settings_check, +}; + +/* + * metric { } block settings + */ + +#undef DEF +#define DEF(type, name) \ + SETTING_DEFINE_STRUCT_##type(#name, name, struct stats_metric_settings) + +static const struct setting_define stats_metric_setting_defines[] = { + DEF(STR, metric_name), + DEF(STR, fields), + DEF(STR, group_by), + DEF(STR, filter), + DEF(STR, exporter), + DEF(STR, exporter_include), + DEF(STR, description), + SETTING_DEFINE_LIST_END +}; + +static const struct stats_metric_settings stats_metric_default_settings = { + .metric_name = "", + .fields = "", + .filter = "", + .exporter = "", + .group_by = "", + .exporter_include = STATS_METRIC_SETTINGS_DEFAULT_EXPORTER_INCLUDE, + .description = "", +}; + +const struct setting_parser_info stats_metric_setting_parser_info = { + .defines = stats_metric_setting_defines, + .defaults = &stats_metric_default_settings, + + .type_offset = offsetof(struct stats_metric_settings, metric_name), + .struct_size = sizeof(struct stats_metric_settings), + + .parent_offset = SIZE_MAX, + .check_func = stats_metric_settings_check, +}; + +/* + * top-level settings + */ + +#undef DEF +#define DEF(type, name) \ + SETTING_DEFINE_STRUCT_##type(#name, name, struct stats_settings) +#undef DEFLIST_UNIQUE +#define DEFLIST_UNIQUE(field, name, defines) \ + { .type = SET_DEFLIST_UNIQUE, .key = name, \ + .offset = offsetof(struct stats_settings, field), \ + .list_info = defines } + +static const struct setting_define stats_setting_defines[] = { + DEF(STR, stats_http_rawlog_dir), + + DEFLIST_UNIQUE(metrics, "metric", &stats_metric_setting_parser_info), + DEFLIST_UNIQUE(exporters, "event_exporter", &stats_exporter_setting_parser_info), + SETTING_DEFINE_LIST_END +}; + +const struct stats_settings stats_default_settings = { + .stats_http_rawlog_dir = "", + + .metrics = ARRAY_INIT, + .exporters = ARRAY_INIT, +}; + +const struct setting_parser_info stats_setting_parser_info = { + .module_name = "stats", + .defines = stats_setting_defines, + .defaults = &stats_default_settings, + + .type_offset = SIZE_MAX, + .struct_size = sizeof(struct stats_settings), + + .parent_offset = SIZE_MAX, + .check_func = stats_settings_check, +}; + +/* <settings checks> */ +static bool parse_format_args_set_time(struct stats_exporter_settings *set, + enum event_exporter_time_fmt fmt, + const char **error_r) +{ + if ((set->parsed_time_format != EVENT_EXPORTER_TIME_FMT_NATIVE) && + (set->parsed_time_format != fmt)) { + *error_r = t_strdup_printf("Exporter '%s' specifies multiple " + "time format args", set->name); + return FALSE; + } + + set->parsed_time_format = fmt; + + return TRUE; +} + +static bool parse_format_args(struct stats_exporter_settings *set, + const char **error_r) +{ + const char *const *tmp; + + /* Defaults */ + set->parsed_time_format = EVENT_EXPORTER_TIME_FMT_NATIVE; + + tmp = t_strsplit_spaces(set->format_args, " "); + + /* + * If the config contains multiple types of the same type (e.g., + * both time-rfc3339 and time-unix) we fail the config check. + * + * Note: At the moment, we have only time-* tokens. In the future + * when we have other tokens, they should be parsed here. + */ + for (; *tmp != NULL; tmp++) { + enum event_exporter_time_fmt fmt; + + if (strcmp(*tmp, "time-rfc3339") == 0) { + fmt = EVENT_EXPORTER_TIME_FMT_RFC3339; + } else if (strcmp(*tmp, "time-unix") == 0) { + fmt = EVENT_EXPORTER_TIME_FMT_UNIX; + } else { + *error_r = t_strdup_printf("Unknown exporter format " + "arg: %s", *tmp); + return FALSE; + } + + if (!parse_format_args_set_time(set, fmt, error_r)) + return FALSE; + } + + return TRUE; +} + +static bool stats_exporter_settings_check(void *_set, pool_t pool ATTR_UNUSED, + const char **error_r) +{ + struct stats_exporter_settings *set = _set; + bool time_fmt_required; + + if (set->name[0] == '\0') { + *error_r = "Exporter name can't be empty"; + return FALSE; + } + + /* TODO: The following should be plugable. + * + * Note: Make sure to mirror any changes to the below code in + * stats_exporters_add_set(). + */ + if (set->format[0] == '\0') { + *error_r = "Exporter format name can't be empty"; + return FALSE; + } else if (strcmp(set->format, "none") == 0) { + time_fmt_required = FALSE; + } else if (strcmp(set->format, "json") == 0) { + time_fmt_required = TRUE; + } else if (strcmp(set->format, "tab-text") == 0) { + time_fmt_required = TRUE; + } else { + *error_r = t_strdup_printf("Unknown exporter format '%s'", + set->format); + return FALSE; + } + + /* TODO: The following should be plugable. + * + * Note: Make sure to mirror any changes to the below code in + * stats_exporters_add_set(). + */ + if (set->transport[0] == '\0') { + *error_r = "Exporter transport name can't be empty"; + return FALSE; + } else if (strcmp(set->transport, "drop") == 0 || + strcmp(set->transport, "http-post") == 0 || + strcmp(set->transport, "log") == 0) { + /* no-op */ + } else { + *error_r = t_strdup_printf("Unknown transport type '%s'", + set->transport); + return FALSE; + } + + if (!parse_format_args(set, error_r)) + return FALSE; + + /* Some formats don't have a native way of serializing time stamps */ + if (time_fmt_required && + set->parsed_time_format == EVENT_EXPORTER_TIME_FMT_NATIVE) { + *error_r = t_strdup_printf("%s exporter format requires a " + "time-* argument", set->format); + return FALSE; + } + + return TRUE; +} + +static bool parse_metric_group_by_common(const char *func, + const char *const *params, + intmax_t *min_r, + intmax_t *max_r, + intmax_t *other_r, + const char **error_r) +{ + intmax_t min, max, other; + + if ((str_array_length(params) != 3) || + (str_to_intmax(params[0], &min) < 0) || + (str_to_intmax(params[1], &max) < 0) || + (str_to_intmax(params[2], &other) < 0)) { + *error_r = t_strdup_printf("group_by '%s' aggregate function takes " + "3 int args", func); + return FALSE; + } + + if ((min < 0) || (max < 0) || (other < 0)) { + *error_r = t_strdup_printf("group_by '%s' aggregate function " + "arguments must be >= 0", func); + return FALSE; + } + + if (min >= max) { + *error_r = t_strdup_printf("group_by '%s' aggregate function " + "min must be < max (%ju must be < %ju)", + func, min, max); + return FALSE; + } + + *min_r = min; + *max_r = max; + *other_r = other; + + return TRUE; +} + +static bool parse_metric_group_by_exp(pool_t pool, struct stats_metric_settings_group_by *group_by, + const char *const *params, const char **error_r) +{ + intmax_t min, max, base; + + if (!parse_metric_group_by_common("exponential", params, &min, &max, &base, error_r)) + return FALSE; + + if ((base != 2) && (base != 10)) { + *error_r = t_strdup_printf("group_by 'exponential' aggregate function " + "base must be one of: 2, 10 (base=%ju)", + base); + return FALSE; + } + + group_by->func = STATS_METRIC_GROUPBY_QUANTIZED; + + /* + * Allocate the bucket range array and fill it in + * + * The first bucket is special - it contains everything less than or + * equal to 'base^min'. The last bucket is also special - it + * contains everything greater than 'base^max'. + * + * The second bucket begins at 'base^min + 1', the third bucket + * begins at 'base^(min + 1) + 1', and so on. + */ + group_by->num_ranges = max - min + 2; + group_by->ranges = p_new(pool, struct stats_metric_settings_bucket_range, + group_by->num_ranges); + + /* set up min & max buckets */ + group_by->ranges[0].min = INTMAX_MIN; + group_by->ranges[0].max = pow(base, min); + group_by->ranges[group_by->num_ranges - 1].min = pow(base, max); + group_by->ranges[group_by->num_ranges - 1].max = INTMAX_MAX; + + /* remaining buckets */ + for (unsigned int i = 1; i < group_by->num_ranges - 1; i++) { + group_by->ranges[i].min = pow(base, min + (i - 1)); + group_by->ranges[i].max = pow(base, min + i); + } + + return TRUE; +} + +static bool parse_metric_group_by_lin(pool_t pool, struct stats_metric_settings_group_by *group_by, + const char *const *params, const char **error_r) +{ + intmax_t min, max, step; + + if (!parse_metric_group_by_common("linear", params, &min, &max, &step, error_r)) + return FALSE; + + if ((min + step) > max) { + *error_r = t_strdup_printf("group_by 'linear' aggregate function " + "min+step must be <= max (%ju must be <= %ju)", + min + step, max); + return FALSE; + } + + group_by->func = STATS_METRIC_GROUPBY_QUANTIZED; + + /* + * Allocate the bucket range array and fill it in + * + * The first bucket is special - it contains everything less than or + * equal to 'min'. The last bucket is also special - it contains + * everything greater than 'max'. + * + * The second bucket begins at 'min + 1', the third bucket begins at + * 'min + 1 * step + 1', the fourth at 'min + 2 * step + 1', and so on. + */ + group_by->num_ranges = (max - min) / step + 2; + group_by->ranges = p_new(pool, struct stats_metric_settings_bucket_range, + group_by->num_ranges); + + /* set up min & max buckets */ + group_by->ranges[0].min = INTMAX_MIN; + group_by->ranges[0].max = min; + group_by->ranges[group_by->num_ranges - 1].min = max; + group_by->ranges[group_by->num_ranges - 1].max = INTMAX_MAX; + + /* remaining buckets */ + for (unsigned int i = 1; i < group_by->num_ranges - 1; i++) { + group_by->ranges[i].min = min + (i - 1) * step; + group_by->ranges[i].max = min + i * step; + } + + return TRUE; +} + +static bool parse_metric_group_by(struct stats_metric_settings *set, + pool_t pool, const char **error_r) +{ + const char *const *tmp = t_strsplit_spaces(set->group_by, " "); + + if (tmp[0] == NULL) + return TRUE; + + p_array_init(&set->parsed_group_by, pool, str_array_length(tmp)); + + /* For each group_by field */ + for (; *tmp != NULL; tmp++) { + struct stats_metric_settings_group_by group_by; + const char *const *params; + + i_zero(&group_by); + + /* <field name>:<aggregation func>... */ + params = t_strsplit(*tmp, ":"); + + if (params[1] == NULL) { + /* <field name> - alias for <field>:discrete */ + group_by.func = STATS_METRIC_GROUPBY_DISCRETE; + } else if (strcmp(params[1], "discrete") == 0) { + /* <field>:discrete */ + group_by.func = STATS_METRIC_GROUPBY_DISCRETE; + if (params[2] != NULL) { + *error_r = "group_by 'discrete' aggregate function " + "does not take any args"; + return FALSE; + } + } else if (strcmp(params[1], "exponential") == 0) { + /* <field>:exponential:<min mag>:<max mag>:<base> */ + if (!parse_metric_group_by_exp(pool, &group_by, ¶ms[2], error_r)) + return FALSE; + } else if (strcmp(params[1], "linear") == 0) { + /* <field>:linear:<min val>:<max val>:<step> */ + if (!parse_metric_group_by_lin(pool, &group_by, ¶ms[2], error_r)) + return FALSE; + } else { + *error_r = t_strdup_printf("unknown aggregation function " + "'%s' on field '%s'", params[1], params[0]); + return FALSE; + } + + group_by.field = p_strdup(pool, params[0]); + + array_push_back(&set->parsed_group_by, &group_by); + } + + return TRUE; +} + +static bool stats_metric_settings_check(void *_set, pool_t pool, const char **error_r) +{ + struct stats_metric_settings *set = _set; + + if (set->metric_name[0] == '\0') { + *error_r = "Metric name can't be empty"; + return FALSE; + } + + if (set->filter[0] == '\0') { + *error_r = t_strdup_printf("metric %s { filter } is empty - " + "will not match anything", set->metric_name); + return FALSE; + } + + set->parsed_filter = event_filter_create_fragment(pool); + if (event_filter_parse(set->filter, set->parsed_filter, error_r) < 0) + return FALSE; + + if (!parse_metric_group_by(set, pool, error_r)) + return FALSE; + + return TRUE; +} + +static bool stats_settings_check(void *_set, pool_t pool ATTR_UNUSED, + const char **error_r) +{ + struct stats_settings *set = _set; + struct stats_exporter_settings *exporter; + struct stats_metric_settings *metric; + + if (!array_is_created(&set->metrics) || !array_is_created(&set->exporters)) + return TRUE; + + /* check that all metrics refer to exporters that exist */ + array_foreach_elem(&set->metrics, metric) { + bool found = FALSE; + + if (metric->exporter[0] == '\0') + continue; /* metric not exported */ + + array_foreach_elem(&set->exporters, exporter) { + if (strcmp(metric->exporter, exporter->name) == 0) { + found = TRUE; + break; + } + } + + if (!found) { + *error_r = t_strdup_printf("metric %s refers to " + "non-existent exporter '%s'", + metric->metric_name, + metric->exporter); + return FALSE; + } + } + + return TRUE; +} +/* </settings checks> */ diff --git a/src/stats/stats-settings.h b/src/stats/stats-settings.h new file mode 100644 index 0000000..54b7f9e --- /dev/null +++ b/src/stats/stats-settings.h @@ -0,0 +1,125 @@ +#ifndef STATS_SETTINGS_H +#define STATS_SETTINGS_H + +#define STATS_METRIC_SETTINGS_DEFAULT_EXPORTER_INCLUDE \ + "name hostname timestamps categories fields" + +/* <settings checks> */ +/* + * We allow a selection of a timestamp format. + * + * The 'time-unix' format generates a number with the number of seconds + * since 1970-01-01 00:00 UTC. + * + * The 'time-rfc3339' format uses the YYYY-MM-DDTHH:MM:SS.uuuuuuZ format as + * defined by RFC 3339. + * + * The special native format (not explicitly selectable in the config, but + * default if no time-* token is used) uses the format's native timestamp + * format. Note that not all formats have a timestamp data format. + * + * The native format and the rules below try to address the question: can a + * parser that doesn't have any knowledge of fields' values' types losslessly + * reconstruct the fields? + * + * For example, JSON only has strings and numbers, so it cannot represent a + * timestamp in a "context-free lossless" way. Therefore, when making a + * JSON blob, we need to decide which way to serialize timestamps. No + * matter how we do it, we incur some loss. If a decoder sees 1557232304 in + * a field, it cannot be certain if the field is an integer that just + * happens to be a reasonable timestamp, or if it actually is a timestamp. + * Same goes with RFC3339 - it could just be that the user supplied a string + * that looks like a timestamp, and that string made it into an event field. + * + * Other common serialization formats, such as CBOR, have a lossless way of + * encoding timestamps. + * + * Note that there are two concepts at play: native and default. + * + * The rules for how the format's timestamp formats are used: + * + * 1. The default time format is the native format. + * 2. The native time format may or may not exist for a given format (e.g., + * in JSON) + * 3. If the native format doesn't exist and no time format was specified in + * the config, it is a config error. + * + * We went with these rules because: + * + * 1. It prevents type information loss by default. + * 2. It completely isolates the policy from the algorithm. + * 3. It defers the decision whether each format without a native timestamp + * type should have a default acting as native until after we've had some + * operational experience. + * 4. A future decision to add a default (via 3. point) will be 100% compatible. + */ +enum event_exporter_time_fmt { + EVENT_EXPORTER_TIME_FMT_NATIVE = 0, + EVENT_EXPORTER_TIME_FMT_UNIX, + EVENT_EXPORTER_TIME_FMT_RFC3339, +}; +/* </settings checks> */ + +struct stats_exporter_settings { + const char *name; + const char *transport; + const char *transport_args; + unsigned int transport_timeout; + const char *format; + const char *format_args; + + /* parsed values */ + enum event_exporter_time_fmt parsed_time_format; +}; + +/* <settings checks> */ +enum stats_metric_group_by_func { + STATS_METRIC_GROUPBY_DISCRETE = 0, + STATS_METRIC_GROUPBY_QUANTIZED, +}; + +/* + * A range covering a stats bucket. The the interval is half closed - the + * minimum is excluded and the maximum is included. In other words: (min, max]. + * Because we don't have a +Inf and -Inf, we use INTMAX_MIN and INTMAX_MAX + * respectively. + */ +struct stats_metric_settings_bucket_range { + intmax_t min; + intmax_t max; +}; + +struct stats_metric_settings_group_by { + const char *field; + enum stats_metric_group_by_func func; + unsigned int num_ranges; + struct stats_metric_settings_bucket_range *ranges; +}; +/* </settings checks> */ + +struct stats_metric_settings { + const char *metric_name; + const char *description; + const char *fields; + const char *group_by; + const char *filter; + + ARRAY(struct stats_metric_settings_group_by) parsed_group_by; + struct event_filter *parsed_filter; + + /* exporter related fields */ + const char *exporter; + const char *exporter_include; +}; + +struct stats_settings { + const char *stats_http_rawlog_dir; + + ARRAY(struct stats_exporter_settings *) exporters; + ARRAY(struct stats_metric_settings *) metrics; +}; + +extern const struct setting_parser_info stats_setting_parser_info; +extern const struct setting_parser_info stats_metric_setting_parser_info; + +#endif diff --git a/src/stats/test-client-reader.c b/src/stats/test-client-reader.c new file mode 100644 index 0000000..e9eed0c --- /dev/null +++ b/src/stats/test-client-reader.c @@ -0,0 +1,235 @@ +/* Copyright (c) 2019 Dovecot authors, see the included COPYING file */ + +#include "test-stats-common.h" +#include "master-service-private.h" +#include "client-reader.h" +#include "connection.h" +#include "ostream.h" + +static struct connection_list *conn_list; + +struct test_connection { + struct connection conn; + + unsigned int row_count; +}; + +static void test_reader_server_destroy(struct connection *conn) +{ + io_loop_stop(conn->ioloop); +} + +static struct connection_settings client_set = { + .service_name_in = "stats-reader-server", + .service_name_out = "stats-reader-client", + .major_version = 2, + .minor_version = 0, + .allow_empty_args_input = TRUE, + .input_max_size = SIZE_MAX, + .output_max_size = SIZE_MAX, + .client = TRUE, +}; + +bool test_stats_callback(struct event *event, + enum event_callback_type type ATTR_UNUSED, + struct failure_context *ctx, const char *fmt ATTR_UNUSED, + va_list args ATTR_UNUSED) +{ + if (stats_metrics != NULL) { + stats_metrics_event(stats_metrics, event, ctx); + struct event_filter *filter = + stats_metrics_get_event_filter(stats_metrics); + return !event_filter_match(filter, event, ctx); + } + return TRUE; +} + +static const char *settings_blob_1 = +"metric=test\n" +"metric/test/metric_name=test\n" +"metric/test/filter=event=test\n" +"\n"; + +static int test_reader_server_input_args(struct connection *conn ATTR_UNUSED, + const char *const *args) +{ + if (args[0] == NULL) + return -1; + + test_assert_strcmp(args[0], "test"); + test_assert_strcmp(args[1], "1"); + + return 1; +} + +static void test_dump_metrics(void) +{ + int fds[2]; + + test_assert(socketpair(AF_UNIX, SOCK_STREAM, 0, fds) == 0); + + struct connection *conn = i_new(struct connection, 1); + + struct ioloop *loop = io_loop_create(); + + client_reader_create(fds[1]); + connection_init_client_fd(conn_list, conn, "stats", fds[0], fds[0]); + o_stream_nsend_str(conn->output, "DUMP\tcount\n"); + + io_loop_run(loop); + connection_deinit(conn); + i_free(conn); + + /* allow client-reader to finish up */ + io_loop_set_running(loop); + io_loop_handler_run(loop); + + io_loop_destroy(&loop); +} + +static void test_client_reader(void) +{ + const struct connection_vfuncs client_vfuncs = { + .input_args = test_reader_server_input_args, + .destroy = test_reader_server_destroy, + }; + + test_begin("client reader"); + + /* register some stats */ + test_init(settings_blob_1); + + client_readers_init(); + conn_list = connection_list_init(&client_set, &client_vfuncs); + + /* push event in */ + struct event *event = event_create(NULL); + event_add_category(event, &test_category); + event_set_name(event, "test"); + test_event_send(event); + event_unref(&event); + + test_assert(get_stats_dist_field("test", STATS_DIST_COUNT) == 1); + test_assert(get_stats_dist_field("test", STATS_DIST_SUM) > 0); + + /* check output from reader */ + test_dump_metrics(); + + test_deinit(); + + client_readers_deinit(); + connection_list_deinit(&conn_list); + + test_end(); +} + +static const char *settings_blob_2 = +"metric=test\n" +"metric/test/metric_name=test\n" +"metric/test/filter=event=test\n" +"metric/test/group_by=test_name\n" +"\n"; + +static int +test_reader_server_input_args_group_by(struct connection *conn, + const char *const *args) +{ + struct test_connection *tconn = + container_of(conn, struct test_connection, conn); + + if (args[0] == NULL) + return -1; + + tconn->row_count++; + + if (tconn->row_count == 1) { + test_assert_strcmp(args[0], "test"); + test_assert_strcmp(args[1], "1"); + } else if (tconn->row_count == 2) { + test_assert_strcmp(args[0], "test_alpha"); + test_assert_strcmp(args[1], "1"); + } + return 1; +} + +static void test_dump_metrics_group_by(void) +{ + int fds[2]; + + test_assert(socketpair(AF_UNIX, SOCK_STREAM, 0, fds) == 0); + + struct test_connection *conn = i_new(struct test_connection, 1); + + struct ioloop *loop = io_loop_create(); + + client_reader_create(fds[1]); + connection_init_client_fd(conn_list, &conn->conn, "stats", fds[0], fds[0]); + o_stream_nsend_str(conn->conn.output, "DUMP\tcount\n"); + + io_loop_run(loop); + connection_deinit(&conn->conn); + i_free(conn); + + io_loop_set_running(loop); + io_loop_handler_run(loop); + + io_loop_destroy(&loop); +} + +static void test_client_reader_group_by(void) +{ + const struct connection_vfuncs client_vfuncs = { + .input_args = test_reader_server_input_args_group_by, + .destroy = test_reader_server_destroy, + }; + + test_begin("client reader (group by)"); + + /* register some stats */ + test_init(settings_blob_2); + + client_readers_init(); + conn_list = connection_list_init(&client_set, &client_vfuncs); + + /* push event in */ + struct event *event = event_create(NULL); + event_add_category(event, &test_category); + event_set_name(event, "test"); + event_add_str(event, "event_name", "alpha"); + test_event_send(event); + event_unref(&event); + + test_assert(get_stats_dist_field("test", STATS_DIST_COUNT) == 1); + test_assert(get_stats_dist_field("test", STATS_DIST_SUM) > 0); + + /* check output from reader */ + test_dump_metrics_group_by(); + + test_deinit(); + + client_readers_deinit(); + connection_list_deinit(&conn_list); + + test_end(); +} + +int main(void) { + /* fake master service to pretend destroying + connections. */ + struct master_service local_master_service = { + .stopping = TRUE, + .total_available_count = 100, + .service_count_left = 100, + }; + void (*const test_functions[])(void) = { + test_client_reader, + test_client_reader_group_by, + NULL + }; + + master_service = &local_master_service; + + int ret = test_run(test_functions); + + return ret; +} diff --git a/src/stats/test-client-writer.c b/src/stats/test-client-writer.c new file mode 100644 index 0000000..1256da9 --- /dev/null +++ b/src/stats/test-client-writer.c @@ -0,0 +1,152 @@ +/* Copyright (c) 2019 Dovecot authors, see the included COPYING file */ + +#include "test-stats-common.h" +#include "master-service-private.h" +#include "client-writer.h" +#include "connection.h" +#include "ostream.h" + +static struct event *last_sent_event = NULL; +static bool recurse_back = FALSE; +static struct connection_list *conn_list; + +static void test_writer_server_destroy(struct connection *conn) +{ + io_loop_stop(conn->ioloop); +} + +static int test_writer_server_input_args(struct connection *conn, + const char *const *args ATTR_UNUSED) +{ + /* check filter */ + test_assert_strcmp(args[0], "FILTER"); + test_assert_strcmp(args[1], "(event=\"test\")"); + /* send commands now */ + string_t *send_buf = t_str_new(128); + o_stream_nsend_str(conn->output, "CATEGORY\ttest\n"); + str_printfa(send_buf, "BEGIN\t%"PRIu64"\t0\t0\t", last_sent_event->id); + event_export(last_sent_event, send_buf); + str_append_c(send_buf, '\n'); + o_stream_nsend(conn->output, str_data(send_buf), str_len(send_buf)); + str_truncate(send_buf, 0); + str_printfa(send_buf, "END\t%"PRIu64"\n", last_sent_event->id); + o_stream_nsend(conn->output, str_data(send_buf), str_len(send_buf)); + /* disconnect immediately */ + return -1; +} + +static struct connection_settings client_set = { + .service_name_in = "stats-server", + .service_name_out = "stats-client", + .major_version = 4, + .minor_version = 0, + + .input_max_size = SIZE_MAX, + .output_max_size = SIZE_MAX, + .client = TRUE, +}; + +static const struct connection_vfuncs client_vfuncs = { + .input_args = test_writer_server_input_args, + .destroy = test_writer_server_destroy, +}; + +static void test_write_one(struct event *event ATTR_UNUSED) +{ + int fds[2]; + + test_assert(socketpair(AF_UNIX, SOCK_STREAM, 0, fds) == 0); + + struct connection *conn = i_new(struct connection, 1); + + struct ioloop *loop = io_loop_create(); + + client_writer_create(fds[1]); + connection_init_client_fd(conn_list, conn, "stats", fds[0], fds[0]); + + last_sent_event = event; + io_loop_run(loop); + last_sent_event = NULL; + connection_deinit(conn); + i_free(conn); + + /* client-writer needs two loops to deinit */ + io_loop_set_running(loop); + io_loop_handler_run(loop); + io_loop_set_running(loop); + io_loop_handler_run(loop); + + io_loop_destroy(&loop); +} + +bool test_stats_callback(struct event *event, + enum event_callback_type type ATTR_UNUSED, + struct failure_context *ctx ATTR_UNUSED, + const char *fmt ATTR_UNUSED, + va_list args ATTR_UNUSED) +{ + if (recurse_back) + return TRUE; + + recurse_back = TRUE; + if (stats_metrics != NULL) { + test_write_one(event); + } + recurse_back = FALSE; + + return TRUE; +} + +static const char *settings_blob_1 = +"metric=test\n" +"metric/test/metric_name=test\n" +"metric/test/filter=event=test\n" +"\n"; + +static void test_client_writer(void) +{ + test_begin("client writer"); + + /* register some stats */ + test_init(settings_blob_1); + + client_writers_init(); + conn_list = connection_list_init(&client_set, &client_vfuncs); + + /* push event in */ + struct event *event = event_create(NULL); + event_add_category(event, &test_category); + event_set_name(event, "test"); + test_event_send(event); + event_unref(&event); + + test_assert(get_stats_dist_field("test", STATS_DIST_COUNT) == 1); + test_assert(get_stats_dist_field("test", STATS_DIST_SUM) > 0); + + test_deinit(); + + client_writers_deinit(); + connection_list_deinit(&conn_list); + + test_end(); +} + +int main(void) { + /* fake master service to pretend destroying + connections. */ + struct master_service local_master_service = { + .stopping = TRUE, + .total_available_count = 100, + .service_count_left = 100, + }; + void (*const test_functions[])(void) = { + test_client_writer, + NULL + }; + + master_service = &local_master_service; + + int ret = test_run(test_functions); + + return ret; +} diff --git a/src/stats/test-stats-common.c b/src/stats/test-stats-common.c new file mode 100644 index 0000000..3e307a9 --- /dev/null +++ b/src/stats/test-stats-common.c @@ -0,0 +1,99 @@ +/* Copyright (c) 2019 Dovecot authors, see the included COPYING file */ + +#include "test-stats-common.h" +#include <time.h> +#include <unistd.h> + +struct event_category test_category = { + .name = "test", +}; + +struct event_category child_test_category = { + .name = "child", + .parent = &test_category, +}; + +pool_t test_pool; +struct stats_metrics *stats_metrics = NULL; +time_t stats_startup_time; + +static bool callback_added = FALSE; + +static struct stats_settings *read_settings(const char *settings) +{ + struct istream *is = test_istream_create(settings); + const char *error; + struct setting_parser_context *ctx = + settings_parser_init(test_pool, &stats_setting_parser_info, 0); + if (settings_parse_stream_read(ctx, is) < 0) + i_fatal("Failed to parse settings: %s", + settings_parser_get_error(ctx)); + if (!settings_parser_check(ctx, test_pool, &error)) + i_fatal("Failed to parse settings: %s", + error); + struct stats_settings *set = settings_parser_get(ctx); + settings_parser_deinit(&ctx); + i_stream_unref(&is); + return set; +} + +void test_init(const char *settings_blob) +{ + if (!callback_added) { + event_register_callback(test_stats_callback); + callback_added = TRUE; + } + + stats_event_categories_init(); + test_pool = pool_alloconly_create(MEMPOOL_GROWING"test pool", 2048); + stats_startup_time = time(NULL); + + /* register test categories */ + stats_event_category_register(test_category.name, NULL); + stats_event_category_register(child_test_category.name, + &test_category); + struct stats_settings *set = read_settings(settings_blob); + stats_metrics = stats_metrics_init(set); +} + +void test_deinit(void) +{ + stats_metrics_deinit(&stats_metrics); + stats_event_categories_deinit(); + pool_unref(&test_pool); +} + +void test_event_send(struct event *event) +{ + struct failure_context ctx = { + .type = LOG_TYPE_DEBUG, + }; + + usleep(1); /* make sure duration>0 always */ + event_send(event, &ctx, "hello"); +} + +uint64_t get_stats_dist_field(const char *metric_name, enum stats_dist_field field) +{ + struct stats_metrics_iter *iter = + stats_metrics_iterate_init(stats_metrics); + const struct metric *metric; + while((metric = stats_metrics_iterate(iter)) != NULL) + if (strcmp(metric->name, metric_name) == 0) + break; + + /* bug in test if not found */ + i_assert(metric != NULL); + + stats_metrics_iterate_deinit(&iter); + + switch(field) { + case STATS_DIST_COUNT: + return stats_dist_get_count(metric->duration_stats); + case STATS_DIST_SUM: + return stats_dist_get_sum(metric->duration_stats); + default: + i_unreached(); + } +} + diff --git a/src/stats/test-stats-common.h b/src/stats/test-stats-common.h new file mode 100644 index 0000000..86c03f4 --- /dev/null +++ b/src/stats/test-stats-common.h @@ -0,0 +1,36 @@ +#ifndef TEST_STATS_COMMON +#define TEST_STATS_COMMON 1 + +#include "stats-common.h" +#include "event-filter.h" +#include "istream.h" +#include "settings-parser.h" +#include "str.h" +#include "test-common.h" +#include "lib-event-private.h" +#include "stats-dist.h" +#include "stats-event-category.h" +#include "stats-metrics.h" + +extern struct event_category test_category; +extern struct event_category child_test_category; +extern pool_t test_pool; + +bool test_stats_callback(struct event *event, + enum event_callback_type type ATTR_UNUSED, + struct failure_context *ctx, const char *fmt ATTR_UNUSED, + va_list args ATTR_UNUSED); + +void test_init(const char *settings_blob); +void test_deinit(void); + +void test_event_send(struct event *event); + +enum stats_dist_field { + STATS_DIST_COUNT, + STATS_DIST_SUM, +}; + +uint64_t get_stats_dist_field(const char *metric_name, enum stats_dist_field field); + +#endif diff --git a/src/stats/test-stats-metrics.c b/src/stats/test-stats-metrics.c new file mode 100644 index 0000000..c384014 --- /dev/null +++ b/src/stats/test-stats-metrics.c @@ -0,0 +1,449 @@ +/* Copyright (c) 2019 Dovecot authors, see the included COPYING file */ + +#include "test-stats-common.h" +#include "array.h" + +bool test_stats_callback(struct event *event, + enum event_callback_type type ATTR_UNUSED, + struct failure_context *ctx, const char *fmt ATTR_UNUSED, + va_list args ATTR_UNUSED) +{ + if (stats_metrics != NULL) { + stats_metrics_event(stats_metrics, event, ctx); + struct event_filter *filter = + stats_metrics_get_event_filter(stats_metrics); + return !event_filter_match(filter, event, ctx); + } + return TRUE; +} + +static const char *settings_blob_1 = +"metric=test\n" +"metric/test/metric_name=test\n" +"metric/test/filter=event=test\n" +"\n"; + +static void test_stats_metrics(void) +{ + test_begin("stats metrics (event counting)"); + + /* register some stats */ + test_init(settings_blob_1); + + /* push event in */ + struct event *event = event_create(NULL); + event_add_category(event, &test_category); + event_set_name(event, "test"); + test_event_send(event); + event_unref(&event); + + test_assert(get_stats_dist_field("test", STATS_DIST_COUNT) == 1); + test_assert(get_stats_dist_field("test", STATS_DIST_SUM) > 0); + + test_deinit(); + test_end(); +} + +static const char *settings_blob_2 = +"metric=test\n" +"metric/test/metric_name=test\n" +"metric/test/filter=(event=test AND test_field=value)\n" +"\n"; + +static void test_stats_metrics_filter(void) +{ + test_begin("stats metrics (filter)"); + + test_init(settings_blob_2); + + /* check filter */ + struct event_filter *filter = + stats_metrics_get_event_filter(stats_metrics); + string_t *str_filter = t_str_new(64); + event_filter_export(filter, str_filter); + test_assert_strcmp("((event=\"test\" AND \"test_field\"=\"value\"))", + str_c(str_filter)); + + /* send event */ + struct event *event = event_create(NULL); + event_add_category(event, &test_category); + event_set_name(event, "test"); + event_add_str(event, "test_field", "value"); + test_event_send(event); + event_unref(&event); + + test_assert(get_stats_dist_field("test", STATS_DIST_COUNT) == 1); + test_assert(get_stats_dist_field("test", STATS_DIST_SUM) > 0); + + /* send another event */ + event = event_create(NULL); + event_add_category(event, &test_category); + event_set_name(event, "test"); + event_add_str(event, "test_field", "nother value"); + e_debug(event, "test"); + event_unref(&event); + + test_assert(get_stats_dist_field("test", STATS_DIST_COUNT) == 1); + test_assert(get_stats_dist_field("test", STATS_DIST_SUM) > 0); + + test_deinit(); + test_end(); +} + +static void test_stats_metrics_group_by_check_one(const struct metric *metric, + const char *sub_name, + unsigned int total_count, + unsigned int submetric_count, + unsigned int group_by_count, + enum stats_metric_group_by_func group_by_func, + const char *group_by_field, + enum metric_value_type value_type) +{ + test_assert_strcmp(metric->name, "test"); + + if (sub_name != NULL) + test_assert_strcmp(metric->sub_name, sub_name); + else + test_assert(metric->sub_name == NULL); + + test_assert(stats_dist_get_count(metric->duration_stats) == total_count); + + if (submetric_count > 0) { + test_assert(array_is_created(&metric->sub_metrics)); + test_assert(array_count(&metric->sub_metrics) == submetric_count); + } else { + test_assert(!array_is_created(&metric->sub_metrics)); + } + + if (group_by_count > 0) { + test_assert(metric->group_by_count == group_by_count); + i_assert(metric->group_by != NULL); + test_assert(metric->group_by[0].func == group_by_func); + test_assert_strcmp(metric->group_by[0].field, group_by_field); + } else { + test_assert(metric->group_by_count == 0); + test_assert(metric->group_by == NULL); + } + + test_assert(metric->group_value.type == value_type); +} + +#define DISCRETE_TEST_VAL_COUNT 3 +struct discrete_test { + const char *settings_blob; + unsigned int num_values; + const char *values_first[DISCRETE_TEST_VAL_COUNT]; + const char *values_second[DISCRETE_TEST_VAL_COUNT]; +}; + +static const struct discrete_test discrete_tests[] = { + { + "test_name sub_name", + 3, + { "eta", "kappa", "nu", }, + { "upsilon", "pi", "epsilon", }, + }, + { + "test_name:discrete sub_name:discrete", + 3, + { "apple", "bannana", "orange", }, + { "pie", "yoghurt", "cobbler", }, + }, + { + "test_name sub_name:discrete", + 3, + { "apollo", "gaia", "hermes", }, + { "thor", "odin", "loki", }, + }, +}; + +static void test_stats_metrics_group_by_discrete_real(const struct discrete_test *test) +{ + struct event *event; + unsigned int i, j; + + test_begin(t_strdup_printf("stats metrics (discrete group by) - %s", + test->settings_blob)); + + test_init(t_strdup_printf("metric=test\n" + "metric/test/metric_name=test\n" + "metric/test/filter=event=test\n" + "metric/test/group_by=%s\n" + "\n", test->settings_blob)); + + for (i = 0; i < test->num_values; i++) { + for (j = 0; j < test->num_values; j++) { + event = event_create(NULL); + event_add_category(event, &test_category); + event_set_name(event, "test"); + event_add_str(event, "test_name", test->values_first[i]); + event_add_str(event, "sub_name", test->values_second[j]); + test_event_send(event); + event_unref(&event); + } + } + + /* check total number of events */ + test_assert(get_stats_dist_field("test", STATS_DIST_COUNT) == test->num_values * test->num_values); + + /* analyze the structure */ + struct stats_metrics_iter *iter = stats_metrics_iterate_init(stats_metrics); + const struct metric *root_metric = stats_metrics_iterate(iter); + stats_metrics_iterate_deinit(&iter); + + test_stats_metrics_group_by_check_one(root_metric, + NULL, + test->num_values * test->num_values, + test->num_values, + 2, STATS_METRIC_GROUPBY_DISCRETE, + "test_name", 0); + + struct metric *const *first = array_idx(&root_metric->sub_metrics, 0); + + /* examime each sub-metric */ + for (i = 0; i < test->num_values; i++) { + test_stats_metrics_group_by_check_one(first[i], + test->values_first[i], + test->num_values, + test->num_values, + 1, STATS_METRIC_GROUPBY_DISCRETE, + "sub_name", + METRIC_VALUE_TYPE_STR); + + struct metric *const *second = array_idx(&first[i]->sub_metrics, 0); + + /* examine each sub-sub-metric */ + for (j = 0; j < test->num_values; j++) { + test_stats_metrics_group_by_check_one(second[j], + test->values_second[j], + 1, 0, 0, 0, NULL, + METRIC_VALUE_TYPE_STR); + } + } + + test_deinit(); + test_end(); +} + +static void test_stats_metrics_group_by_discrete(void) +{ + unsigned int i; + + for (i = 0; i < N_ELEMENTS(discrete_tests); i++) + test_stats_metrics_group_by_discrete_real(&discrete_tests[i]); +} + +#define QUANTIZED_TEST_VAL_COUNT 15 +struct quantized_test { + const char *settings_blob; + unsigned int num_inputs; + intmax_t input_vals[QUANTIZED_TEST_VAL_COUNT]; + + unsigned int num_sub_metrics; + + unsigned int num_ranges; + struct { + struct stats_metric_settings_bucket_range range; + intmax_t count; + } ranges[QUANTIZED_TEST_VAL_COUNT]; +}; + +static const struct quantized_test quantized_tests[] = { + { + "linear:100:1000:100", + 13, + { 0, 50, 100, 101, 200, 201, 250, 301, 900, 901, 1000, 1001, 2000 }, + 7, + 11, + { { { INTMAX_MIN, 100 }, 3 }, + { { 100, 200 }, 2 }, + { { 200, 300 }, 2 }, + { { 300, 400 }, 1 }, + { { 400, 500 }, 0 }, + { { 500, 600 }, 0 }, + { { 600, 700 }, 0 }, + { { 700, 800 }, 0 }, + { { 800, 900 }, 1 }, + { { 900, 1000 }, 2 }, + { { 1000, INTMAX_MAX }, 2 }, + } + }, + { + /* start at 0 */ + "exponential:0:6:10", + 12, + { 0, 5, 10, 11, 100, 101, 500, 1000, 1001, 1000000, 1000001, 2000000 }, + 7, + 8, + { { { INTMAX_MIN, 1 }, 1 }, + { { 1, 10 }, 2 }, + { { 10, 100 }, 2 }, + { { 100, 1000 }, 3 }, + { { 1000, 10000 }, 1 }, + { { 10000, 100000 }, 0 }, + { { 100000, 1000000 }, 1 }, + { { 1000000, INTMAX_MAX }, 2 }, + } + }, + { + /* start at 0 */ + "exponential:0:6:2", + 9, + { 0, 1, 2, 4, 5, 20, 64, 65, 100 }, + 7, + 8, + { { { INTMAX_MIN, 1 }, 2 }, + { { 1, 2 }, 1 }, + { { 2, 4 }, 1 }, + { { 4, 8 }, 1 }, + { { 8, 16 }, 0 }, + { { 16, 32 }, 1 }, + { { 32, 64 }, 1 }, + { { 64, INTMAX_MAX }, 2 }, + } + }, + { + /* start at >0 */ + "exponential:2:6:10", + 12, + { 0, 5, 10, 11, 100, 101, 500, 1000, 1001, 1000000, 1000001, 2000000 }, + 5, + 6, + { { { INTMAX_MIN, 100 }, 5 }, + { { 100, 1000 }, 3 }, + { { 1000, 10000 }, 1 }, + { { 10000, 100000 }, 0 }, + { { 100000, 1000000 }, 1 }, + { { 1000000, INTMAX_MAX }, 2 }, + } + }, + { + /* start at >0 */ + "exponential:2:6:2", + 9, + { 0, 1, 2, 4, 5, 20, 64, 65, 100 }, + 5, + 6, + { { { INTMAX_MIN, 4 }, 4 }, + { { 4, 8 }, 1 }, + { { 8, 16 }, 0 }, + { { 16, 32 }, 1 }, + { { 32, 64 }, 1 }, + { { 64, INTMAX_MAX }, 2 }, + } + }, +}; + +static void test_stats_metrics_group_by_quantized_real(const struct quantized_test *test) +{ + unsigned int i; + + test_begin(t_strdup_printf("stats metrics (quantized group by) - %s", + test->settings_blob)); + + test_init(t_strdup_printf("metric=test\n" + "metric/test/metric_name=test\n" + "metric/test/filter=event=test\n" + "metric/test/group_by=test_name foobar:%s\n" + "\n", test->settings_blob)); + + struct event *event; + + for (i = 0; i < test->num_inputs; i++) { + event = event_create(NULL); + event_add_category(event, &test_category); + event_set_name(event, "test"); + event_add_str(event, "test_name", "alpha"); + event_add_int(event, "foobar", test->input_vals[i]); + test_event_send(event); + event_unref(&event); + } + + /* check total number of events */ + test_assert(get_stats_dist_field("test", STATS_DIST_COUNT) == test->num_inputs); + + /* analyze the structure */ + struct stats_metrics_iter *iter = stats_metrics_iterate_init(stats_metrics); + const struct metric *root_metric = stats_metrics_iterate(iter); + stats_metrics_iterate_deinit(&iter); + + test_stats_metrics_group_by_check_one(root_metric, NULL, test->num_inputs, + 1, 2, STATS_METRIC_GROUPBY_DISCRETE, + "test_name", 0); + + /* examine first level sub-metric */ + struct metric *const *first = array_idx(&root_metric->sub_metrics, 0); + test_stats_metrics_group_by_check_one(first[0], + "alpha", + test->num_inputs, + test->num_sub_metrics, + 1, + STATS_METRIC_GROUPBY_QUANTIZED, + "foobar", + METRIC_VALUE_TYPE_STR); + + /* check the ranges */ + test_assert(first[0]->group_by[0].num_ranges == test->num_ranges); + for (i = 0; i < test->num_ranges; i++) { + test_assert(first[0]->group_by[0].ranges[i].min == test->ranges[i].range.min); + test_assert(first[0]->group_by[0].ranges[i].max == test->ranges[i].range.max); + } + + /* examine second level sub-metrics */ + struct metric *const *second = array_idx(&first[0]->sub_metrics, 0); + + for (i = 0; i < test->num_sub_metrics; i++) { + const char *sub_name; + intmax_t range_idx; + + /* we check these first, before we use the value below */ + test_assert(second[i]->group_value.type == METRIC_VALUE_TYPE_BUCKET_INDEX); + test_assert(second[i]->group_value.intmax < test->num_ranges); + + range_idx = second[i]->group_value.intmax; + + /* construct the expected sub-metric name */ + if (test->ranges[range_idx].range.min == INTMAX_MIN) { + sub_name = t_strdup_printf("foobar_ninf_%jd", + test->ranges[range_idx].range.max); + } else if (test->ranges[range_idx].range.max == INTMAX_MAX) { + sub_name = t_strdup_printf("foobar_%jd_inf", + test->ranges[range_idx].range.min + 1); + } else { + sub_name = t_strdup_printf("foobar_%jd_%jd", + test->ranges[range_idx].range.min + 1, + test->ranges[range_idx].range.max); + } + + test_stats_metrics_group_by_check_one(second[i], + sub_name, + test->ranges[second[i]->group_value.intmax].count, + 0, 0, 0, NULL, + METRIC_VALUE_TYPE_BUCKET_INDEX); + } + + test_deinit(); + test_end(); +} + +static void test_stats_metrics_group_by_quantized(void) +{ + unsigned int i; + + for (i = 0; i < N_ELEMENTS(quantized_tests); i++) + test_stats_metrics_group_by_quantized_real(&quantized_tests[i]); +} + +int main(void) { + void (*const test_functions[])(void) = { + test_stats_metrics, + test_stats_metrics_filter, + test_stats_metrics_group_by_discrete, + test_stats_metrics_group_by_quantized, + NULL + }; + + int ret = test_run(test_functions); + + return ret; +} |