diff options
Diffstat (limited to '')
| -rw-r--r-- | collectors/statsd.plugin/Makefile.am | 20 | ||||
| -rw-r--r-- | collectors/statsd.plugin/Makefile.in (renamed from node.d/Makefile.in) | 149 | ||||
| -rw-r--r-- | collectors/statsd.plugin/README.md | 523 | ||||
| -rw-r--r-- | collectors/statsd.plugin/example.conf (renamed from conf.d/statsd.d/example.conf) | 23 | ||||
| -rw-r--r-- | collectors/statsd.plugin/statsd.c (renamed from src/statsd.c) | 373 | ||||
| -rw-r--r-- | collectors/statsd.plugin/statsd.h | 25 |
6 files changed, 855 insertions, 258 deletions
diff --git a/collectors/statsd.plugin/Makefile.am b/collectors/statsd.plugin/Makefile.am new file mode 100644 index 000000000..7f09bacd7 --- /dev/null +++ b/collectors/statsd.plugin/Makefile.am @@ -0,0 +1,20 @@ +# SPDX-License-Identifier: GPL-3.0-or-later + +AUTOMAKE_OPTIONS = subdir-objects +MAINTAINERCLEANFILES = $(srcdir)/Makefile.in + +dist_noinst_DATA = \ + README.md \ + $(NULL) + +statsdconfigdir=$(libconfigdir)/statsd.d +dist_statsdconfig_DATA = \ + $(top_srcdir)/installer/.keep \ + example.conf \ + $(NULL) + +userstatsdconfigdir=$(configdir)/statsd.d +dist_userstatsdconfig_DATA = \ + $(top_srcdir)/installer/.keep \ + $(NULL) + diff --git a/node.d/Makefile.in b/collectors/statsd.plugin/Makefile.in index dd572ee8b..5c16a86d1 100644 --- a/node.d/Makefile.in +++ b/collectors/statsd.plugin/Makefile.in @@ -14,6 +14,8 @@ @SET_MAKE@ +# SPDX-License-Identifier: GPL-3.0-or-later + VPATH = @srcdir@ am__is_gnu_make = test -n '$(MAKEFILE_LIST)' && test -n '$(MAKELEVEL)' am__make_running_with_option = \ @@ -78,19 +80,21 @@ PRE_UNINSTALL = : POST_UNINSTALL = : build_triplet = @build@ host_triplet = @host@ -subdir = node.d +subdir = collectors/statsd.plugin DIST_COMMON = $(srcdir)/Makefile.in $(srcdir)/Makefile.am \ - $(dist_node_DATA) $(dist_nodemodules_DATA) \ - $(dist_nodemoduleslibber_DATA) + $(dist_noinst_DATA) $(dist_statsdconfig_DATA) \ + $(dist_userstatsdconfig_DATA) ACLOCAL_M4 = $(top_srcdir)/aclocal.m4 -am__aclocal_m4_deps = $(top_srcdir)/m4/ax_c___atomic.m4 \ - $(top_srcdir)/m4/ax_c__generic.m4 $(top_srcdir)/m4/ax_c_lto.m4 \ - $(top_srcdir)/m4/ax_c_mallinfo.m4 \ - $(top_srcdir)/m4/ax_c_mallopt.m4 \ - $(top_srcdir)/m4/ax_check_compile_flag.m4 \ - $(top_srcdir)/m4/ax_gcc_func_attribute.m4 \ - $(top_srcdir)/m4/ax_pthread.m4 $(top_srcdir)/m4/jemalloc.m4 \ - $(top_srcdir)/m4/tcmalloc.m4 $(top_srcdir)/configure.ac +am__aclocal_m4_deps = $(top_srcdir)/build/m4/ax_c___atomic.m4 \ + $(top_srcdir)/build/m4/ax_c__generic.m4 \ + $(top_srcdir)/build/m4/ax_c_lto.m4 \ + $(top_srcdir)/build/m4/ax_c_mallinfo.m4 \ + $(top_srcdir)/build/m4/ax_c_mallopt.m4 \ + $(top_srcdir)/build/m4/ax_check_compile_flag.m4 \ + $(top_srcdir)/build/m4/ax_gcc_func_attribute.m4 \ + $(top_srcdir)/build/m4/ax_pthread.m4 \ + $(top_srcdir)/build/m4/jemalloc.m4 \ + $(top_srcdir)/build/m4/tcmalloc.m4 $(top_srcdir)/configure.ac am__configure_deps = $(am__aclocal_m4_deps) $(CONFIGURE_DEPENDENCIES) \ $(ACLOCAL_M4) mkinstalldirs = $(install_sh) -d @@ -143,10 +147,10 @@ am__uninstall_files_from_dir = { \ || { echo " ( cd '$$dir' && rm -f" $$files ")"; \ $(am__cd) "$$dir" && rm -f $$files; }; \ } -am__installdirs = "$(DESTDIR)$(nodedir)" "$(DESTDIR)$(nodemodulesdir)" \ - "$(DESTDIR)$(nodemoduleslibberdir)" -DATA = $(dist_node_DATA) $(dist_nodemodules_DATA) \ - $(dist_nodemoduleslibber_DATA) +am__installdirs = "$(DESTDIR)$(statsdconfigdir)" \ + "$(DESTDIR)$(userstatsdconfigdir)" +DATA = $(dist_noinst_DATA) $(dist_statsdconfig_DATA) \ + $(dist_userstatsdconfig_DATA) am__tagged_files = $(HEADERS) $(SOURCES) $(TAGS_FILES) $(LISP) DISTFILES = $(DIST_COMMON) $(DIST_SOURCES) $(TEXINFOS) $(EXTRA_DIST) ACLOCAL = @ACLOCAL@ @@ -246,6 +250,7 @@ build = @build@ build_alias = @build_alias@ build_cpu = @build_cpu@ build_os = @build_os@ +build_target = @build_target@ build_vendor = @build_vendor@ builddir = @builddir@ cachedir = @cachedir@ @@ -267,6 +272,7 @@ htmldir = @htmldir@ includedir = @includedir@ infodir = @infodir@ install_sh = @install_sh@ +libconfigdir = @libconfigdir@ libdir = @libdir@ libexecdir = @libexecdir@ localedir = @localedir@ @@ -293,33 +299,22 @@ top_builddir = @top_builddir@ top_srcdir = @top_srcdir@ varlibdir = @varlibdir@ webdir = @webdir@ +AUTOMAKE_OPTIONS = subdir-objects MAINTAINERCLEANFILES = $(srcdir)/Makefile.in -dist_node_DATA = \ +dist_noinst_DATA = \ README.md \ - named.node.js \ - fronius.node.js \ - sma_webbox.node.js \ - snmp.node.js \ - stiebeleltron.node.js \ $(NULL) -nodemodulesdir = $(nodedir)/node_modules -dist_nodemodules_DATA = \ - node_modules/netdata.js \ - node_modules/extend.js \ - node_modules/pixl-xml.js \ - node_modules/net-snmp.js \ - node_modules/asn1-ber.js \ - $(NULL) +statsdconfigdir = $(libconfigdir)/statsd.d +dist_statsdconfig_DATA = \ + $(top_srcdir)/installer/.keep \ + example.conf \ + $(NULL) -nodemoduleslibberdir = $(nodedir)/node_modules/lib/ber -dist_nodemoduleslibber_DATA = \ - node_modules/lib/ber/index.js \ - node_modules/lib/ber/errors.js \ - node_modules/lib/ber/reader.js \ - node_modules/lib/ber/types.js \ - node_modules/lib/ber/writer.js \ - $(NULL) +userstatsdconfigdir = $(configdir)/statsd.d +dist_userstatsdconfig_DATA = \ + $(top_srcdir)/installer/.keep \ + $(NULL) all: all-am @@ -333,9 +328,9 @@ $(srcdir)/Makefile.in: @MAINTAINER_MODE_TRUE@ $(srcdir)/Makefile.am $(am__confi exit 1;; \ esac; \ done; \ - echo ' cd $(top_srcdir) && $(AUTOMAKE) --gnu node.d/Makefile'; \ + echo ' cd $(top_srcdir) && $(AUTOMAKE) --gnu collectors/statsd.plugin/Makefile'; \ $(am__cd) $(top_srcdir) && \ - $(AUTOMAKE) --gnu node.d/Makefile + $(AUTOMAKE) --gnu collectors/statsd.plugin/Makefile .PRECIOUS: Makefile Makefile: $(srcdir)/Makefile.in $(top_builddir)/config.status @case '$?' in \ @@ -354,69 +349,48 @@ $(top_srcdir)/configure: @MAINTAINER_MODE_TRUE@ $(am__configure_deps) $(ACLOCAL_M4): @MAINTAINER_MODE_TRUE@ $(am__aclocal_m4_deps) cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh $(am__aclocal_m4_deps): -install-dist_nodeDATA: $(dist_node_DATA) - @$(NORMAL_INSTALL) - @list='$(dist_node_DATA)'; test -n "$(nodedir)" || list=; \ - if test -n "$$list"; then \ - echo " $(MKDIR_P) '$(DESTDIR)$(nodedir)'"; \ - $(MKDIR_P) "$(DESTDIR)$(nodedir)" || exit 1; \ - fi; \ - for p in $$list; do \ - if test -f "$$p"; then d=; else d="$(srcdir)/"; fi; \ - echo "$$d$$p"; \ - done | $(am__base_list) | \ - while read files; do \ - echo " $(INSTALL_DATA) $$files '$(DESTDIR)$(nodedir)'"; \ - $(INSTALL_DATA) $$files "$(DESTDIR)$(nodedir)" || exit $$?; \ - done - -uninstall-dist_nodeDATA: - @$(NORMAL_UNINSTALL) - @list='$(dist_node_DATA)'; test -n "$(nodedir)" || list=; \ - files=`for p in $$list; do echo $$p; done | sed -e 's|^.*/||'`; \ - dir='$(DESTDIR)$(nodedir)'; $(am__uninstall_files_from_dir) -install-dist_nodemodulesDATA: $(dist_nodemodules_DATA) +install-dist_statsdconfigDATA: $(dist_statsdconfig_DATA) @$(NORMAL_INSTALL) - @list='$(dist_nodemodules_DATA)'; test -n "$(nodemodulesdir)" || list=; \ + @list='$(dist_statsdconfig_DATA)'; test -n "$(statsdconfigdir)" || list=; \ if test -n "$$list"; then \ - echo " $(MKDIR_P) '$(DESTDIR)$(nodemodulesdir)'"; \ - $(MKDIR_P) "$(DESTDIR)$(nodemodulesdir)" || exit 1; \ + echo " $(MKDIR_P) '$(DESTDIR)$(statsdconfigdir)'"; \ + $(MKDIR_P) "$(DESTDIR)$(statsdconfigdir)" || exit 1; \ fi; \ for p in $$list; do \ if test -f "$$p"; then d=; else d="$(srcdir)/"; fi; \ echo "$$d$$p"; \ done | $(am__base_list) | \ while read files; do \ - echo " $(INSTALL_DATA) $$files '$(DESTDIR)$(nodemodulesdir)'"; \ - $(INSTALL_DATA) $$files "$(DESTDIR)$(nodemodulesdir)" || exit $$?; \ + echo " $(INSTALL_DATA) $$files '$(DESTDIR)$(statsdconfigdir)'"; \ + $(INSTALL_DATA) $$files "$(DESTDIR)$(statsdconfigdir)" || exit $$?; \ done -uninstall-dist_nodemodulesDATA: +uninstall-dist_statsdconfigDATA: @$(NORMAL_UNINSTALL) - @list='$(dist_nodemodules_DATA)'; test -n "$(nodemodulesdir)" || list=; \ + @list='$(dist_statsdconfig_DATA)'; test -n "$(statsdconfigdir)" || list=; \ files=`for p in $$list; do echo $$p; done | sed -e 's|^.*/||'`; \ - dir='$(DESTDIR)$(nodemodulesdir)'; $(am__uninstall_files_from_dir) -install-dist_nodemoduleslibberDATA: $(dist_nodemoduleslibber_DATA) + dir='$(DESTDIR)$(statsdconfigdir)'; $(am__uninstall_files_from_dir) +install-dist_userstatsdconfigDATA: $(dist_userstatsdconfig_DATA) @$(NORMAL_INSTALL) - @list='$(dist_nodemoduleslibber_DATA)'; test -n "$(nodemoduleslibberdir)" || list=; \ + @list='$(dist_userstatsdconfig_DATA)'; test -n "$(userstatsdconfigdir)" || list=; \ if test -n "$$list"; then \ - echo " $(MKDIR_P) '$(DESTDIR)$(nodemoduleslibberdir)'"; \ - $(MKDIR_P) "$(DESTDIR)$(nodemoduleslibberdir)" || exit 1; \ + echo " $(MKDIR_P) '$(DESTDIR)$(userstatsdconfigdir)'"; \ + $(MKDIR_P) "$(DESTDIR)$(userstatsdconfigdir)" || exit 1; \ fi; \ for p in $$list; do \ if test -f "$$p"; then d=; else d="$(srcdir)/"; fi; \ echo "$$d$$p"; \ done | $(am__base_list) | \ while read files; do \ - echo " $(INSTALL_DATA) $$files '$(DESTDIR)$(nodemoduleslibberdir)'"; \ - $(INSTALL_DATA) $$files "$(DESTDIR)$(nodemoduleslibberdir)" || exit $$?; \ + echo " $(INSTALL_DATA) $$files '$(DESTDIR)$(userstatsdconfigdir)'"; \ + $(INSTALL_DATA) $$files "$(DESTDIR)$(userstatsdconfigdir)" || exit $$?; \ done -uninstall-dist_nodemoduleslibberDATA: +uninstall-dist_userstatsdconfigDATA: @$(NORMAL_UNINSTALL) - @list='$(dist_nodemoduleslibber_DATA)'; test -n "$(nodemoduleslibberdir)" || list=; \ + @list='$(dist_userstatsdconfig_DATA)'; test -n "$(userstatsdconfigdir)" || list=; \ files=`for p in $$list; do echo $$p; done | sed -e 's|^.*/||'`; \ - dir='$(DESTDIR)$(nodemoduleslibberdir)'; $(am__uninstall_files_from_dir) + dir='$(DESTDIR)$(userstatsdconfigdir)'; $(am__uninstall_files_from_dir) tags TAGS: ctags CTAGS: @@ -458,7 +432,7 @@ check-am: all-am check: check-am all-am: Makefile $(DATA) installdirs: - for dir in "$(DESTDIR)$(nodedir)" "$(DESTDIR)$(nodemodulesdir)" "$(DESTDIR)$(nodemoduleslibberdir)"; do \ + for dir in "$(DESTDIR)$(statsdconfigdir)" "$(DESTDIR)$(userstatsdconfigdir)"; do \ test -z "$$dir" || $(MKDIR_P) "$$dir"; \ done install: install-am @@ -512,8 +486,8 @@ info: info-am info-am: -install-data-am: install-dist_nodeDATA install-dist_nodemodulesDATA \ - install-dist_nodemoduleslibberDATA +install-data-am: install-dist_statsdconfigDATA \ + install-dist_userstatsdconfigDATA install-dvi: install-dvi-am @@ -557,25 +531,24 @@ ps: ps-am ps-am: -uninstall-am: uninstall-dist_nodeDATA uninstall-dist_nodemodulesDATA \ - uninstall-dist_nodemoduleslibberDATA +uninstall-am: uninstall-dist_statsdconfigDATA \ + uninstall-dist_userstatsdconfigDATA .MAKE: install-am install-strip .PHONY: all all-am check check-am clean clean-generic cscopelist-am \ ctags-am distclean distclean-generic distdir dvi dvi-am html \ html-am info info-am install install-am install-data \ - install-data-am install-dist_nodeDATA \ - install-dist_nodemodulesDATA \ - install-dist_nodemoduleslibberDATA install-dvi install-dvi-am \ + install-data-am install-dist_statsdconfigDATA \ + install-dist_userstatsdconfigDATA install-dvi install-dvi-am \ install-exec install-exec-am install-html install-html-am \ install-info install-info-am install-man install-pdf \ install-pdf-am install-ps install-ps-am install-strip \ installcheck installcheck-am installdirs maintainer-clean \ maintainer-clean-generic mostlyclean mostlyclean-generic pdf \ pdf-am ps ps-am tags-am uninstall uninstall-am \ - uninstall-dist_nodeDATA uninstall-dist_nodemodulesDATA \ - uninstall-dist_nodemoduleslibberDATA + uninstall-dist_statsdconfigDATA \ + uninstall-dist_userstatsdconfigDATA # Tell versions [3.59,3.63) of GNU make to not export all variables. diff --git a/collectors/statsd.plugin/README.md b/collectors/statsd.plugin/README.md new file mode 100644 index 000000000..6ef038343 --- /dev/null +++ b/collectors/statsd.plugin/README.md @@ -0,0 +1,523 @@ +# Netdata Statsd + +statsd is a system to collect data from any application. Applications are sending metrics to it, usually via non-blocking UDP communication, and statsd servers collect these metrics, perform a few simple calculations on them and push them to backend time-series databases. + +There is a [plethora of client libraries](https://github.com/etsy/statsd/wiki#client-implementations) for embedding statsd metrics to any application framework. This makes statsd quite popular for custom application metrics. + +## netdata statsd + +netdata is a fully featured statsd server. It can collect statsd formatted metrics, visualize them on its dashboards, stream them to other netdata servers or archive them to backend time-series databases. + +netdata statsd is inside netdata (an internal plugin, running inside the netdata daemon), it is configured via `netdata.conf` and by-default listens on standard statsd ports (tcp and udp 8125 - yes, netdata statsd server supports both tcp and udp at the same time). + +Since statsd is embedded in netdata, it means you now have a statsd server embedded on all your servers. So, the application can send its metrics to `localhost:8125`. This provides a distributed statsd implementation. + +netdata statsd is fast. It can collect more than **1.200.000 metrics per second** on modern hardware, more than **200Mbps of sustained statsd traffic**, using 1 CPU core (yes, it is single threaded - actually double-threaded, one thread collects metrics, another one updates the charts from the collected data). + +## metrics supported by netdata + +netdata fully supports the statsd protocol. All statsd client libraries can be used with netdata too. + +- **Gauges** + + The application sends `name:value|g`, where `value` is any **decimal/fractional** number, statsd reports the latest value collected and the number of times it was updated (events). + + The application may increment or decrement a previous value, by setting the first character of the value to ` + ` or ` - ` (so, the only way to set a gauge to an absolute negative value, is to first set it to zero). + + Sampling rate is supported (check below). + + When a gauge is not collected and the setting is not to show gaps on the charts (the default), the last value will be shown, until a data collection event changes it. + +- **Counters** and **Meters** + + The application sends `name:value|c`, `name:value|C` or `name:value|m`, where `value` is a positive or negative **integer** number of events occurred, statsd reports the **rate** and the number of times it was updated (events). + + `:value` can be omitted and statsd will assume it is `1`. `|c`, `|C` and `|m` can be omitted an statsd will assume it is `|m`. So, the application may send just `name` and statsd will parse it as `name:1|m`. + + For counters use `|c` (esty/statsd compatible) or `|C` (brubeck compatible), for meters use `|m`. + + Sampling rate is supported (check below). + + When a counter or meter is not collected and the setting is not to show gaps on the charts (the default), zero will be shown, until a data collection event changes it. + +- **Timers** and **Histograms** + + The application sends `name:value|ms` or `name:value|h`, where ` value` is any **decimal/fractional** number, statsd reports **min**, **max**, **average**, **sum**, **95th percentile**, **median** and **standard deviation** and the total number of times it was updated (events). + + For timers use `|ms`, or histograms use `|h`. The only difference between the two, is the `units` of the charts (timers report milliseconds). + + Sampling rate is supported (check below). + + When a timer or histogram is not collected and the setting is not to show gaps on the charts (the default), zero will be shown, until a data collection event changes it. + +- **Sets** + + The application sends `name:value|s`, where `value` is anything (**number or text**, leading and trailing spaces are removed), statsd reports the number of unique values sent and the number of times it was updated (events). + + Sampling rate is **not** supported for Sets. `value` is always considered text. + + When a set is not collected and the setting is not to show gaps on the charts (the default), zero will be shown, until a data collection event changes it. + +#### Sampling Rates + +The application may append `|@sampling_rate`, where `sampling_rate` is a number from `0.0` to `1.0`, to have statsd extrapolate the value, to predict to total for the whole period. So, if the application reports to statsd a value for 1/10th of the time, it can append `|@0.1` to the metrics it sends to statsd. + +#### Overlapping metrics + +netdata statsd maintains different indexes for each of the types supported. This means the same metric `name` may exist under different types concurrently. + +#### Multiple metrics per packet + +netdata accepts multiple metrics per packet if each is terminated with `\n`. + +#### TCP packets + +netdata listens for both TCP and UDP packets. For TCP though, is it important to always append `\n` on each metric. netdata uses this to detect if a metric is split into multiple TCP packets. On disconnect, even the remaining (non terminated with `\n`) buffer, is processed. + +#### UDP packets + +When sending multiple packets over UDP, it is important not to exceed the network MTU (usually 1500 bytes minus a few bytes for the headers). netdata will accept UDP packets up to 9000 bytes, but the underlying network will not exceed MTU. + +## configuration + +This is the statsd configuration at `/etc/netdata/netdata.conf`: + +``` +[statsd] + # enabled = yes + # decimal detail = 1000 + # update every (flushInterval) = 1 + # udp messages to process at once = 10 + # create private charts for metrics matching = * + # max private charts allowed = 200 + # max private charts hard limit = 1000 + # private charts memory mode = save + # private charts history = 3996 + # histograms and timers percentile (percentThreshold) = 95.00000 + # add dimension for number of events received = yes + # gaps on gauges (deleteGauges) = no + # gaps on counters (deleteCounters) = no + # gaps on meters (deleteMeters) = no + # gaps on sets (deleteSets) = no + # gaps on histograms (deleteHistograms) = no + # gaps on timers (deleteTimers) = no + # listen backlog = 4096 + # default port = 8125 + # bind to = udp:localhost:8125 tcp:localhost:8125 +``` + +### statsd main config options +- `enabled = yes|no` + + controls if statsd will be enabled for this netdata. The default is enabled. + +- `default port = 8125` + + controls the port statsd will use. This is the default, since the next line, allows defining ports too. + +- `bind to = udp:localhost tcp:localhost` + + is a space separated list of IPs and ports to listen to. The format is `PROTOCOL:IP:PORT` - if `PORT` is omitted, the `default port` will be used. If `IP` is IPv6, it needs to be enclosed in `[]`. `IP` can also be ` * ` (to listen on all IPs) or even a hostname. + +- `update every (flushInterval) = 1` seconds, controls the frequency statsd will push the collected metrics to netdata charts. + +- `decimal detail = 1000` controls the number of fractional digits in gauges and histograms. netdata collects metrics using signed 64 bit integers and their fractional detail is controlled using multipliers and divisors. This setting is used to multiply all collected values to convert them to integers and is also set as the divisors, so that the final data will be a floating point number with this fractional detail (1000 = X.0 - X.999, 10000 = X.0 - X.9999, etc). + +The rest of the settings are discussed below. + +## statsd charts + +netdata can visualize statsd collected metrics in 2 ways: + +1. Each metric gets its own **private chart**. This is the default and does not require any configuration (although there are a few options to tweak). + +2. **Synthetic charts** can be created, combining multiple metrics, independently of their metric types. For this type of charts, special configuration is required, to define the chart title, type, units, its dimensions, etc. + +### private metric charts + +Private charts are controlled with `create private charts for metrics matching = *`. This setting accepts a space separated list of simple patterns (use `*` as wildcard, prepend a pattern with `!` for a negative match, the order of patterns is important). + +So to render charts for all `myapp.*` metrics, except `myapp.*.badmetric`, use: + +``` +create private charts for metrics matching = !myapp.*.badmetric myapp.* +``` + +The default is to render private charts for all metrics. + +The `memory mode` of the round robin database and the `history` of private metric charts are controlled with `private charts memory mode` and `private charts history`. The defaults for both settings is to use the global netdata settings. So, you need to edit them only when you want statsd to use different settings compared to the global ones. + +If you have thousands of metrics, each with its own private chart, you may notice that your web browser becomes slow when you view the netdata dashboard (this is a web browser issue we need to address at the netdata UI). So, netdata has a protection to stop creating charts when `max private charts allowed = 200` (soft limit) is reached. + +The metrics above this soft limit are still processed by netdata and will be available to be sent to backend time-series databases, up to `max private charts hard limit = 1000`. So, between 200 and 1000 charts, netdata will still generate charts, but they will automatically be created with `memory mode = none` (netdata will not maintain a database for them). These metrics will be sent to backend time series databases, if the backend configuration is set to `as collected`. + +Metrics above the hard limit are still collected, but they can only be used in synthetic charts (once a metric is added to chart, it will be sent to backend servers too). + +Example private charts (automatically generated without any configuration): + +#### counters + +- Scope: **count the events of something** (e.g. number of file downloads) +- Format: `name:INTEGER|c` or `name:INTEGER|C` or `name|c` +- statsd increments the counter by the `INTEGER` number supplied (positive, or negative). + + + +#### gauges + +- Scope: **report the value of something** (e.g. cache memory used by the application server) +- Format: `name:FLOAT|g` +- statsd remembers the last value supplied, and can increment or decrement the latest value if `FLOAT` begins with ` + ` or ` - `. + + + +#### histograms + +- Scope: **statistics on a size of events** (e.g. statistics on the sizes of files downloaded) +- Format: `name:FLOAT|h` +- statsd maintains a list of all the values supplied and provides statistics on them. + + + +The same chart with `sum` unselected, to show the detail of the dimensions supported: + + +#### meters + +This is identical to `counter`. + +- Scope: **count the events of something** (e.g. number of file downloads) +- Format: `name:INTEGER|m` or `name|m` or just `name` +- statsd increments the counter by the `INTEGER` number supplied (positive, or negative). + + + +#### sets + +- Scope: **count the unique occurrences of something** (e.g. unique filenames downloaded, or unique users that downloaded files) +- Format: `name:TEXT|s` +- statsd maintains a unique index of all values supplied, and reports the unique entries in it. + + + +#### timers + +- Scope: **statistics on the duration of events** (e.g. statistics for the duration of file downloads) +- Format: `name:FLOAT|ms` +- statsd maintains a list of all the values supplied and provides statistics on them. + + + +The same chart with the `sum` unselected: + + + + +### synthetic statsd charts + +Using synthetic charts, you can create dedicated sections on the dashboard to render the charts. You can control everything: the main menu, the submenus, the charts, the dimensions on each chart, etc. + +Synthetic charts are organized in + +- **applications** (i.e. entries at the main menu of the netdata dashboard) +- **charts for each application** (grouped in families - i.e. submenus at the dashboard menu) +- **statsd metrics for each chart** (i.e. dimensions of the charts) + +For each application you need to create a `.conf` file in `/etc/netdata/statsd.d`. + +So, to create the statsd application `myapp`, you can create the file `/etc/netdata/statsd.d/myapp.conf`, with this content: + +``` +[app] + name = myapp + metrics = myapp.* + private charts = no + gaps when not collected = no + memory mode = ram + history = 60 + +[dictionary] + m1 = metric1 + m2 = metric2 + +# replace 'mychart' with the chart id +# the chart will be named: myapp.mychart +[mychart] + name = mychart + title = my chart title + family = my family + context = chart.context + units = tests/s + priority = 91000 + type = area + dimension = myapp.metric1 m1 + dimension = myapp.metric2 m2 +``` + +Using the above configuration `myapp` should get its own section on the dashboard, having one chart with 2 dimensions. + +`[app]` starts a new application definition. The supported settings in this section are: + +- `name` defines the name of the app. +- `metrics` is a netdata simple pattern (space separated patterns, using `*` for wildcard, possibly starting with `!` for negative match). This pattern should match all the possible statsd metrics that will be participating in the application `myapp`. +- `private charts = yes|no`, enables or disables private charts for the metrics matched. +- `gaps when not collected = yes|no`, enables or disables gaps on the charts of the application, when metrics are not collected. +- `memory mode` sets the memory mode for all charts of the application. The default is the global default for netdata (not the global default for statsd private charts). +- `history` sets the size of the round robin database for this application. The default is the global default for netdata (not the global default for statsd private charts). + +`[dictionary]` defines name-value associations. These are used to renaming metrics, when added to synthetic charts. Metric names are also defined at each `dimension` line. However, using the dictionary dimension names can be declared globally, for each app and is the only way to rename dimensions when using patterns. Of course the dictionary can be empty or missing. + +Then, you can add any number of charts. Each chart should start with `[id]`. The chart will be called `app_name.id`. `family` controls the submenu on the dashboard. `context` controls the alarm templates. `priority` controls the ordering of the charts on the dashboard. The rest of the settings are informational. + +You can add any number of metrics to a chart, using `dimension` lines. These lines accept 5 space separated parameters: + +1. the metric name, as it is collected (it has to be matched by the `metrics = ` pattern of the app) +2. the dimension name, as it should be shown on the chart +3. an optional selector (type) of the value to shown (see below) +4. an optional multiplier +5. an optional divider +6. optional flags, space separated and enclosed in quotes. All the external plugins `DIMENSION` flags can be used. Currently the only usable flag is `hidden`, to add the dimension, but not show it on the dashboard. This is usually needed to have the values available for percentage calculation, or use them in alarms. + +So, the format is this: +``` +dimension = [pattern] METRIC NAME TYPE MULTIPLIER DIVIDER OPTIONS +``` + +`pattern` is a keyword. When set, `METRIC` is expected to be a netdata simple pattern that will be used to match all the statsd metrics to be added to the chart. So, `pattern` automatically matches any number of statsd metrics, all of which will be added as separate chart dimensions. + +`TYPE`, `MUTLIPLIER`, `DIVIDER` and `OPTIONS` are optional. + +`TYPE` can be: + +- `events` to show the number of events received by statsd for this metric +- `last` to show the last value, as calculated at the flush interval of the metric (the default) + +Then for histograms and timers the following types are also supported: + +- `min`, show the minimum value +- `max`, show the maximum value +- `sum`, show the sum of all values +- `average` (same as `last`) +- `percentile`, show the 95th percentile (or any other percentile, as configured at statsd global config) +- `median`, show the median of all values (i.e. sort all values and get the middle value) +- `stddev`, show the standard deviation of the values + +#### example synthetic charts + +statsd metrics: `foo` and `bar`. + +Contents of file `/etc/netdata/stats.d/foobar.conf`: + +``` +[app] + name = foobarapp + metrics = foo bar + private charts = yes + +[foobar_chart1] + title = Hey, foo and bar together + family = foobar_family + context = foobarapp.foobars + units = foobars + type = area + dimension = foo 'foo me' last 1 1 + dimension = bar 'bar me' last 1 1 +``` + +I sent to statsd: `foo:10|g` and `bar:20|g`. + +I got these private charts: + + + +and this synthetic chart: + + + +#### dictionary to name dimensions + +The `[dictionary]` section accepts any number of `name = value` pairs. + +netdata uses this dictionary as follows: + +1. When a `dimension` has a non-empty `NAME`, that name is looked up at the dictionary. + +2. If the above lookup gives nothing, or the `dimension` has an empty `NAME`, the original statsd metric name is looked up at the dictionary. + +3. If any of the above succeeds, netdata uses the `value` of the dictionary, to set the name of the dimension. The dimensions will have as ID the original statsd metric name, and as name, the dictionary value. + +So, you can use the dictionary in 2 ways: + +1. set `dimension = myapp.metric1 ''` and have at the dictionary `myapp.metric1 = metric1 name` +2. set `dimension = myapp.metric1 'm1'` and have at the dictionary `m1 = metric1 name` + +In both cases, the dimension will be added with ID `myapp.metric1` and will be named `metric1 name`. So, in alarms you can use either of the 2 as `${myapp.metric1}` or `${metric1 name}`. + +> keep in mind that if you add multiple times the same statsd metric to a chart, netdata will append `TYPE` to the dimension ID, so `myapp.metric1` will be added as `myapp.metric1_last` or `myapp.metric1_events`, etc. If you add multiple times the same metric with the same `TYPE` to a chart, netdata will also append an incremental counter to the dimension ID, i.e. `myapp.metric1_last1`, `myapp.metric1_last2`, etc. + +#### dimension patterns + +netdata allows adding multiple dimensions to a chart, by matching the statsd metrics with a netdata simple pattern. + +Assume we have an API that provides statsd metrics for each response code per method it supports, like these: + +``` +myapp.api.get.200 +myapp.api.get.400 +myapp.api.get.500 +myapp.api.del.200 +myapp.api.del.400 +myapp.api.del.500 +myapp.api.post.200 +myapp.api.post.400 +myapp.api.post.500 +myapp.api.all.200 +myapp.api.all.400 +myapp.api.all.500 +``` + +To add all response codes of `myapp.api.get` to a chart use this: + +``` +[api_get_responses] + ... + dimension = pattern 'myapp.api.get.* '' last 1 1 +``` + +The above will add dimension named `200`, `400` and `500` (yes, netdata extracts the wildcarded part of the metric name - so the dimensions will be named with whatever the `*` matched). You can rename the dimensions with this: + +``` +[dictionary] + get.200 = 200 ok + get.400 = 400 bad request + get.500 = 500 cannot connect to db + +[api_get_responses] + ... + dimension = pattern 'myapp.api.get.* 'get.' last 1 1 +``` + +Note that we added a `NAME` to the dimension line with `get.`. This is prefixed to the wildcarded part of the metric name, to compose the key for looking up the dictionary. So `500` became `get.500` which was looked up to the dictionary to find value `500 cannot connect to db`. This way we can have different dimension names, for each of the API methods (i.e. `get.500 = 500 cannot connect to db` while `post.500 = 500 cannot write to disk`). + +To add all API methods to a chart, do this: + +``` +[ok_by_method] + ... + dimension = pattern 'myapp.api.*.200 '' last 1 1 +``` + +The above will add `get`, `post`, `del` and `all` to the chart. + +If `all` is not wanted (a `stacked` chart does not need the `all` dimension, since the sum of the dimensions provides the total), the line should be: + +``` +[ok_by_method] + ... + dimension = pattern '!myapp.api.all.* myapp.api.*.200 '' last 1 1 +``` + +With the above, all methods except `all` will be added to the chart. + +To automatically rename the methods, use this: + +``` +[dictionary] + method.get = GET + method.post = ADD + method.del = DELETE + +[ok_by_method] + ... + dimension = pattern '!myapp.api.all.* myapp.api.*.200 'method.' last 1 1 +``` + +Using the above, the dimensions will be added as `GET`, `ADD` and `DELETE`. + + +## interpolation + +~~If you send just one value to statsd, you will notice that the chart is created but no value is shown. The reason is that netdata interpolates all values at second boundaries. For incremental values (`counters` and `meters` in statsd terminology), if you send 10 at 00:00:00.500, 20 at 00:00:01.500 and 30 at 00:00:02.500, netdata will show 15 at 00:00:01 and 25 at 00:00:02.~~ + +~~This interpolation is automatic and global in netdata for all charts, for incremental values. This means that for the chart to start showing values you need to send 2 values across 2 flush intervals.~~ + +~~(although this is required for incremental values, netdata allows mixing incremental and absolute values on the same charts, so this little limitation [i.e. 2 values to start visualization], is applied on all netdata dimensions).~~ + +(statsd metrics do not loose their first data collection due to interpolation anymore - fixed with [PR #2411](https://github.com/netdata/netdata/pull/2411)) + +## sending statsd metrics from shell scripts + +You can send/update statsd metrics from shell scripts. You can use this feature, to visualize in netdata automated jobs you run on your servers. + +The command you need to run is: + +```sh +echo "NAME:VALUE|TYPE" | nc -u --send-only localhost 8125 +``` + +Where: + +- `NAME` is the metric name +- `VALUE` is the value for that metric (**gauges** `|g`, **timers** `|ms` and **histograms** `|h` accept decimal/fractional numbers, **counters** `|c` and **meters** `|m` accept integers, **sets** `|s` accept anything) +- `TYPE` is one of `g`, `ms`, `h`, `c`, `m`, `s` to select the metric type. + +So, to set `metric1` as gauge to value `10`, use: + +```sh +echo "metric1:10|g" | nc -u --send-only localhost 8125 +``` + +To increment `metric2` by `10`, as a counter, use: + +```sh +echo "metric2:10|c" | nc -u --send-only localhost 8125 +``` + +You can send multiple metrics like this: + +```sh +# send multiple metrics via UDP +printf "metric1:10|g\nmetric2:10|c\n" | nc -u --send-only localhost 8125 +``` + +Remember, for UDP communication each packet should not exceed the MTU. So, if you plan to push too many metrics at once, prefer TCP communication: + +```sh +# send multiple metrics via TCP +printf "metric1:10|g\nmetric2:10|c\n" | nc --send-only localhost 8125 +``` + +You can also use this little function to take care of all the details: + +```sh +#!/usr/bin/env bash + +STATSD_HOST="localhost" +STATSD_PORT="8125" +statsd() { + local udp="-u" all="${*}" + + # if the string length of all parameters given is above 1000, use TCP + [ "${#all}" -gt 1000 ] && udp= + + while [ ! -z "${1}" ] + do + printf "${1}\n" + shift + done | nc ${udp} --send-only ${STATSD_HOST} ${STATSD_PORT} || return 1 + + return 0 +} +``` + +You can use it like this: + +```sh +# first, source it in your script +source statsd.sh + +# then, at any point: +statsd "metric1:10|g" "metric2:10|c" ... +``` + +The function is smart enough to call `nc` just once and pass all the metrics to it. It will also automatically switch to TCP if the metrics to send are above 1000 bytes. diff --git a/conf.d/statsd.d/example.conf b/collectors/statsd.plugin/example.conf index 0af9dd27d..2c7de6c7b 100644 --- a/conf.d/statsd.d/example.conf +++ b/collectors/statsd.plugin/example.conf @@ -1,16 +1,17 @@ # statsd synthetic charts configuration -# You can add many .conf files, one for each of your apps +# You can add many .conf files in /etc/netdata/statsd.d/, +# one for each of your apps. # start a new app - you can add many apps in the same file [app] # give a name for this app # this controls the main menu on the dashboard # and will be the prefix for all charts of the app - name = myapp + name = myexampleapp # match all the metrics of the app - metrics = myapp.* + metrics = myexampleapp.* # shall private charts of these metrics be created? private charts = no @@ -26,13 +27,11 @@ # the default is to use the global history #history = 3600 - - # create a chart -# this is its id - the chart will be named myapp.mychart -[mychart] +# this is its id - the chart will be named myexampleapp.myexamplechart +[myexamplechart] # a name for the chart, similar to the id (2 names for each chart) - name = mychart + name = myexamplechart # the chart title title = my chart title @@ -57,9 +56,9 @@ # events = the number of events for this metric # last = the last value collected # all the others are only valid for histograms and timers - dimension = myapp.metric1 avg average 1 1 - dimension = myapp.metric1 lower min 1 1 - dimension = myapp.metric1 upper max 1 1 - dimension = myapp.metric2 other last 1 1 + dimension = myexampleapp.metric1 avg average 1 1 + dimension = myexampleapp.metric1 lower min 1 1 + dimension = myexampleapp.metric1 upper max 1 1 + dimension = myexampleapp.metric2 other last 1 1 # You can add as many charts as needed diff --git a/src/statsd.c b/collectors/statsd.plugin/statsd.c index 44ebd8894..c92bfd1c2 100644 --- a/src/statsd.c +++ b/collectors/statsd.plugin/statsd.c @@ -1,7 +1,10 @@ -#include "common.h" +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "statsd.h" #define STATSD_CHART_PREFIX "statsd" -#define STATSD_CHART_PRIORITY 90000 + +#define PLUGIN_STATSD_NAME "statsd.plugin" // -------------------------------------------------------------------------------------- @@ -87,7 +90,10 @@ typedef enum statsd_metric_options { STATSD_METRIC_OPTION_PRIVATE_CHART_ENABLED = 0x00000002, // render a private chart for this metric STATSD_METRIC_OPTION_PRIVATE_CHART_CHECKED = 0x00000004, // the metric has been checked if it should get private chart or not STATSD_METRIC_OPTION_CHART_DIMENSION_COUNT = 0x00000008, // show the count of events for this private chart - STATSD_METRIC_OPTION_CHECKED_IN_APPS = 0x00000010, // set when this metric has been checked agains apps + STATSD_METRIC_OPTION_CHECKED_IN_APPS = 0x00000010, // set when this metric has been checked against apps + STATSD_METRIC_OPTION_USED_IN_APPS = 0x00000020, // set when this metric is used in apps + STATSD_METRIC_OPTION_CHECKED = 0x00000040, // set when the charting thread checks this metric for use in charts (its usefulness) + STATSD_METRIC_OPTION_USEFUL = 0x00000080, // set when the charting thread finds the metric useful (i.e. used in a chart) } STATS_METRIC_OPTIONS; typedef enum statsd_metric_type { @@ -101,7 +107,7 @@ typedef enum statsd_metric_type { typedef struct statsd_metric { - avl avl; // indexing + avl avl; // indexing - has to be first const char *name; // the name of the metric uint32_t hash; // hash of the name @@ -122,14 +128,15 @@ typedef struct statsd_metric { // chart related members STATS_METRIC_OPTIONS options; // STATSD_METRIC_OPTION_* (bitfield) - char reset; // set to 1 to reset this metric to zero + char reset; // set to 1 by the charting thread to instruct the collector thread(s) to reset this metric collected_number last; // the last value sent to netdata - RRDSET *st; // the chart of this metric + RRDSET *st; // the private chart of this metric RRDDIM *rd_value; // the dimension of this metric value RRDDIM *rd_count; // the dimension for the number of events received // linking, used for walking through all metrics struct statsd_metric *next; + struct statsd_metric *next_useful; } STATSD_METRIC; @@ -140,10 +147,12 @@ typedef struct statsd_index { char *name; // the name of the index of metrics size_t events; // the number of events processed for this index size_t metrics; // the number of metrics in this index + size_t useful; // the number of useful metrics in this index STATSD_AVL_TREE index; // the AVL tree STATSD_METRIC *first; // the linked list of metrics (new metrics are added in front) + STATSD_METRIC *first_useful; // the linked list of useful metrics (new metrics are added in front) STATSD_FIRST_PTR_MUTEX; // when mutli-threading is enabled, a lock to protect the linked list STATS_METRIC_OPTIONS default_options; // default options for all metrics in this index @@ -255,13 +264,13 @@ static struct statsd { SIMPLE_PATTERN *charts_for; size_t tcp_idle_timeout; - size_t decimal_detail; + collected_number decimal_detail; size_t private_charts; size_t max_private_charts; size_t max_private_charts_hard; RRD_MEMORY_MODE private_charts_memory_mode; long private_charts_rrd_history_entries; - int private_charts_hidden; + unsigned int private_charts_hidden:1; STATSD_APP *apps; size_t recvmmsg_size; @@ -344,6 +353,7 @@ static struct statsd { .threads = 0, .collection_threads_status = NULL, .sockets = { + .config = &netdata_config, .config_section = CONFIG_SECTION_STATSD, .default_bind_to = "udp:localhost tcp:localhost", .default_port = STATSD_LISTEN_PORT, @@ -427,6 +437,13 @@ static inline LONG_DOUBLE statsd_parse_float(const char *v, LONG_DOUBLE def) { return value; } +static inline LONG_DOUBLE statsd_parse_sampling_rate(const char *v) { + LONG_DOUBLE sampling_rate = statsd_parse_float(v, 1.0); + if(unlikely(isless(sampling_rate, 0.001))) sampling_rate = 0.001; + if(unlikely(isgreater(sampling_rate, 1.0))) sampling_rate = 1.0; + return sampling_rate; +} + static inline long long statsd_parse_int(const char *v, long long def) { long long value; @@ -455,7 +472,12 @@ static inline int value_is_zinit(const char *value) { return (value && *value == 'z' && *++value == 'i' && *++value == 'n' && *++value == 'i' && *++value == 't' && *++value == '\0'); } +#define is_metric_checked(m) ((m)->options & STATSD_METRIC_OPTION_CHECKED) +#define is_metric_useful_for_collection(m) (!is_metric_checked(m) || ((m)->options & STATSD_METRIC_OPTION_USEFUL)) + static inline void statsd_process_gauge(STATSD_METRIC *m, const char *value, const char *sampling) { + if(!is_metric_useful_for_collection(m)) return; + if(unlikely(!value || !*value)) { error("STATSD: metric '%s' of type gauge, with empty value is ignored.", m->name); return; @@ -471,16 +493,18 @@ static inline void statsd_process_gauge(STATSD_METRIC *m, const char *value, con } else { if (unlikely(*value == '+' || *value == '-')) - m->gauge.value += statsd_parse_float(value, 1.0) / statsd_parse_float(sampling, 1.0); + m->gauge.value += statsd_parse_float(value, 1.0) / statsd_parse_sampling_rate(sampling); else - m->gauge.value = statsd_parse_float(value, 1.0) / statsd_parse_float(sampling, 1.0); + m->gauge.value = statsd_parse_float(value, 1.0); m->events++; m->count++; } } -static inline void statsd_process_counter(STATSD_METRIC *m, const char *value, const char *sampling) { +static inline void statsd_process_counter_or_meter(STATSD_METRIC *m, const char *value, const char *sampling) { + if(!is_metric_useful_for_collection(m)) return; + // we accept empty values for counters if(unlikely(m->reset)) statsd_reset_metric(m); @@ -489,21 +513,21 @@ static inline void statsd_process_counter(STATSD_METRIC *m, const char *value, c // magic loading of metric, without affecting anything } else { - m->counter.value += llrintl((LONG_DOUBLE) statsd_parse_int(value, 1) / statsd_parse_float(sampling, 1.0)); + m->counter.value += llrintl((LONG_DOUBLE) statsd_parse_int(value, 1) / statsd_parse_sampling_rate(sampling)); m->events++; m->count++; } } -static inline void statsd_process_meter(STATSD_METRIC *m, const char *value, const char *sampling) { - // this is the same with the counter - statsd_process_counter(m, value, sampling); -} +#define statsd_process_counter(m, value, sampling) statsd_process_counter_or_meter(m, value, sampling) +#define statsd_process_meter(m, value, sampling) statsd_process_counter_or_meter(m, value, sampling) + +static inline void statsd_process_histogram_or_timer(STATSD_METRIC *m, const char *value, const char *sampling, const char *type) { + if(!is_metric_useful_for_collection(m)) return; -static inline void statsd_process_histogram(STATSD_METRIC *m, const char *value, const char *sampling) { if(unlikely(!value || !*value)) { - error("STATSD: metric '%s' of type histogram, with empty value is ignored.", m->name); + error("STATSD: metric of type %s, with empty value is ignored.", type); return; } @@ -516,31 +540,35 @@ static inline void statsd_process_histogram(STATSD_METRIC *m, const char *value, // magic loading of metric, without affecting anything } else { - if (unlikely(m->histogram.ext->used == m->histogram.ext->size)) { - netdata_mutex_lock(&m->histogram.ext->mutex); - m->histogram.ext->size += statsd.histogram_increase_step; - m->histogram.ext->values = reallocz(m->histogram.ext->values, sizeof(LONG_DOUBLE) * m->histogram.ext->size); - netdata_mutex_unlock(&m->histogram.ext->mutex); - } + LONG_DOUBLE v = statsd_parse_float(value, 1.0); + LONG_DOUBLE sampling_rate = statsd_parse_sampling_rate(sampling); + if(unlikely(isless(sampling_rate, 0.01))) sampling_rate = 0.01; + if(unlikely(isgreater(sampling_rate, 1.0))) sampling_rate = 1.0; + + long long samples = llrintl(1.0 / sampling_rate); + while(samples-- > 0) { + + if(unlikely(m->histogram.ext->used == m->histogram.ext->size)) { + netdata_mutex_lock(&m->histogram.ext->mutex); + m->histogram.ext->size += statsd.histogram_increase_step; + m->histogram.ext->values = reallocz(m->histogram.ext->values, sizeof(LONG_DOUBLE) * m->histogram.ext->size); + netdata_mutex_unlock(&m->histogram.ext->mutex); + } - m->histogram.ext->values[m->histogram.ext->used++] = statsd_parse_float(value, 1.0) / statsd_parse_float(sampling, 1.0); + m->histogram.ext->values[m->histogram.ext->used++] = v; + } m->events++; m->count++; } } -static inline void statsd_process_timer(STATSD_METRIC *m, const char *value, const char *sampling) { - if(unlikely(!value || !*value)) { - error("STATSD: metric of type timer, with empty value is ignored."); - return; - } - - // timers are a use case of histogram - statsd_process_histogram(m, value, sampling); -} +#define statsd_process_timer(m, value, sampling) statsd_process_histogram_or_timer(m, value, sampling, "timer") +#define statsd_process_histogram(m, value, sampling) statsd_process_histogram_or_timer(m, value, sampling, "histogram") static inline void statsd_process_set(STATSD_METRIC *m, const char *value) { + if(!is_metric_useful_for_collection(m)) return; + if(unlikely(!value || !*value)) { error("STATSD: metric of type set, with empty value is ignored."); return; @@ -578,8 +606,10 @@ static inline void statsd_process_set(STATSD_METRIC *m, const char *value) { // -------------------------------------------------------------------------------------------------------------------- // statsd parsing -static void statsd_process_metric(const char *name, const char *value, const char *type, const char *sampling) { - debug(D_STATSD, "STATSD: raw metric '%s', value '%s', type '%s', rate '%s'", name?name:"(null)", value?value:"(null)", type?type:"(null)", sampling?sampling:"(null)"); +static void statsd_process_metric(const char *name, const char *value, const char *type, const char *sampling, const char *tags) { + (void)tags; + + debug(D_STATSD, "STATSD: raw metric '%s', value '%s', type '%s', sampling '%s', tags '%s'", name?name:"(null)", value?value:"(null)", type?type:"(null)", sampling?sampling:"(null)", tags?tags:"(null)"); if(unlikely(!name || !*name)) return; if(unlikely(!type || !*type)) type = "m"; @@ -663,8 +693,8 @@ static inline size_t statsd_process(char *buffer, size_t size, int require_newli const char *s = buffer; while(*s) { - const char *name = NULL, *value = NULL, *type = NULL, *sampling = NULL; - char *name_end = NULL, *value_end = NULL, *type_end = NULL, *sampling_end = NULL; + const char *name = NULL, *value = NULL, *type = NULL, *sampling = NULL, *tags = NULL; + char *name_end = NULL, *value_end = NULL, *type_end = NULL, *sampling_end = NULL, *tags_end = NULL; s = name_end = (char *)statsd_parse_skip_up_to(name = s, ':', '|'); if(name == name_end) { @@ -679,10 +709,15 @@ static inline size_t statsd_process(char *buffer, size_t size, int require_newli s = type_end = (char *) statsd_parse_skip_up_to(type = ++s, '|', '@'); if(likely(*s == '|' || *s == '@')) { - s = sampling_end = (char *) statsd_parse_skip_up_to(sampling = ++s, '\r', '\n'); + s = sampling_end = (char *) statsd_parse_skip_up_to(sampling = ++s, '|', '#'); if(*sampling == '@') sampling++; } + if(likely(*s == '|' || *s == '#')) { + s = tags_end = (char *) statsd_parse_skip_up_to(tags = ++s, '|', '|'); + if(*tags == '#') tags++; + } + // skip everything until the end of the line while(*s && *s != '\n') s++; @@ -700,6 +735,7 @@ static inline size_t statsd_process(char *buffer, size_t size, int require_newli , statsd_parse_field_trim(value, value_end) , statsd_parse_field_trim(type, type_end) , statsd_parse_field_trim(sampling, sampling_end) + , statsd_parse_field_trim(tags, tags_end) ); } @@ -997,7 +1033,7 @@ void *statsd_collector_thread(void *ptr) { #define STATSD_CONF_LINE_MAX 8192 -static STATSD_APP_CHART_DIM_VALUE_TYPE string2valuetype(const char *type, size_t line, const char *path, const char *filename) { +static STATSD_APP_CHART_DIM_VALUE_TYPE string2valuetype(const char *type, size_t line, const char *filename) { if(!type || !*type) type = "last"; if(!strcmp(type, "events")) return STATSD_APP_CHART_DIM_VALUE_TYPE_EVENTS; @@ -1010,7 +1046,7 @@ static STATSD_APP_CHART_DIM_VALUE_TYPE string2valuetype(const char *type, size_t else if(!strcmp(type, "stddev")) return STATSD_APP_CHART_DIM_VALUE_TYPE_STDDEV; else if(!strcmp(type, "percentile")) return STATSD_APP_CHART_DIM_VALUE_TYPE_PERCENTILE; - error("STATSD: invalid type '%s' at line %zu of file '%s/%s'. Using 'last'.", type, line, path, filename); + error("STATSD: invalid type '%s' at line %zu of file '%s'. Using 'last'.", type, line, filename); return STATSD_APP_CHART_DIM_VALUE_TYPE_LAST; } @@ -1076,19 +1112,14 @@ static STATSD_APP_CHART_DIM *add_dimension_to_app_chart( return dim; } -static int statsd_readfile(const char *path, const char *filename, STATSD_APP *app, STATSD_APP_CHART *chart, DICTIONARY *dict) { - debug(D_STATSD, "STATSD configuration reading file '%s/%s'", path, filename); +static int statsd_readfile(const char *filename, STATSD_APP *app, STATSD_APP_CHART *chart, DICTIONARY *dict) { + debug(D_STATSD, "STATSD configuration reading file '%s'", filename); char *buffer = mallocz(STATSD_CONF_LINE_MAX + 1); - if(filename[0] == '/') - strncpyz(buffer, filename, STATSD_CONF_LINE_MAX); - else - snprintfz(buffer, STATSD_CONF_LINE_MAX, "%s/%s", path, filename); - - FILE *fp = fopen(buffer, "r"); + FILE *fp = fopen(filename, "r"); if(!fp) { - error("STATSD: cannot open file '%s'.", buffer); + error("STATSD: cannot open file '%s'.", filename); freez(buffer); return -1; } @@ -1101,18 +1132,31 @@ static int statsd_readfile(const char *path, const char *filename, STATSD_APP *a s = trim(buffer); if (!s || *s == '#') { - debug(D_STATSD, "STATSD: ignoring line %zu of file '%s/%s', it is empty.", line, path, filename); + debug(D_STATSD, "STATSD: ignoring line %zu of file '%s', it is empty.", line, filename); continue; } - debug(D_STATSD, "STATSD: processing line %zu of file '%s/%s': %s", line, path, filename, buffer); + debug(D_STATSD, "STATSD: processing line %zu of file '%s': %s", line, filename, buffer); if(*s == 'i' && strncmp(s, "include", 7) == 0) { s = trim(&s[7]); - if(s && *s) - statsd_readfile(path, s, app, chart, dict); + if(s && *s) { + char *tmp; + if(*s == '/') + tmp = strdupz(s); + else { + // the file to be included is relative to current file + // find the directory name from the file we already read + char *filename2 = strdupz(filename); // copy filename, since dirname() will change it + char *dir = dirname(filename2); // find the directory part of the filename + tmp = strdupz_path_subpath(dir, s); // compose the new filename to read; + freez(filename2); // free the filename we copied + } + statsd_readfile(tmp, app, chart, dict); + freez(tmp); + } else - error("STATSD: ignoring line %zu of file '%s/%s', include filename is empty", line, path, s); + error("STATSD: ignoring line %zu of file '%s', include filename is empty", line, filename); continue; } @@ -1160,7 +1204,7 @@ static int statsd_readfile(const char *path, const char *filename, STATSD_APP *a chart->context = strdupz(s); chart->family = strdupz("overview"); chart->units = strdupz("value"); - chart->priority = STATSD_CHART_PRIORITY; + chart->priority = NETDATA_CHART_PRIO_STATSD_PRIVATE; chart->chart_type = RRDSET_TYPE_LINE; chart->next = app->charts; @@ -1174,20 +1218,20 @@ static int statsd_readfile(const char *path, const char *filename, STATSD_APP *a } } else - error("STATSD: ignoring line %zu ('%s') of file '%s/%s', [app] is not defined.", line, s, path, filename); + error("STATSD: ignoring line %zu ('%s') of file '%s', [app] is not defined.", line, s, filename); continue; } if(!app) { - error("STATSD: ignoring line %zu ('%s') of file '%s/%s', it is outside all sections.", line, s, path, filename); + error("STATSD: ignoring line %zu ('%s') of file '%s', it is outside all sections.", line, s, filename); continue; } char *name = s; char *value = strchr(s, '='); if(!value) { - error("STATSD: ignoring line %zu ('%s') of file '%s/%s', there is no = in it.", line, s, path, filename); + error("STATSD: ignoring line %zu ('%s') of file '%s', there is no = in it.", line, s, filename); continue; } *value = '\0'; @@ -1197,11 +1241,11 @@ static int statsd_readfile(const char *path, const char *filename, STATSD_APP *a value = trim(value); if(!name || *name == '#') { - error("STATSD: ignoring line %zu of file '%s/%s', name is empty.", line, path, filename); + error("STATSD: ignoring line %zu of file '%s', name is empty.", line, filename); continue; } if(!value) { - debug(D_CONFIG, "STATSD: ignoring line %zu of file '%s/%s', value is empty.", line, path, filename); + debug(D_CONFIG, "STATSD: ignoring line %zu of file '%s', value is empty.", line, filename); continue; } @@ -1241,7 +1285,7 @@ static int statsd_readfile(const char *path, const char *filename, STATSD_APP *a app->rrd_history_entries = 5; } else { - error("STATSD: ignoring line %zu ('%s') of file '%s/%s'. Unknown keyword for the [app] section.", line, name, path, filename); + error("STATSD: ignoring line %zu ('%s') of file '%s'. Unknown keyword for the [app] section.", line, name, filename); continue; } } @@ -1326,14 +1370,14 @@ static int statsd_readfile(const char *path, const char *filename, STATSD_APP *a , (multipler && *multipler)?str2l(multipler):1 , (divisor && *divisor)?str2l(divisor):1 , flags - , string2valuetype(type, line, path, filename) + , string2valuetype(type, line, filename) ); if(pattern) dim->metric_pattern = simple_pattern_create(dim->metric, NULL, SIMPLE_PATTERN_EXACT); } else { - error("STATSD: ignoring line %zu ('%s') of file '%s/%s'. Unknown keyword for the [%s] section.", line, name, path, filename, chart->id); + error("STATSD: ignoring line %zu ('%s') of file '%s'. Unknown keyword for the [%s] section.", line, name, filename, chart->id); continue; } } @@ -1344,49 +1388,13 @@ static int statsd_readfile(const char *path, const char *filename, STATSD_APP *a return 0; } -static void statsd_readdir(const char *path) { - size_t pathlen = strlen(path); - - debug(D_STATSD, "STATSD configuration reading directory '%s'", path); - - DIR *dir = opendir(path); - if (!dir) { - error("STATSD configuration cannot open directory '%s'.", path); - return; - } - - struct dirent *de = NULL; - while ((de = readdir(dir))) { - size_t len = strlen(de->d_name); - - if(de->d_type == DT_DIR - && ( - (de->d_name[0] == '.' && de->d_name[1] == '\0') - || (de->d_name[0] == '.' && de->d_name[1] == '.' && de->d_name[2] == '\0') - )) { - debug(D_STATSD, "STATSD: ignoring directory '%s'", de->d_name); - continue; - } - - else if(de->d_type == DT_DIR) { - char *s = mallocz(pathlen + strlen(de->d_name) + 2); - strcpy(s, path); - strcat(s, "/"); - strcat(s, de->d_name); - statsd_readdir(s); - freez(s); - continue; - } - - else if((de->d_type == DT_LNK || de->d_type == DT_REG || de->d_type == DT_UNKNOWN) && - len > 5 && !strcmp(&de->d_name[len - 5], ".conf")) { - statsd_readfile(path, de->d_name, NULL, NULL, NULL); - } - - else debug(D_STATSD, "STATSD: ignoring file '%s'", de->d_name); - } +static int statsd_file_callback(const char *filename, void *data) { + (void)data; + return statsd_readfile(filename, NULL, NULL, NULL); +} - closedir(dir); +static inline void statsd_readdir(const char *user_path, const char *stock_path, const char *subpath) { + recursive_config_double_dir_load(user_path, stock_path, subpath, statsd_file_callback, NULL, 0); } // -------------------------------------------------------------------------------------------------------------------- @@ -1445,7 +1453,7 @@ static inline RRDSET *statsd_private_rrdset_create( , context // context , title // title , units // units - , "statsd" // plugin + , PLUGIN_STATSD_NAME // plugin , "private_chart" // module , priority // priority , update_every // update every @@ -1484,7 +1492,7 @@ static inline void statsd_private_chart_gauge(STATSD_METRIC *m) { , context // context , title // title , "value" // units - , STATSD_CHART_PRIORITY + , NETDATA_CHART_PRIO_STATSD_PRIVATE , statsd.update_every , RRDSET_TYPE_LINE ); @@ -1526,7 +1534,7 @@ static inline void statsd_private_chart_counter_or_meter(STATSD_METRIC *m, const , context // context , title // title , "events/s" // units - , STATSD_CHART_PRIORITY + , NETDATA_CHART_PRIO_STATSD_PRIVATE , statsd.update_every , RRDSET_TYPE_AREA ); @@ -1568,7 +1576,7 @@ static inline void statsd_private_chart_set(STATSD_METRIC *m) { , context // context , title // title , "entries" // units - , STATSD_CHART_PRIORITY + , NETDATA_CHART_PRIO_STATSD_PRIVATE , statsd.update_every , RRDSET_TYPE_LINE ); @@ -1610,7 +1618,7 @@ static inline void statsd_private_chart_timer_or_histogram(STATSD_METRIC *m, con , context // context , title // title , units // units - , STATSD_CHART_PRIORITY + , NETDATA_CHART_PRIO_STATSD_PRIVATE , statsd.update_every , RRDSET_TYPE_AREA ); @@ -1649,14 +1657,14 @@ static inline void statsd_flush_gauge(STATSD_METRIC *m) { debug(D_STATSD, "flushing gauge metric '%s'", m->name); int updated = 0; - if(m->count && !m->reset) { + if(unlikely(!m->reset && m->count)) { m->last = (collected_number) (m->gauge.value * statsd.decimal_detail); m->reset = 1; updated = 1; } - if(m->options & STATSD_METRIC_OPTION_PRIVATE_CHART_ENABLED && (updated || !(m->options & STATSD_METRIC_OPTION_SHOW_GAPS_WHEN_NOT_COLLECTED))) + if(unlikely(m->options & STATSD_METRIC_OPTION_PRIVATE_CHART_ENABLED && (updated || !(m->options & STATSD_METRIC_OPTION_SHOW_GAPS_WHEN_NOT_COLLECTED)))) statsd_private_chart_gauge(m); } @@ -1664,14 +1672,14 @@ static inline void statsd_flush_counter_or_meter(STATSD_METRIC *m, const char *d debug(D_STATSD, "flushing %s metric '%s'", dim, m->name); int updated = 0; - if(m->count && !m->reset) { + if(unlikely(!m->reset && m->count)) { m->last = m->counter.value; m->reset = 1; updated = 1; } - if(m->options & STATSD_METRIC_OPTION_PRIVATE_CHART_ENABLED && (updated || !(m->options & STATSD_METRIC_OPTION_SHOW_GAPS_WHEN_NOT_COLLECTED))) + if(unlikely(m->options & STATSD_METRIC_OPTION_PRIVATE_CHART_ENABLED && (updated || !(m->options & STATSD_METRIC_OPTION_SHOW_GAPS_WHEN_NOT_COLLECTED)))) statsd_private_chart_counter_or_meter(m, dim, family); } @@ -1687,40 +1695,27 @@ static inline void statsd_flush_set(STATSD_METRIC *m) { debug(D_STATSD, "flushing set metric '%s'", m->name); int updated = 0; - if(m->count && !m->reset) { + if(unlikely(!m->reset && m->count)) { m->last = (collected_number)m->set.unique; m->reset = 1; updated = 1; } + else { + m->last = 0; + } - if(m->options & STATSD_METRIC_OPTION_PRIVATE_CHART_ENABLED && (updated || !(m->options & STATSD_METRIC_OPTION_SHOW_GAPS_WHEN_NOT_COLLECTED))) + if(unlikely(m->options & STATSD_METRIC_OPTION_PRIVATE_CHART_ENABLED && (updated || !(m->options & STATSD_METRIC_OPTION_SHOW_GAPS_WHEN_NOT_COLLECTED)))) statsd_private_chart_set(m); } static inline void statsd_flush_timer_or_histogram(STATSD_METRIC *m, const char *dim, const char *family, const char *units) { debug(D_STATSD, "flushing %s metric '%s'", dim, m->name); - netdata_mutex_lock(&m->histogram.ext->mutex); - - if(unlikely(!m->histogram.ext->zeroed)) { - // reset the metrics - // if we collected anything, they will be updated below - // this ensures that we report zeros if nothing is collected - - m->histogram.ext->last_min = 0; - m->histogram.ext->last_max = 0; - m->last = 0; - m->histogram.ext->last_median = 0; - m->histogram.ext->last_stddev = 0; - m->histogram.ext->last_sum = 0; - m->histogram.ext->last_percentile = 0; - - m->histogram.ext->zeroed = 1; - } - int updated = 0; - if(m->count && !m->reset && m->histogram.ext->used > 0) { + if(unlikely(!m->reset && m->count && m->histogram.ext->used > 0)) { + netdata_mutex_lock(&m->histogram.ext->mutex); + size_t len = m->histogram.ext->used; LONG_DOUBLE *series = m->histogram.ext->values; sort_series(series, len); @@ -1738,6 +1733,8 @@ static inline void statsd_flush_timer_or_histogram(STATSD_METRIC *m, const char else m->histogram.ext->last_percentile = (collected_number)roundl(series[pct_len - 1] * statsd.decimal_detail); + netdata_mutex_unlock(&m->histogram.ext->mutex); + debug(D_STATSD, "STATSD %s metric %s: min " COLLECTED_NUMBER_FORMAT ", max " COLLECTED_NUMBER_FORMAT ", last " COLLECTED_NUMBER_FORMAT ", pcent " COLLECTED_NUMBER_FORMAT ", median " COLLECTED_NUMBER_FORMAT ", stddev " COLLECTED_NUMBER_FORMAT ", sum " COLLECTED_NUMBER_FORMAT, dim, m->name, m->histogram.ext->last_min, m->histogram.ext->last_max, m->last, m->histogram.ext->last_percentile, m->histogram.ext->last_median, m->histogram.ext->last_stddev, m->histogram.ext->last_sum); @@ -1745,11 +1742,24 @@ static inline void statsd_flush_timer_or_histogram(STATSD_METRIC *m, const char m->reset = 1; updated = 1; } + else if(unlikely(!m->histogram.ext->zeroed)) { + // reset the metrics + // if we collected anything, they will be updated below + // this ensures that we report zeros if nothing is collected - if(m->options & STATSD_METRIC_OPTION_PRIVATE_CHART_ENABLED && (updated || !(m->options & STATSD_METRIC_OPTION_SHOW_GAPS_WHEN_NOT_COLLECTED))) - statsd_private_chart_timer_or_histogram(m, dim, family, units); + m->histogram.ext->last_min = 0; + m->histogram.ext->last_max = 0; + m->last = 0; + m->histogram.ext->last_median = 0; + m->histogram.ext->last_stddev = 0; + m->histogram.ext->last_sum = 0; + m->histogram.ext->last_percentile = 0; - netdata_mutex_unlock(&m->histogram.ext->mutex); + m->histogram.ext->zeroed = 1; + } + + if(unlikely(m->options & STATSD_METRIC_OPTION_PRIVATE_CHART_ENABLED && (updated || !(m->options & STATSD_METRIC_OPTION_SHOW_GAPS_WHEN_NOT_COLLECTED)))) + statsd_private_chart_timer_or_histogram(m, dim, family, units); } static inline void statsd_flush_timer(STATSD_METRIC *m) { @@ -1837,6 +1847,7 @@ static inline void link_metric_to_app_dimension(STATSD_APP *app, STATSD_METRIC * } chart->dimensions_linked_count++; + m->options |= STATSD_METRIC_OPTION_USED_IN_APPS; debug(D_STATSD, "metric '%s' of type %u linked with app '%s', chart '%s', dimension '%s', algorithm '%s'", m->name, m->type, app->name, chart->id, dim->name, rrd_algorithm_name(dim->algorithm)); } @@ -1980,7 +1991,7 @@ static inline void statsd_update_app_chart(STATSD_APP *app, STATSD_APP_CHART *ch , chart->context // context , chart->title // title , chart->units // units - , "statsd" // plugin + , PLUGIN_STATSD_NAME // plugin , chart->source // module , chart->priority // priority , statsd.update_every // update every @@ -2043,7 +2054,13 @@ const char *statsd_metric_type_string(STATSD_METRIC_TYPE type) { static inline void statsd_flush_index_metrics(STATSD_INDEX *index, void (*flush_metric)(STATSD_METRIC *)) { STATSD_METRIC *m; + + // find the useful metrics (incremental = each time we are called, we check the new metrics only) for(m = index->first; m ; m = m->next) { + // since we add new metrics at the beginning + // check for useful charts, until the point we last checked + if(unlikely(is_metric_checked(m))) break; + if(unlikely(!(m->options & STATSD_METRIC_OPTION_CHECKED_IN_APPS))) { log_access("NEW STATSD METRIC '%s': '%s'", statsd_metric_type_string(m->type), m->name); check_if_metric_is_for_app(index, m); @@ -2051,7 +2068,7 @@ static inline void statsd_flush_index_metrics(STATSD_INDEX *index, void (*flush_ } if(unlikely(!(m->options & STATSD_METRIC_OPTION_PRIVATE_CHART_CHECKED))) { - if(statsd.private_charts >= statsd.max_private_charts_hard) { + if(unlikely(statsd.private_charts >= statsd.max_private_charts_hard)) { debug(D_STATSD, "STATSD: metric '%s' will not be charted, because the hard limit of the maximum number of charts has been reached.", m->name); info("STATSD: metric '%s' will not be charted, because the hard limit of the maximum number of charts (%zu) has been reached. Increase the number of charts by editing netdata.conf, [statsd] section.", m->name, statsd.max_private_charts); m->options &= ~STATSD_METRIC_OPTION_PRIVATE_CHART_ENABLED; @@ -2069,6 +2086,20 @@ static inline void statsd_flush_index_metrics(STATSD_INDEX *index, void (*flush_ m->options |= STATSD_METRIC_OPTION_PRIVATE_CHART_CHECKED; } + // mark it as checked + m->options |= STATSD_METRIC_OPTION_CHECKED; + + // check if it is used in charts + if((m->options & (STATSD_METRIC_OPTION_PRIVATE_CHART_ENABLED|STATSD_METRIC_OPTION_USED_IN_APPS)) && !(m->options & STATSD_METRIC_OPTION_USEFUL)) { + m->options |= STATSD_METRIC_OPTION_USEFUL; + index->useful++; + m->next_useful = index->first_useful; + index->first_useful = m; + } + } + + // flush all the useful metrics + for(m = index->first_useful; m ; m = m->next_useful) { flush_metric(m); } } @@ -2130,9 +2161,9 @@ void *statsd_main(void *ptr) { statsd.max_private_charts_hard = (size_t)config_get_number(CONFIG_SECTION_STATSD, "max private charts hard limit", (long long)statsd.max_private_charts * 5); statsd.private_charts_memory_mode = rrd_memory_mode_id(config_get(CONFIG_SECTION_STATSD, "private charts memory mode", rrd_memory_mode_name(default_rrd_memory_mode))); statsd.private_charts_rrd_history_entries = (int)config_get_number(CONFIG_SECTION_STATSD, "private charts history", default_rrd_history_entries); - statsd.decimal_detail = (size_t)config_get_number(CONFIG_SECTION_STATSD, "decimal detail", (long long int)statsd.decimal_detail); + statsd.decimal_detail = (collected_number)config_get_number(CONFIG_SECTION_STATSD, "decimal detail", (long long int)statsd.decimal_detail); statsd.tcp_idle_timeout = (size_t) config_get_number(CONFIG_SECTION_STATSD, "disconnect idle tcp clients after seconds", (long long int)statsd.tcp_idle_timeout); - statsd.private_charts_hidden = (int)config_get_boolean(CONFIG_SECTION_STATSD, "private charts hidden", statsd.private_charts_hidden); + statsd.private_charts_hidden = (unsigned int)config_get_boolean(CONFIG_SECTION_STATSD, "private charts hidden", statsd.private_charts_hidden); statsd.histogram_percentile = (double)config_get_float(CONFIG_SECTION_STATSD, "histograms and timers percentile (percentThreshold)", statsd.histogram_percentile); if(isless(statsd.histogram_percentile, 0) || isgreater(statsd.histogram_percentile, 100)) { @@ -2186,11 +2217,7 @@ void *statsd_main(void *ptr) { #endif // read custom application definitions - { - char filename[FILENAME_MAX + 1]; - snprintfz(filename, FILENAME_MAX, "%s/statsd.d", netdata_configured_config_dir); - statsd_readdir(filename); - } + statsd_readdir(netdata_configured_user_config_dir, netdata_configured_stock_config_dir, "statsd.d"); // ---------------------------------------------------------------------------------------------------------------- // statsd setup @@ -2224,7 +2251,7 @@ void *statsd_main(void *ptr) { , NULL , "Metrics in the netdata statsd database" , "metrics" - , "statsd" + , PLUGIN_STATSD_NAME , "stats" , 132010 , statsd.update_every @@ -2237,6 +2264,27 @@ void *statsd_main(void *ptr) { RRDDIM *rd_metrics_histogram = rrddim_add(st_metrics, "histograms", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); RRDDIM *rd_metrics_set = rrddim_add(st_metrics, "sets", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); + RRDSET *st_useful_metrics = rrdset_create_localhost( + "netdata" + , "statsd_useful_metrics" + , NULL + , "statsd" + , NULL + , "Useful metrics in the netdata statsd database" + , "metrics" + , PLUGIN_STATSD_NAME + , "stats" + , 132010 + , statsd.update_every + , RRDSET_TYPE_STACKED + ); + RRDDIM *rd_useful_metrics_gauge = rrddim_add(st_useful_metrics, "gauges", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); + RRDDIM *rd_useful_metrics_counter = rrddim_add(st_useful_metrics, "counters", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); + RRDDIM *rd_useful_metrics_timer = rrddim_add(st_useful_metrics, "timers", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); + RRDDIM *rd_useful_metrics_meter = rrddim_add(st_useful_metrics, "meters", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); + RRDDIM *rd_useful_metrics_histogram = rrddim_add(st_useful_metrics, "histograms", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); + RRDDIM *rd_useful_metrics_set = rrddim_add(st_useful_metrics, "sets", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); + RRDSET *st_events = rrdset_create_localhost( "netdata" , "statsd_events" @@ -2245,7 +2293,7 @@ void *statsd_main(void *ptr) { , NULL , "Events processed by the netdata statsd server" , "events/s" - , "statsd" + , PLUGIN_STATSD_NAME , "stats" , 132011 , statsd.update_every @@ -2268,7 +2316,7 @@ void *statsd_main(void *ptr) { , NULL , "Read operations made by the netdata statsd server" , "reads/s" - , "statsd" + , PLUGIN_STATSD_NAME , "stats" , 132012 , statsd.update_every @@ -2285,7 +2333,7 @@ void *statsd_main(void *ptr) { , NULL , "Bytes read by the netdata statsd server" , "kilobits/s" - , "netdata" + , PLUGIN_STATSD_NAME , "stats" , 132013 , statsd.update_every @@ -2302,7 +2350,7 @@ void *statsd_main(void *ptr) { , NULL , "Network packets processed by the netdata statsd server" , "packets/s" - , "netdata" + , PLUGIN_STATSD_NAME , "stats" , 132014 , statsd.update_every @@ -2319,7 +2367,7 @@ void *statsd_main(void *ptr) { , NULL , "statsd server TCP connects and disconnects" , "events" - , "statsd" + , PLUGIN_STATSD_NAME , "stats" , 132015 , statsd.update_every @@ -2336,7 +2384,7 @@ void *statsd_main(void *ptr) { , NULL , "statsd server TCP connected sockets" , "connected" - , "statsd" + , PLUGIN_STATSD_NAME , "stats" , 132016 , statsd.update_every @@ -2352,7 +2400,7 @@ void *statsd_main(void *ptr) { , NULL , "Private metric charts created by the netdata statsd server" , "charts" - , "statsd" + , PLUGIN_STATSD_NAME , "stats" , 132020 , statsd.update_every @@ -2368,7 +2416,7 @@ void *statsd_main(void *ptr) { , "netdata.statsd_cpu" , "NetData statsd charting thread CPU usage" , "milliseconds/s" - , "statsd" + , PLUGIN_STATSD_NAME , "stats" , 132001 , statsd.update_every @@ -2394,7 +2442,7 @@ void *statsd_main(void *ptr) { , "netdata.statsd_cpu" , title , "milliseconds/s" - , "statsd" + , PLUGIN_STATSD_NAME , "stats" , 132002 + i , statsd.update_every @@ -2430,6 +2478,7 @@ void *statsd_main(void *ptr) { if(likely(hb_dt)) { rrdset_next(st_metrics); + rrdset_next(st_useful_metrics); rrdset_next(st_events); rrdset_next(st_reads); rrdset_next(st_bytes); @@ -2450,6 +2499,14 @@ void *statsd_main(void *ptr) { rrddim_set_by_pointer(st_metrics, rd_metrics_set, (collected_number)statsd.sets.metrics); rrdset_done(st_metrics); + rrddim_set_by_pointer(st_useful_metrics, rd_useful_metrics_gauge, (collected_number)statsd.gauges.useful); + rrddim_set_by_pointer(st_useful_metrics, rd_useful_metrics_counter, (collected_number)statsd.counters.useful); + rrddim_set_by_pointer(st_useful_metrics, rd_useful_metrics_timer, (collected_number)statsd.timers.useful); + rrddim_set_by_pointer(st_useful_metrics, rd_useful_metrics_meter, (collected_number)statsd.meters.useful); + rrddim_set_by_pointer(st_useful_metrics, rd_useful_metrics_histogram, (collected_number)statsd.histograms.useful); + rrddim_set_by_pointer(st_useful_metrics, rd_useful_metrics_set, (collected_number)statsd.sets.useful); + rrdset_done(st_useful_metrics); + rrddim_set_by_pointer(st_events, rd_events_gauge, (collected_number)statsd.gauges.events); rrddim_set_by_pointer(st_events, rd_events_counter, (collected_number)statsd.counters.events); rrddim_set_by_pointer(st_events, rd_events_timer, (collected_number)statsd.timers.events); diff --git a/collectors/statsd.plugin/statsd.h b/collectors/statsd.plugin/statsd.h new file mode 100644 index 000000000..b741be76d --- /dev/null +++ b/collectors/statsd.plugin/statsd.h @@ -0,0 +1,25 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef NETDATA_STATSD_H +#define NETDATA_STATSD_H 1 + +#include "../../daemon/common.h" + +#define STATSD_LISTEN_PORT 8125 +#define STATSD_LISTEN_BACKLOG 4096 + +#define NETDATA_PLUGIN_HOOK_STATSD \ + { \ + .name = "STATSD", \ + .config_section = NULL, \ + .config_name = NULL, \ + .enabled = 1, \ + .thread = NULL, \ + .init_routine = NULL, \ + .start_routine = statsd_main \ + }, + + +extern void *statsd_main(void *ptr); + +#endif //NETDATA_STATSD_H |