diff options
Diffstat (limited to '')
| -rw-r--r-- | plugins.d/Makefile.am | 16 | ||||
| -rw-r--r-- | plugins.d/README.md | 236 | ||||
| -rwxr-xr-x | plugins.d/charts.d.dryrun-helper.sh | 73 | ||||
| -rwxr-xr-x | plugins.d/charts.d.plugin | 592 | ||||
| -rw-r--r-- | plugins.d/loopsleepms.sh.inc | 90 | ||||
| -rwxr-xr-x | plugins.d/node.d.plugin | 278 | ||||
| -rwxr-xr-x | plugins.d/tc-qos-helper.sh | 105 |
7 files changed, 1390 insertions, 0 deletions
diff --git a/plugins.d/Makefile.am b/plugins.d/Makefile.am new file mode 100644 index 000000000..a89ee4cdd --- /dev/null +++ b/plugins.d/Makefile.am @@ -0,0 +1,16 @@ +# +# Copyright (C) 2015 Alon Bar-Lev <alon.barlev@gmail.com> +# +MAINTAINERCLEANFILES= $(srcdir)/Makefile.in + +dist_plugins_DATA = \ + README.md \ + $(NULL) + +dist_plugins_SCRIPTS = \ + charts.d.dryrun-helper.sh \ + charts.d.plugin \ + node.d.plugin \ + tc-qos-helper.sh \ + loopsleepms.sh.inc \ + $(NULL) diff --git a/plugins.d/README.md b/plugins.d/README.md new file mode 100644 index 000000000..35b9a2d99 --- /dev/null +++ b/plugins.d/README.md @@ -0,0 +1,236 @@ +netdata plugins +=============== + +Any program that can print a few values to its standard output can become +a netdata plugin. + +There are 5 lines netdata parses. lines starting with: + +- `CHART` - create a new chart +- `DIMENSION` - add a dimension to the chart just created +- `BEGIN` - initialize data collection for a chart +- `SET` - set the value of a dimension for the initialized chart +- `END` - complete data collection for the initialized chart + +a single program can produce any number of charts with any number of dimensions +each. + +charts can also be added any time (not just the beginning). + +### command line parameters + +The plugin should accept just **one** parameter: **the number of seconds it is +expected to update the values for its charts**. The value passed by netdata +to the plugin is controlled via its configuration file (so there is not need +for the plugin to handle this configuration option). + +The script can overwrite the update frequency. For example, the server may +request per second updates, but the script may overwrite this to one update +every 5 seconds. + +### environment variables + +There are a few environment variables that are set by `netdata` and are +available for the plugin to use. + +variable|description +:------:|:---------- +`NETDATA_CONFIG_DIR`|The directory where all netdata related configuration should be stored. If the plugin requires custom configuration, this is the place to save it. +`NETDATA_PLUGINS_DIR`|The directory where all netdata plugins are stored. +`NETDATA_WEB_DIR`|The directory where the web files of netdata are saved. +`NETDATA_CACHE_DIR`|The directory where the cache files of netdata are stored. Use this directory if the plugin requires a place to store data. A new directory should be created for the plugin for this purpose, inside this directory. +`NETDATA_LOG_DIR`|The directory where the log files are stored. By default the `stderr` output of the plugin will be saved in the `error.log` file of netdata. +`NETDATA_HOST_PREFIX`|This is used in environments where system directories like `/sys` and `/proc` have to be accessed at a different path. +`NETDATA_DEBUG_FLAGS`|This is number (probably in hex starting with `0x`), that enables certain netdata debugging features. +`NETDATA_UPDATE_EVERY`|The minimum number of seconds between chart refreshes. This is like the **internal clock** of netdata (it is user configurable, defaulting to `1`). There is no meaning for a plugin to update its values more frequently than this number of seconds. + + +# the output of the plugin + +The plugin should output instructions for netdata to its output (`stdout`). + +## CHART + +`CHART` defines a new chart. + +the template is: + +> CHART type.id name title units [family [category [charttype [priority [update_every]]]]] + + where: + - `type.id` + + uniquely identifies the chart, + this is what will be needed to add values to the chart + + - `name` + + is the name that will be presented to the used for this chart + + - `title` + + the text above the chart + + - `units` + + the label of the vertical axis of the chart, + all dimensions added to a chart should have the same units + of measurement + + - `family` + + is used to group charts together + (for example all eth0 charts should say: eth0), + if empty or missing, the `id` part of `type.id` will be used + + - `category` + + the section under which the chart will appear + (for example mem.ram should appear in the 'system' section), + the special word 'none' means: do not show this chart on the home page, + if empty or missing, the `type` part of `type.id` will be used + + - `charttype` + + one of `line`, `area` or `stacked`, + if empty or missing, the `line` will be used + + - `priority` + + is the relative priority of the charts as rendered on the web page, + lower numbers make the charts appear before the ones with higher numbers, + if empty or missing, `1000` will be used + + - `update_every` + + overwrite the update frequency set by the server, + if empty or missing, the user configured value will be used + + +## DIMENSION + +`DIMENSION` defines a new dimension for the chart + +the template is: + +> DIMENSION id [name [algorithm [multiplier [divisor [hidden]]]]] + + where: + + - `id` + + the `id` of this dimension (it is a text value, not numeric), + this will be needed later to add values to the dimension + + - `name` + + the name of the dimension as it will appear at the legend of the chart, + if empty or missing the `id` will be used + + - `algorithm` + + one of: + + * `absolute` + + the value is to drawn as-is (interpolated to second boundary), + if `algorithm` is empty, invalid or missing, `absolute` is used + + * `incremental` + + the value increases over time, + the difference from the last value is presented in the chart, + the server interpolates the value and calculates a per second figure + + * `percentage-of-absolute-row` + + the % of this value compared to the total of all dimensions + + * `percentage-of-incremental-row` + + the % of this value compared to the incremental total of + all dimensions + + - `multiplier` + + an integer value to multiply the collected value, + if empty or missing, `1` is used + + - `divisor` + + an integer value to divide the collected value, + if empty or missing, `1` is used + + - `hidden` + + giving the keyword `hidden` will make this dimension hidden, + it will take part in the calculations but will not be presented in the chart + + +## data collection + +data collection is defined as a series of `BEGIN` -> `SET` -> `END` lines + +> BEGIN type.id [microseconds] + + - `type.id` + + is the unique identification of the chart (as given in `CHART`) + + - `microseconds` + + is the number of microseconds since the last update of the chart, + it is optional. + + Under heavy system load, the system may have some latency transfering + data from the plugins to netdata via the pipe. This number improves + accuracy significantly, since the plugin is able to calculate the + duration between its iterations better than netdata. + + The first time the plugin is started, no microseconds should be given + to netdata. + +> SET id = value + + - `id` + + is the unique identification of the dimension (of the chart just began) + + - `value` + + is the collected value + +> END + + END does not take any parameters, it commits the collected values to the chart. + +More `SET` lines may appear to update all the dimensions of the chart. +All of them in one `BEGIN` -> `END` block. + +All `SET` lines within a single `BEGIN` -> `END` block have to refer to the +same chart. + +If more charts need to be updated, each chart should have its own +`BEGIN` -> `SET` -> `END` block. + +If, for any reason, a plugin has issued a `BEGIN` but wants to cancel it, +it can issue a `FLUSH`. The `FLUSH` command will instruct netdata to ignore +the last `BEGIN` command. + +If a plugin does not behave properly (outputs invalid lines, or does not +follow these guidelines), will be disabled by netdata. + + +### collected values + +netdata will collect any **signed** value in the 64bit range: +`-9.223.372.036.854.775.808` to `+9.223.372.036.854.775.807` + +Internally, all calculations are made using 128 bit double precision and are +stored in 30 bits as floating point. + +If a value is not collected, leave it empty, like this: + +`SET id = ` + +or do not output the line at all. diff --git a/plugins.d/charts.d.dryrun-helper.sh b/plugins.d/charts.d.dryrun-helper.sh new file mode 100755 index 000000000..646452892 --- /dev/null +++ b/plugins.d/charts.d.dryrun-helper.sh @@ -0,0 +1,73 @@ +#!/bin/sh + +# will stop the script for any error +set -e + +me="$0" +name="$1" +chart="$2" +conf="$3" + +can_diff=1 + +tmp1="`mktemp`" +tmp2="`mktemp`" + +myset() { + set | grep -v "^_=" | grep -v "^PIPESTATUS=" | grep -v "^BASH_LINENO=" +} + +# save 2 'set' +myset >"$tmp1" +myset >"$tmp2" + +# make sure they don't differ +diff "$tmp1" "$tmp2" >/dev/null 2>&1 +if [ $? -ne 0 ] +then + # they differ, we cannot do the check + echo >&2 "$me: cannot check with diff." + can_diff=0 +fi + +# do it again, now including the script +myset >"$tmp1" + +# include the plugin and its config +if [ -f "$conf" ] +then + . "$conf" + if [ $? -ne 0 ] + then + echo >&2 "$me: cannot load config file $conf" + rm "$tmp1" "$tmp2" + exit 1 + fi +fi + +. "$chart" +if [ $? -ne 0 ] +then + echo >&2 "$me: cannot load chart file $chart" + rm "$tmp1" "$tmp2" + exit 1 +fi + +# remove all variables starting with the plugin name +myset | grep -v "^$name" >"$tmp2" + +if [ $can_diff -eq 1 ] +then + # check if they are different + # make sure they don't differ + diff "$tmp1" "$tmp2" >&2 + if [ $? -ne 0 ] + then + # they differ + rm "$tmp1" "$tmp2" + exit 1 + fi +fi + +rm "$tmp1" "$tmp2" +exit 0 diff --git a/plugins.d/charts.d.plugin b/plugins.d/charts.d.plugin new file mode 100755 index 000000000..40c0356c8 --- /dev/null +++ b/plugins.d/charts.d.plugin @@ -0,0 +1,592 @@ +#!/bin/bash + +PROGRAM_FILE="$0" +PROGRAM_NAME="$(basename $0)" +PROGRAM_NAME="${PROGRAM_NAME/.plugin}" + +# if you need to run parallel charts.d processes +# just link this files with a different name +# in the same directory, with a .plugin suffix +# netdata will start multiple of them +# each will have a different config file + +echo >&2 "$PROGRAM_NAME: started from '$PROGRAM_FILE' with options: $*" + +if [ $(( ${BASH_VERSINFO[0]} )) -lt 4 ] +then + echo >&2 + echo >&2 "$PROGRAM_NAME: ERROR" + echo >&2 "BASH version 4 or later is required." + echo >&2 "You are running version: ${BASH_VERSION}" + echo >&2 "Please upgrade." + echo >&2 + exit 1 +fi + +# check a few commands +require_cmd() { + which "$1" >/dev/null + if [ $? -ne 0 ] + then + echo >&2 "$PROGRAM_NAME: ERROR: Command '$1' is not found in the system path." + return 1 + fi + return 0 +} + +require_cmd date || exit 1 +require_cmd sed || exit 1 +require_cmd basename || exit 1 +require_cmd dirname || exit 1 +require_cmd cat || exit 1 +require_cmd grep || exit 1 +require_cmd egrep || exit 1 +require_cmd mktemp || exit 1 +require_cmd awk || exit 1 + +# ----------------------------------------------------------------------------- +# insternal defaults +# netdata exposes a few environment variables for us + +pause_method="sleep" # use either "suspend" or "sleep" + # DO NOT USE SUSPEND - LINUX WILL SUSPEND NETDATA TOO + # THE WHOLE PROCESS GROUP - NOT JUST THE SHELL + +pluginsd="${NETDATA_PLUGINS_DIR}" +[ -z "$pluginsd" ] && pluginsd="$( dirname $PROGRAM_FILE )" + +confd="${NETDATA_CONFIG_DIR-/etc/netdata}" +chartsd="$pluginsd/../charts.d" + +myconfig="$confd/$PROGRAM_NAME.conf" + +minimum_update_frequency="${NETDATA_UPDATE_EVERY-1}" +update_every=${minimum_update_frequency} # this will be overwritten by the command line + +# work around for non BASH shells +charts_create="_create" +charts_update="_update" +charts_check="_check" +charts_undescore="_" + +# when making iterations, charts.d can loop more frequently +# to prevent plugins missing iterations. +# this is a percentage relative to update_every to align its +# iterations. +# The minimum is 10%, the maximum 100%. +# So, if update_every is 1 second and time_divisor is 50, +# charts.d will iterate every 500ms. +# Charts will be called to collect data only if the time +# passed since the last time the collected data is equal or +# above their update_every. +time_divisor=50 + +# number of seconds to run without restart +# after this time, charts.d.plugin will exit +# netdata will restart it +restart_timeout=$[3600 * 4] + +# check if the charts.d plugins are using global variables +# they should not. +# It does not currently support BASH v4 arrays, so it is +# disabled +dryrunner=0 + +# check for timeout command +check_for_timeout=1 + +# the default enable/disable value for all charts +enable_all_charts="yes" + +# ----------------------------------------------------------------------------- +# parse parameters + +debug=0 +check=0 +chart_only= +while [ ! -z "$1" ] +do + if [ "$1" = "check" ] + then + check=1 + shift + continue + fi + + if [ "$1" = "debug" -o "$1" = "all" ] + then + debug=1 + shift + continue + fi + + if [ -f "$chartsd/$1.chart.sh" ] + then + debug=1 + chart_only="$( echo $1.chart.sh | sed "s/\.chart\.sh$//g" )" + shift + continue + fi + + if [ -f "$chartsd/$1" ] + then + debug=1 + chart_only="$( echo $1 | sed "s/\.chart\.sh$//g" )" + shift + continue + fi + + # number check + n="$1" + x=$(( n )) + if [ "$x" = "$n" ] + then + shift + update_every=$x + [ $update_every -lt $minimum_update_frequency ] && update_every=$minimum_update_frequency + continue + fi + + echo >&2 "Cannot understand parameter $1. Aborting." + echo "DISABLE" + exit 1 +done + + +# ----------------------------------------------------------------------------- +# load my configuration + +if [ -f "$myconfig" ] + then + . "$myconfig" + if [ $? -ne 0 ] + then + echo >&2 "$PROGRAM_NAME: cannot load $myconfig" + echo "DISABLE" + exit 1 + fi + time_divisor=$((time_divisor)) + [ $time_divisor -lt 10 ] && time_divisor=10 + [ $time_divisor -gt 100 ] && time_divisor=100 +else + echo >&2 "$PROGRAM_NAME: configuration file '$myconfig' not found. Using defaults." +fi + +if [ "$pause_method" = "suspend" ] +then + # enable bash job control + # this is required for suspend to work + set -m +fi + +# we check for the timeout command, after we load our +# configuration, so that the user may overwrite the +# timeout command we use, providing a function that +# can emulate the timeout command we need: +# > timeout SECONDS command ... +if [ $check_for_timeout -eq 1 ] + then + require_cmd timeout || exit 1 +fi + +# ----------------------------------------------------------------------------- +# internal checks + +# netdata passes the requested update frequency as the first argument +update_every=$(( update_every + 1 - 1)) # makes sure it is a number +test $update_every -eq 0 && update_every=1 # if it is zero, make it 1 + +# check the charts.d directory +if [ ! -d "$chartsd" ] + then + echo >&2 "$PROGRAM_NAME: cannot find charts directory '$chartsd'" + echo "DISABLE" +fi + + +# ----------------------------------------------------------------------------- +# loop control + +# default sleep function +LOOPSLEEPMS_HIGHRES=0 +loopsleepms() { + [ "$1" = "tellwork" ] && shift + sleep $1 +} + +now_ms= +current_time_ms() { + now_ms="$(date +'%s')000" +} + +# if found and included, this file overwrites loopsleepms() +# and current_time_ms() with a high resolution timer function +# for precise looping. +. "$pluginsd/loopsleepms.sh.inc" + + +# ----------------------------------------------------------------------------- +# library functions + +fixid() { + echo "$*" |\ + tr -c "[A-Z][a-z][0-9]" "_" |\ + sed -e "s|^_\+||g" -e "s|_\+$||g" -e "s|_\+|_|g" |\ + tr "[A-Z]" "[a-z]" +} + +# convert any floating point number +# to integer, give a multiplier +# the result is stored in ${FLOAT2INT_RESULT} +# so that no fork is necessary +# the multiplier must be a power of 10 +float2int() { + local f m="$2" a b l v=($1) + f=${v[0]} + + # echo >&2 "value='${1}' f='${f}', m='${m}'" + + # the length of the multiplier - 1 + l=$[ ${#m} - 1 ] + + # split the floating point number + # in integer (a) and decimal (b) + a=${f/.*/} + b=${f/*./} + + # prepend a zero to the integer part + # if it is missing + [ -z "${a}" ] && a="0" + # strip leading zeros - base 10 convertion + a=$[10#$a] + + # check the length of the decimal part + # against the length of the multiplier + if [ ${#b} -gt ${l} ] + then + # too many digits - take the most significant + b=${b:0:${l}} + elif [ ${#b} -lt ${l} ] + then + # too few digits - pad with zero on the right + local z="00000000000000000000000" r=$[l - ${#b}] + b="${b}${z:0:${r}}" + fi + # strip leading zeros - base 10 convertion + b=$[10#$b] + + # store the result + FLOAT2INT_RESULT=$[ (a * m) + b ] + #echo >&2 "FLOAT2INT_RESULT='${FLOAT2INT_RESULT}'" +} + + +# ----------------------------------------------------------------------------- +# charts check functions + +all_charts() { + cd "$chartsd" + [ $? -ne 0 ] && echo >&2 "$PROGRAM_NAME: Cannot cd to $chartsd" && return 1 + + ls *.chart.sh | sed "s/\.chart\.sh$//g" +} + +all_enabled_charts() { + local charts= enabled= + + # find all enabled charts + + for chart in $( all_charts ) + do + eval "enabled=\$$chart" + if [ -z "${enabled}" ] + then + enabled="${enable_all_charts}" + fi + + if [ ! "${enabled}" = "yes" ] + then + echo >&2 "$PROGRAM_NAME: '$chart' is NOT enabled. Add a line with $chart=yes in $myconfig to enable it (or remove the line that disables it)." + else + [ $debug -eq 1 ] && echo >&2 "$PROGRAM_NAME: '$chart' is enabled." + local charts="$charts $chart" + fi + done + + local charts2= + for chart in $charts + do + # check the enabled charts + local check="$( cat "$chartsd/$chart.chart.sh" | sed "s/^ \+//g" | grep "^$chart$charts_check()" )" + if [ -z "$check" ] + then + echo >&2 "$PROGRAM_NAME: chart '$chart' does not seem to have a $chart$charts_check() function. Disabling it." + continue + fi + + local create="$( cat "$chartsd/$chart.chart.sh" | sed "s/^ \+//g" | grep "^$chart$charts_create()" )" + if [ -z "$create" ] + then + echo >&2 "$PROGRAM_NAME: chart '$chart' does not seem to have a $chart$charts_create() function. Disabling it." + continue + fi + + local update="$( cat "$chartsd/$chart.chart.sh" | sed "s/^ \+//g" | grep "^$chart$charts_update()" )" + if [ -z "$update" ] + then + echo >&2 "$PROGRAM_NAME: chart '$chart' does not seem to have a $chart$charts_update() function. Disabling it." + continue + fi + + # check its config + #if [ -f "$confd/$chart.conf" ] + #then + # if [ ! -z "$( cat "$confd/$chart.conf" | sed "s/^ \+//g" | grep -v "^$" | grep -v "^#" | grep -v "^$chart$charts_undescore" )" ] + # then + # echo >&2 "$PROGRAM_NAME: chart's $chart config $confd/$chart.conf should only have lines starting with $chart$charts_undescore . Disabling it." + # continue + # fi + #fi + + #if [ $dryrunner -eq 1 ] + # then + # "$pluginsd/charts.d.dryrun-helper.sh" "$chart" "$chartsd/$chart.chart.sh" "$confd/$chart.conf" >/dev/null + # if [ $? -ne 0 ] + # then + # echo >&2 "$PROGRAM_NAME: chart's $chart did not pass the dry run check. This means it uses global variables not starting with $chart. Disabling it." + # continue + # fi + #fi + + local charts2="$charts2 $chart" + done + + echo $charts2 + [ $debug -eq 1 ] && echo >&2 "$PROGRAM_NAME: enabled charts: $charts2" +} + + +# ----------------------------------------------------------------------------- +# load the charts + +suffix_update_every="_update_every" +active_charts= +for chart in $( all_enabled_charts ) +do + [ $debug -eq 1 ] && echo >&2 "$PROGRAM_NAME: loading chart: '$chartsd/$chart.chart.sh'" + . "$chartsd/$chart.chart.sh" + + if [ -f "$confd/$chart.conf" ] + then + [ $debug -eq 1 ] && echo >&2 "$PROGRAM_NAME: loading chart options: '$confd/$chart.conf'" + . "$confd/$chart.conf" + else + echo >&2 "$PROGRAM_NAME: $chart: configuration file '$confd/$chart.conf' not found. Using defaults." + fi + + eval "dt=\$$chart$suffix_update_every" + dt=$(( dt + 1 - 1 )) # make sure it is a number + if [ $dt -lt $update_every ] + then + eval "$chart$suffix_update_every=$update_every" + fi + + $chart$charts_check + if [ $? -eq 0 ] + then + [ $debug -eq 1 ] && echo >&2 "$PROGRAM_NAME: '$chart' activated" + active_charts="$active_charts $chart" + else + echo >&2 "$PROGRAM_NAME: chart '$chart' check() function reports failure." + fi +done +[ $debug -eq 1 ] && echo >&2 "$PROGRAM_NAME: activated charts: $active_charts" + + +# ----------------------------------------------------------------------------- +# check overwrites + +# enable work time reporting +debug_time= +test $debug -eq 1 && debug_time=tellwork + +# if we only need a specific chart, remove all the others +if [ ! -z "${chart_only}" ] +then + [ $debug -eq 1 ] && echo >&2 "$PROGRAM_NAME: requested to run only for: '${chart_only}'" + check_charts= + for chart in $active_charts + do + if [ "$chart" = "$chart_only" ] + then + check_charts="$chart" + break + fi + done + active_charts="$check_charts" +fi +[ $debug -eq 1 ] && echo >&2 "$PROGRAM_NAME: activated charts: $active_charts" + +# stop if we just need a pre-check +if [ $check -eq 1 ] +then + echo >&2 "CHECK RESULT" + echo >&2 "Will run the charts: $active_charts" + exit 0 +fi + +# ----------------------------------------------------------------------------- +# create temp dir + +TMP_DIR= +chartsd_cleanup() { + cd /tmp + if [ ! -z "$TMP_DIR" -a -d "$TMP_DIR" ] + then + [ $debug -eq 1 ] && echo >&2 "$PROGRAM_NAME: cleaning up temporary directory $TMP_DIR ..." + rm -rf "$TMP_DIR" + fi + exit 0 +} +trap chartsd_cleanup EXIT +trap chartsd_cleanup SIGHUP +trap chartsd_cleanup INT + +if [ $UID = "0" ] +then + TMP_DIR="$( mktemp -d /var/run/netdata-${PROGRAM_NAME}-XXXXXXXXXX )" +else + TMP_DIR="$( mktemp -d /tmp/.netdata-${PROGRAM_NAME}-XXXXXXXXXX )" +fi + +cd "$TMP_DIR" || exit 1 + +# ----------------------------------------------------------------------------- +# create charts + +run_charts= +for chart in $active_charts +do + [ $debug -eq 1 ] && echo >&2 "$PROGRAM_NAME: Calling '$chart$charts_create()'..." + $chart$charts_create + if [ $? -eq 0 ] + then + run_charts="$run_charts $chart" + [ $debug -eq 1 ] && echo >&2 "$PROGRAM_NAME: '$chart' has initialized." + else + echo >&2 "$PROGRAM_NAME: chart '$chart' function '$chart$charts_create()' reports failure." + fi +done +[ $debug -eq 1 ] && echo >&2 "$PROGRAM_NAME: run_charts='$run_charts'" + + +# ----------------------------------------------------------------------------- +# update dimensions + +if [ -z "$run_charts" ] + then + echo >&2 "$PROGRAM_NAME: No charts to collect data from." + echo "DISABLE" + exit 1 +fi + +declare -A charts_last_update=() charts_update_every=() charts_next_update=() charts_run_counter=() +global_update() { + local exit_at \ + c=0 dt ret last_ms exec_start_ms exec_end_ms \ + chart now_charts=() next_charts=($run_charts) + + # return the current time in ms in $now_ms + current_time_ms + + exit_at=$(( now_ms + (restart_timeout * 1000) )) + + for chart in $run_charts + do + eval "charts_update_every[$chart]=\$$chart$suffix_update_every" + test -z "${charts_update_every[$chart]}" && charts_update_every[$charts]=$update_every + charts_last_update[$chart]=$((now_ms - (now_ms % (charts_update_every[$chart] * 1000) ) )) + charts_next_update[$chart]=$(( charts_last_update[$chart] + (charts_update_every[$chart] * 1000) )) + charts_run_counter[$chart]=0 + + echo "CHART netdata.plugin_chartsd_$chart '' 'Execution time for $chart plugin' 'milliseconds / run' charts.d netdata.plugin_charts area 145000 ${charts_update_every[$chart]}" + echo "DIMENSION run_time 'run time' absolute 1 1" + done + + # the main loop + while [ 1 ] + do + c=$((c + 1)) + now_charts=("${next_charts[@]}") + next_charts=() + + # return the current time in ms in $now_ms + current_time_ms + + for chart in "${now_charts[@]}" + do + # echo >&2 "DEBUG: chart: $chart last: ${charts_last_update[$chart]}, next: ${charts_next_update[$chart]}, now: ${now_ms}" + if [ ${now_ms} -ge ${charts_next_update[$chart]} ] + then + last_ms=${charts_last_update[$chart]} + dt=$(( (now_ms - last_ms) )) + # echo >&2 "DEBUG: chart: $chart last: ${charts_last_update[$chart]}, next: ${charts_next_update[$chart]}, now: ${now_ms}, dt: ${dt}" + + charts_last_update[$chart]=${now_ms} + + while [ ${charts_next_update[$chart]} -lt ${now_ms} ] + do + charts_next_update[$chart]=$(( charts_next_update[$chart] + (charts_update_every[$chart] * 1000) )) + done + + # the first call should not give a duration + # so that netdata calibrates to current time + dt=$(( dt * 1000 )) + charts_run_counter[$chart]=$(( charts_run_counter[$chart] + 1 )) + if [ ${charts_run_counter[$chart]} -eq 1 ] + then + dt= + fi + + exec_start_ms=$now_ms + $chart$charts_update $dt + ret=$? + + # return the current time in ms in $now_ms + current_time_ms; exec_end_ms=$now_ms + + echo "BEGIN netdata.plugin_chartsd_$chart $dt" + if [ $ret -eq 0 ] + then + echo "SET run_time = $(( exec_end_ms - exec_start_ms ))" + next_charts+=($chart) + else + echo "SET run_time = $(( (exec_end_ms - exec_start_ms) * -1 ))" + echo >&2 "$PROGRAM_NAME: chart '$chart' update() function reports failure. Disabling it." + fi + echo "END" + else + next_charts+=($chart) + fi + done + + if [ "$pause_method" = "suspend" ] + then + echo "STOPPING_WAKE_ME_UP_PLEASE" + suspend || ( echo >&2 "$PROGRAM_NAME: suspend returned error $?, falling back to sleep."; loopsleepms $debug_time $update_every $time_divisor) + else + # wait the time you are required to + #loopsleepms $debug_time $update_every $time_divisor + if [ ${LOOPSLEEPMS_HIGHRES} -eq 1 ] + then + sleep 0.2 + else + sleep 1 + fi + fi + + test ${now_ms} -ge ${exit_at} && exit 0 + done +} + +global_update diff --git a/plugins.d/loopsleepms.sh.inc b/plugins.d/loopsleepms.sh.inc new file mode 100644 index 000000000..2e22de3d8 --- /dev/null +++ b/plugins.d/loopsleepms.sh.inc @@ -0,0 +1,90 @@ +#!/bin/bash + +# this function is used to sleep a fraction of a second +# it calculates the difference between every time is called +# and tries to align the sleep time to give you exactly the +# loop you need. + +LOOPSLEEP_DATE="$(which date)" +if [ -z "$LOOPSLEEP_DATE" ] + then + echo >&2 "$0: ERROR: Cannot find the command 'date' in the system path." + exit 1 +fi + +LOOPSLEEPMS_LASTRUN=0 +LOOPSLEEPMS_LASTSLEEP=0 +LOOPSLEEPMS_LASTWORK=0 + +LOOPSLEEPMS_HIGHRES=1 +test "$($LOOPSLEEP_DATE +%N)" = "%N" && LOOPSLEEPMS_HIGHRES=0 + +now_ms= +current_time_ms() { + # if high resolution is not supported + # just sleep the time requested, in seconds + if [ $LOOPSLEEPMS_HIGHRES -eq 0 ] + then + now_ms="$($LOOPSLEEP_DATE +'%s')000" + else + now_ms="$(( $( $LOOPSLEEP_DATE +'%s * 1000 + %-N / 1000000' ) ))" + fi +} + +loopsleepms() { + local tellwork=0 t="$1" div s m now mstosleep + + if [ "$t" = "tellwork" ] + then + tellwork=1 + shift + t="$1" + fi + div="${2-100}" + + # $t = the time in seconds to wait + + # if high resolution is not supported + # just sleep the time requested, in seconds + if [ $LOOPSLEEPMS_HIGHRES -eq 0 ] + then + sleep $t + return + fi + + # get the current time, in ms + # milliseconds since epoch (1-1-1970) + now="$(( $( $LOOPSLEEP_DATE +'%s * 1000 + %-N / 1000000' ) ))" + + # calculate required sleep in ms + t=$((t * 1000 * div / 100)) + + # this is our first run + # just wait the requested time + test $LOOPSLEEPMS_LASTRUN -eq 0 && LOOPSLEEPMS_LASTRUN=$now + + # calculate ms since last run + LOOPSLEEPMS_LASTWORK=$((now - LOOPSLEEPMS_LASTRUN - LOOPSLEEPMS_LASTSLEEP)) + # echo "# last loop's work took $LOOPSLEEPMS_LASTWORK ms" + + # calculate ms to sleep + mstosleep=$(( t - LOOPSLEEPMS_LASTWORK )) + # echo "# mstosleep is $mstosleep ms" + + # if we are too slow, sleep some time + test $mstosleep -lt 200 && mstosleep=200 + + s=$((mstosleep / 1000)) + m=$((mstosleep - (s * 1000))) + + test $tellwork -eq 1 && echo >&2 " >>> PERFORMANCE >>> WORK TOOK $LOOPSLEEPMS_LASTWORK ms ( $((LOOPSLEEPMS_LASTWORK * 100 / 1000)).$((LOOPSLEEPMS_LASTWORK % 10))% cpu ) >>> SLEEPING $mstosleep ms" + + # echo "# sleeping $s.$m" + # echo + sleep $s.$m + + # keep the values we need + # for our next run + LOOPSLEEPMS_LASTRUN=$now + LOOPSLEEPMS_LASTSLEEP=$mstosleep +} diff --git a/plugins.d/node.d.plugin b/plugins.d/node.d.plugin new file mode 100755 index 000000000..a1fa754fa --- /dev/null +++ b/plugins.d/node.d.plugin @@ -0,0 +1,278 @@ +#!/bin/sh +':' //; exec "$(command -v nodejs || command -v node || command -v js || echo "ERROR node.js IS NOT AVAILABLE IN THIS SYSTEM")" "$0" "$@" + +// shebang hack from: +// http://unix.stackexchange.com/questions/65235/universal-node-js-shebang + +// Initially this is run as a shell script (#!/bin/sh). +// Then, the second line, finds nodejs or node or js in the system path +// and executes it with the shell parameters. + +// -------------------------------------------------------------------------------------------------------------------- + +'use strict'; + +// -------------------------------------------------------------------------------------------------------------------- +// get NETDATA environment variables + +var NETDATA_PLUGINS_DIR = process.env.NETDATA_PLUGINS_DIR || __dirname; +var NETDATA_CONFIG_DIR = process.env.NETDATA_CONFIG_DIR || '/etc/netdata'; +var NETDATA_UPDATE_EVERY = process.env.NETDATA_UPDATE_EVERY || 1; +var NODE_D_DIR = NETDATA_PLUGINS_DIR + '/../node.d'; + +// make sure the modules are found +process.mainModule.paths.unshift(NODE_D_DIR + '/node_modules'); +process.mainModule.paths.unshift(NODE_D_DIR); + + +// -------------------------------------------------------------------------------------------------------------------- +// load required modules + +var fs = require('fs'); +var url = require('url'); +var http = require('http'); +var path = require('path'); +var extend = require('extend'); +var netdata = require('netdata'); + + +// -------------------------------------------------------------------------------------------------------------------- +// configuration + +function pluginConfig(filename) { + var f = path.basename(filename); + + var m = f.match('.plugin' + '$'); + if(m === null) m = f.match('.node.js' + '$'); + if(m !== null) + return netdata.options.paths.config + '/' + f.substring(0, m.index) + '.conf'; + + return netdata.options.paths.config + '/' + f + '.conf'; +} + +// internal defaults +extend(true, netdata.options, { + filename: path.basename(__filename), + + update_every: NETDATA_UPDATE_EVERY, + + exit_after_ms: 3600 * 4 * 1000, + + paths: { + plugins: NETDATA_PLUGINS_DIR, + config: NETDATA_CONFIG_DIR, + modules: [], + }, + + modules_enable_autodetect: true, + modules_enable_all: true, + modules: {}, +}); +netdata.options.config_filename = pluginConfig(__filename); + +// load configuration file +try { + netdata.options_loaded = JSON.parse(fs.readFileSync(netdata.options.config_filename, 'utf8')); + extend(true, netdata.options, netdata.options_loaded); + console.error('merged netdata object:'); + console.error(netdata); +} +catch(e) { + netdata.error('Cannot read configuration file ' + netdata.options.config_filename + ': ' + e.message + ', using internal defaults.'); + netdata.options_loaded = undefined; + dumpError(e); +} + + +// apply module paths to node.js process +function applyModulePaths() { + var len = netdata.options.paths.modules.length; + while(len--) + process.mainModule.paths.unshift(netdata.options.paths.modules[len]); +} +applyModulePaths(); + + +// -------------------------------------------------------------------------------------------------------------------- +// tracing + +function dumpError(err) { + if (typeof err === 'object') { + if (err.stack) { + netdata.debug(err.stack); + } + } +} + +// -------------------------------------------------------------------------------------------------------------------- +// get command line arguments +{ + var found_myself = false; + var found_number = false; + var found_modules = false; + process.argv.forEach(function (val, index, array) { + netdata.debug('PARAM: ' + val); + + if(!found_myself) { + if(val === __filename) + found_myself = true; + } + else { + switch(val) { + case 'debug': + netdata.options.DEBUG = true; + netdata.debug('DEBUG enabled'); + break; + + default: + if(found_number === true) { + if(found_modules === false) { + for(var i in netdata.options.modules) + netdata.options.modules[i].enabled = false; + } + + if(typeof netdata.options.modules[val] === 'undefined') + netdata.options.modules[val] = {}; + + netdata.options.modules[val].enabled = true; + netdata.options.modules_enable_all = false; + netdata.debug('enabled module ' + val); + } + else { + try { + var x = parseInt(val); + if(x > 0) { + netdata.options.update_every = x; + if(netdata.options.update_every < NETDATA_UPDATE_EVERY) { + netdata.options.update_every = NETDATA_UPDATE_EVERY; + netdata.debug('Update frequency ' + x + 's is too low'); + } + + found_number = true; + netdata.debug('Update frequency set to ' + netdata.options.update_every + ' seconds'); + } + else netdata.error('Ignoring parameter: ' + val); + } + catch(e) { + netdata.error('Cannot get value of parameter: ' + val); + dumpError(e); + } + } + break; + } + } + }); +} + +if(netdata.options.update_every < 1) { + netdata.debug('Adjusting update frequency to 1 second'); + netdata.options.update_every = 1; +} + +// -------------------------------------------------------------------------------------------------------------------- +// find modules + +function findModules() { + var found = 0; + + var files = fs.readdirSync(NODE_D_DIR); + var len = files.length; + while(len--) { + var m = files[len].match('.node.js' + '$'); + if(m !== null) { + var n = files[len].substring(0, m.index); + + if(typeof(netdata.options.modules[n]) === 'undefined') + netdata.options.modules[n] = { name: n, enabled: netdata.options.modules_enable_all }; + + if(netdata.options.modules[n].enabled === true) { + netdata.options.modules[n].name = n; + netdata.options.modules[n].filename = NODE_D_DIR + '/' + files[len]; + netdata.options.modules[n].loaded = false; + + if(typeof(netdata.options.modules[n].config_filename) !== 'string') + netdata.options.modules[n].config_filename = pluginConfig(files[len]); + + // load the module + try { + netdata.debug('loading module ' + netdata.options.modules[n].filename); + netdata.options.modules[n].module = require(netdata.options.modules[n].filename); + netdata.options.modules[n].module.name = n; + netdata.debug('loaded module ' + netdata.options.modules[n].name + ' from ' + netdata.options.modules[n].filename); + } + catch(e) { + netdata.options.modules[n].enabled = false; + netdata.error('Cannot load module: ' + netdata.options.modules[n].filename + ' exception: ' + e); + dumpError(e); + continue; + } + + // load its configuration + var c = { + enable_autodetect: netdata.options.modules_enable_autodetect, + update_every: netdata.options.update_every + }; + try { + netdata.debug('loading module\'s ' + netdata.options.modules[n].name + ' config ' + netdata.options.modules[n].config_filename); + var c2 = JSON.parse(fs.readFileSync(netdata.options.modules[n].config_filename, 'utf8')); + extend(true, c, c2); + netdata.debug('loaded module\'s ' + netdata.options.modules[n].name + ' config ' + netdata.options.modules[n].config_filename); + } + catch(e) { + netdata.error('Cannot load module\'s ' + netdata.options.modules[n].name + ' config from ' + netdata.options.modules[n].config_filename + ' exception: ' + e + ', using internal defaults.'); + dumpError(e); + } + + // call module auto-detection / configuration + try { + netdata.modules_configuring++; + netdata.debug('Configuring module ' + netdata.options.modules[n].name); + var serv = netdata.configure(netdata.options.modules[n].module, c, function() { + netdata.debug('Configured module ' + netdata.options.modules[n].name); + netdata.modules_configuring--; + }); + + netdata.debug('Configuring module ' + netdata.options.modules[n].name + ' reports ' + serv + ' eligible services.'); + } + catch(e) { + netdata.modules_configuring--; + netdata.options.modules[n].enabled = false; + netdata.error('Failed module auto-detection: ' + netdata.options.modules[n].name + ' exception: ' + e + ', disabling module.'); + dumpError(e); + continue; + } + + netdata.options.modules[n].loaded = true; + found++; + } + } + } + + // netdata.debug(netdata.options.modules); + return found; +} + +if(findModules() === 0) { + netdata.error('Cannot load any .node.js module from: ' + NODE_D_DIR); + netdata.disableNodePlugin(); + process.exit(1); +} + + +// -------------------------------------------------------------------------------------------------------------------- +// start + +function start_when_configuring_ends() { + if(netdata.modules_configuring > 0) { + netdata.debug('Waiting modules configuration, still running ' + netdata.modules_configuring); + setTimeout(start_when_configuring_ends, 500); + return; + } + + netdata.modules_configuring = 0; + netdata.start(); +} +start_when_configuring_ends(); + +//netdata.debug('netdata object:') +//netdata.debug(netdata); diff --git a/plugins.d/tc-qos-helper.sh b/plugins.d/tc-qos-helper.sh new file mode 100755 index 000000000..86f717122 --- /dev/null +++ b/plugins.d/tc-qos-helper.sh @@ -0,0 +1,105 @@ +#!/bin/bash + +# default time function +now_ms= +current_time_ms() { + now_ms="$(date +'%s')000" +} + +# default sleep function +LOOPSLEEPMS_LASTWORK=0 +loopsleepms() { + [ "$1" = "tellwork" ] && shift + sleep $1 +} + +# if found and included, this file overwrites loopsleepms() +# with a high resolution timer function for precise looping. +. "$NETDATA_PLUGINS_DIR/loopsleepms.sh.inc" + +# check if we have a valid number for interval +t=$1 +sleep_time=$((t)) +[ $((sleep_time)) -lt 1 ] && $NETDATA_UPDATE_EVERY +[ $((sleep_time)) -lt 1 ] && sleep_time=1 + +tc_cmd="$(which tc)" +if [ -z "$tc_cmd" ] + then + echo >&2 "tc: Cannot find a 'tc' command in this system." + exit 1 +fi + +devices= +fix_names= + +setclassname() { + echo "SETCLASSNAME $3 $2" +} + +show_tc() { + local x="$1" + + echo "BEGIN $x" + $tc_cmd -s class show dev $x + + # check FireQOS names for classes + if [ ! -z "$fix_names" -a -f /var/run/fireqos/ifaces/$x ] + then + name="$(cat /var/run/fireqos/ifaces/$x)" + echo "SETDEVICENAME $name" + + interface_classes= + interface_classes_monitor= + . /var/run/fireqos/$name.conf + for n in $interface_classes_monitor + do + setclassname $(echo $n | tr '|' ' ') + done + echo "SETDEVICEGROUP $interface_dev" + fi + echo "END $x" +} + +all_devices() { + cat /proc/net/dev | grep ":" | cut -d ':' -f 1 | while read dev + do + l=$($tc_cmd class show dev $dev | wc -l) + [ $l -ne 0 ] && echo $dev + done +} + +# update devices and class names +# once every 2 minutes +names_every=$((120 / sleep_time)) + +# exit this script every hour +# it will be restarted automatically +exit_after=$((3600 / sleep_time)) + +c=0 +gc=0 +while [ 1 ] +do + fix_names= + c=$((c + 1)) + gc=$((gc + 1)) + + if [ $c -le 1 -o $c -ge $names_every ] + then + c=1 + fix_names="YES" + devices="$( all_devices )" + fi + + for d in $devices + do + show_tc $d + done + + echo "WORKTIME $LOOPSLEEPMS_LASTWORK" + + loopsleepms $sleep_time + + [ $gc -gt $exit_after ] && exit 0 +done |