diff options
Diffstat (limited to '')
66 files changed, 18641 insertions, 0 deletions
diff --git a/src/.keep b/src/.keep new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/src/.keep diff --git a/src/Makefile.am b/src/Makefile.am new file mode 100644 index 000000000..0c51bb54b --- /dev/null +++ b/src/Makefile.am @@ -0,0 +1,90 @@ +# +# Copyright (C) 2015 Alon Bar-Lev <alon.barlev@gmail.com> +# +MAINTAINERCLEANFILES= $(srcdir)/Makefile.in + +AM_CPPFLAGS = \ + -DCACHE_DIR="\"$(cachedir)\"" \ + -DCONFIG_DIR="\"$(configdir)\"" \ + -DLOG_DIR="\"$(logdir)\"" \ + -DPLUGINS_DIR="\"$(pluginsdir)\"" \ + -DWEB_DIR="\"$(webdir)\"" \ + $(NULL) +AM_CFLAGS = \ + $(OPTIONAL_MATH_CFLAGS) \ + $(OPTIONAL_NFACCT_CLFAGS) \ + $(OPTIONAL_ZLIB_CFLAGS) \ + $(NULL) + +sbin_PROGRAMS = netdata +dist_cache_DATA = .keep +dist_log_DATA = .keep +plugins_PROGRAMS = apps.plugin + +netdata_SOURCES = \ + appconfig.c appconfig.h \ + avl.c avl.h \ + common.c common.h \ + daemon.c daemon.h \ + dictionary.c dictionary.h \ + global_statistics.c global_statistics.h \ + log.c log.h \ + main.c main.h \ + plugin_checks.c plugin_checks.h \ + plugin_idlejitter.c plugin_idlejitter.h \ + plugin_nfacct.c plugin_nfacct.h \ + plugin_proc.c plugin_proc.h \ + plugin_tc.c plugin_tc.h \ + plugins_d.c plugins_d.h \ + popen.c popen.h \ + proc_diskstats.c \ + proc_interrupts.c \ + proc_softirqs.c \ + proc_loadavg.c \ + proc_meminfo.c \ + proc_net_dev.c \ + proc_net_ip_vs_stats.c \ + proc_net_netstat.c \ + proc_net_rpc_nfsd.c \ + proc_net_snmp.c \ + proc_net_stat_conntrack.c \ + proc_stat.c \ + proc_sys_kernel_random_entropy_avail.c \ + proc_vmstat.c \ + sys_kernel_mm_ksm.c \ + procfile.c procfile.h \ + rrd.c rrd.h \ + rrd2json.c rrd2json.h \ + storage_number.c storage_number.h \ + unit_test.c unit_test.h \ + url.c url.h \ + web_buffer.c web_buffer.h \ + web_client.c web_client.h \ + web_server.c web_server.h \ + $(NULL) +netdata_LDADD = \ + $(OPTIONAL_MATH_LIBS) \ + $(OPTIONAL_NFACCT_LIBS) \ + $(OPTIONAL_ZLIB_LIBS) \ + $(NULL) + +apps_plugin_SOURCES = \ + apps_plugin.c \ + avl.c avl.h \ + common.c common.h \ + log.c log.h \ + procfile.c procfile.h \ + $(NULL) + +install-data-hook: + if [ `id -u` == 0 ]; then \ + chown root '$(DESTDIR)$(pluginsdir)/apps.plugin' && \ + chmod 4755 '$(DESTDIR)$(pluginsdir)/apps.plugin'; \ + else \ + echo; \ + echo "ATTENTION"; \ + echo; \ + echo "setuid bit of $(pluginsdir)/apps.plugin must be set, please execute as root:"; \ + echo "chown root '$(pluginsdir)/apps.plugin' && chmod 4755 '$(pluginsdir)/apps.plugin'"; \ + echo; \ + fi diff --git a/src/appconfig.c b/src/appconfig.c new file mode 100755 index 000000000..49eca90f0 --- /dev/null +++ b/src/appconfig.c @@ -0,0 +1,485 @@ +#ifdef HAVE_CONFIG_H +#include <config.h> +#endif +#include <pthread.h> +#include <stdlib.h> +#include <string.h> + +#include "avl.h" +#include "common.h" +#include "appconfig.h" +#include "log.h" + +#define CONFIG_FILE_LINE_MAX ((CONFIG_MAX_NAME + CONFIG_MAX_VALUE + 1024) * 2) + +pthread_rwlock_t config_rwlock = PTHREAD_RWLOCK_INITIALIZER; + +// ---------------------------------------------------------------------------- +// definitions + +#define CONFIG_VALUE_LOADED 0x01 // has been loaded from the config +#define CONFIG_VALUE_USED 0x02 // has been accessed from the program +#define CONFIG_VALUE_CHANGED 0x04 // has been changed from the loaded value +#define CONFIG_VALUE_CHECKED 0x08 // has been checked if the value is different from the default + +struct config_value { + avl avl; // the index - this has to be first! + + uint32_t hash; // a simple hash to speed up searching + // we first compare hashes, and only if the hashes are equal we do string comparisons + + char *name; + char *value; + + uint8_t flags; + + struct config_value *next; +}; + +struct config { + avl avl; + + uint32_t hash; // a simple hash to speed up searching + // we first compare hashes, and only if the hashes are equal we do string comparisons + + char *name; + + struct config_value *values; + avl_tree values_index; + + struct config *next; + + pthread_rwlock_t rwlock; +} *config_root = NULL; + + +// ---------------------------------------------------------------------------- +// config value + +static int config_value_iterator(avl *a) { if(a) {}; return 0; } + +static int config_value_compare(void* a, void* b) { + if(((struct config_value *)a)->hash < ((struct config_value *)b)->hash) return -1; + else if(((struct config_value *)a)->hash > ((struct config_value *)b)->hash) return 1; + else return strcmp(((struct config_value *)a)->name, ((struct config_value *)b)->name); +} + +#define config_value_index_add(co, cv) avl_insert(&((co)->values_index), (avl *)(cv)) +#define config_value_index_del(co, cv) avl_remove(&((co)->values_index), (avl *)(cv)) + +static struct config_value *config_value_index_find(struct config *co, const char *name, uint32_t hash) { + struct config_value *result = NULL, tmp; + tmp.hash = (hash)?hash:simple_hash(name); + tmp.name = (char *)name; + + avl_search(&(co->values_index), (avl *)&tmp, config_value_iterator, (avl **)&result); + return result; +} + +// ---------------------------------------------------------------------------- +// config + +static int config_iterator(avl *a) { if(a) {}; return 0; } + +static int config_compare(void* a, void* b) { + if(((struct config *)a)->hash < ((struct config *)b)->hash) return -1; + else if(((struct config *)a)->hash > ((struct config *)b)->hash) return 1; + else return strcmp(((struct config *)a)->name, ((struct config *)b)->name); +} + +avl_tree config_root_index = { + NULL, + config_compare, +#ifdef AVL_LOCK_WITH_MUTEX + PTHREAD_MUTEX_INITIALIZER +#else + PTHREAD_RWLOCK_INITIALIZER +#endif +}; + +#define config_index_add(cfg) avl_insert(&config_root_index, (avl *)(cfg)) +#define config_index_del(cfg) avl_remove(&config_root_index, (avl *)(cfg)) + +static struct config *config_index_find(const char *name, uint32_t hash) { + struct config *result = NULL, tmp; + tmp.hash = (hash)?hash:simple_hash(name); + tmp.name = (char *)name; + + avl_search(&config_root_index, (avl *)&tmp, config_iterator, (avl **)&result); + return result; +} + +struct config_value *config_value_create(struct config *co, const char *name, const char *value) +{ + debug(D_CONFIG, "Creating config entry for name '%s', value '%s', in section '%s'.", name, value, co->name); + + struct config_value *cv = calloc(1, sizeof(struct config_value)); + if(!cv) fatal("Cannot allocate config_value"); + + cv->name = strdup(name); + if(!cv->name) fatal("Cannot allocate config.name"); + cv->hash = simple_hash(cv->name); + + cv->value = strdup(value); + if(!cv->value) fatal("Cannot allocate config.value"); + + config_value_index_add(co, cv); + + // no need for string termination, due to calloc() + + pthread_rwlock_wrlock(&co->rwlock); + + struct config_value *cv2 = co->values; + if(cv2) { + while (cv2->next) cv2 = cv2->next; + cv2->next = cv; + } + else co->values = cv; + + pthread_rwlock_unlock(&co->rwlock); + + return cv; +} + +struct config *config_create(const char *section) +{ + debug(D_CONFIG, "Creating section '%s'.", section); + + struct config *co = calloc(1, sizeof(struct config)); + if(!co) fatal("Cannot allocate config"); + + co->name = strdup(section); + if(!co->name) fatal("Cannot allocate config.name"); + co->hash = simple_hash(co->name); + + pthread_rwlock_init(&co->rwlock, NULL); + avl_init(&co->values_index, config_value_compare); + + config_index_add(co); + + // no need for string termination, due to calloc() + + pthread_rwlock_wrlock(&config_rwlock); + + struct config *co2 = config_root; + if(co2) { + while (co2->next) co2 = co2->next; + co2->next = co; + } + else config_root = co; + + pthread_rwlock_unlock(&config_rwlock); + + return co; +} + +struct config *config_find_section(const char *section) +{ + return config_index_find(section, 0); +} + +int load_config(char *filename, int overwrite_used) +{ + int line = 0; + struct config *co = NULL; + + char buffer[CONFIG_FILE_LINE_MAX + 1], *s; + + if(!filename) filename = CONFIG_DIR "/" CONFIG_FILENAME; + FILE *fp = fopen(filename, "r"); + if(!fp) { + error("Cannot open file '%s'", CONFIG_DIR "/" CONFIG_FILENAME); + return 0; + } + + while(fgets(buffer, CONFIG_FILE_LINE_MAX, fp) != NULL) { + buffer[CONFIG_FILE_LINE_MAX] = '\0'; + line++; + + s = trim(buffer); + if(!s) { + debug(D_CONFIG, "Ignoring line %d, it is empty.", line); + continue; + } + + int len = (int) strlen(s); + if(*s == '[' && s[len - 1] == ']') { + // new section + s[len - 1] = '\0'; + s++; + + co = config_find_section(s); + if(!co) co = config_create(s); + + continue; + } + + if(!co) { + // line outside a section + error("Ignoring line %d ('%s'), it is outsize all sections.", line, s); + continue; + } + + char *name = s; + char *value = strchr(s, '='); + if(!value) { + error("Ignoring line %d ('%s'), there is no = in it.", line, s); + continue; + } + *value = '\0'; + value++; + + name = trim(name); + value = trim(value); + + if(!name) { + error("Ignoring line %d, name is empty.", line); + continue; + } + if(!value) { + debug(D_CONFIG, "Ignoring line %d, value is empty.", line); + continue; + } + + struct config_value *cv = config_value_index_find(co, name, 0); + + if(!cv) cv = config_value_create(co, name, value); + else { + if(((cv->flags & CONFIG_VALUE_USED) && overwrite_used) || !(cv->flags & CONFIG_VALUE_USED)) { + debug(D_CONFIG, "Overwriting '%s/%s'.", line, co->name, cv->name); + free(cv->value); + cv->value = strdup(value); + if(!cv->value) fatal("Cannot allocate config.value"); + } + else + debug(D_CONFIG, "Ignoring line %d, '%s/%s' is already present and used.", line, co->name, cv->name); + } + cv->flags |= CONFIG_VALUE_LOADED; + } + + fclose(fp); + + return 1; +} + +char *config_get(const char *section, const char *name, const char *default_value) +{ + struct config_value *cv; + + debug(D_CONFIG, "request to get config in section '%s', name '%s', default_value '%s'", section, name, default_value); + + struct config *co = config_find_section(section); + if(!co) co = config_create(section); + + cv = config_value_index_find(co, name, 0); + if(!cv) { + cv = config_value_create(co, name, default_value); + if(!cv) return NULL; + } + cv->flags |= CONFIG_VALUE_USED; + + if((cv->flags & CONFIG_VALUE_LOADED) || (cv->flags & CONFIG_VALUE_CHANGED)) { + // this is a loaded value from the config file + // if it is different that the default, mark it + if(!(cv->flags & CONFIG_VALUE_CHECKED)) { + if(strcmp(cv->value, default_value) != 0) cv->flags |= CONFIG_VALUE_CHANGED; + cv->flags |= CONFIG_VALUE_CHECKED; + } + } + + return(cv->value); +} + +long long config_get_number(const char *section, const char *name, long long value) +{ + char buffer[100], *s; + sprintf(buffer, "%lld", value); + + s = config_get(section, name, buffer); + if(!s) return value; + + return strtoll(s, NULL, 0); +} + +int config_get_boolean(const char *section, const char *name, int value) +{ + char *s; + if(value) s = "yes"; + else s = "no"; + + s = config_get(section, name, s); + if(!s) return value; + + if(!strcmp(s, "yes")) return 1; + else return 0; +} + +int config_get_boolean_ondemand(const char *section, const char *name, int value) +{ + char *s; + + if(value == CONFIG_ONDEMAND_ONDEMAND) + s = "on demand"; + + else if(value == CONFIG_ONDEMAND_NO) + s = "no"; + + else + s = "yes"; + + s = config_get(section, name, s); + if(!s) return value; + + if(!strcmp(s, "yes")) + return CONFIG_ONDEMAND_YES; + else if(!strcmp(s, "no")) + return CONFIG_ONDEMAND_NO; + else if(!strcmp(s, "on demand")) + return CONFIG_ONDEMAND_ONDEMAND; + + return value; +} + +const char *config_set_default(const char *section, const char *name, const char *value) +{ + struct config_value *cv; + + debug(D_CONFIG, "request to set config in section '%s', name '%s', value '%s'", section, name, value); + + struct config *co = config_find_section(section); + if(!co) return config_set(section, name, value); + + cv = config_value_index_find(co, name, 0); + if(!cv) return config_set(section, name, value); + + cv->flags |= CONFIG_VALUE_USED; + + if(cv->flags & CONFIG_VALUE_LOADED) + return cv->value; + + if(strcmp(cv->value, value) != 0) { + cv->flags |= CONFIG_VALUE_CHANGED; + + free(cv->value); + cv->value = strdup(value); + if(!cv->value) fatal("Cannot allocate config.value"); + } + + return cv->value; +} + +const char *config_set(const char *section, const char *name, const char *value) +{ + struct config_value *cv; + + debug(D_CONFIG, "request to set config in section '%s', name '%s', value '%s'", section, name, value); + + struct config *co = config_find_section(section); + if(!co) co = config_create(section); + + cv = config_value_index_find(co, name, 0); + if(!cv) cv = config_value_create(co, name, value); + cv->flags |= CONFIG_VALUE_USED; + + if(strcmp(cv->value, value) != 0) { + cv->flags |= CONFIG_VALUE_CHANGED; + + free(cv->value); + cv->value = strdup(value); + if(!cv->value) fatal("Cannot allocate config.value"); + } + + return value; +} + +long long config_set_number(const char *section, const char *name, long long value) +{ + char buffer[100]; + sprintf(buffer, "%lld", value); + + config_set(section, name, buffer); + + return value; +} + +int config_set_boolean(const char *section, const char *name, int value) +{ + char *s; + if(value) s = "yes"; + else s = "no"; + + config_set(section, name, s); + + return value; +} + +void generate_config(BUFFER *wb, int only_changed) +{ + int i, pri; + struct config *co; + struct config_value *cv; + + for(i = 0; i < 3 ;i++) { + switch(i) { + case 0: + buffer_strcat(wb, + "# NetData Configuration\n" + "# You can uncomment and change any of the options below.\n" + "# The value shown in the commented settings, is the default value.\n" + "\n# global netdata configuration\n"); + break; + + case 1: + buffer_strcat(wb, "\n\n# per plugin configuration\n"); + break; + + case 2: + buffer_strcat(wb, "\n\n# per chart configuration\n"); + break; + } + + pthread_rwlock_wrlock(&config_rwlock); + for(co = config_root; co ; co = co->next) { + if(strcmp(co->name, "global") == 0 || strcmp(co->name, "plugins") == 0) pri = 0; + else if(strncmp(co->name, "plugin:", 7) == 0) pri = 1; + else pri = 2; + + if(i == pri) { + int used = 0; + int changed = 0; + int count = 0; + + pthread_rwlock_wrlock(&co->rwlock); + + for(cv = co->values; cv ; cv = cv->next) { + used += (cv->flags && CONFIG_VALUE_USED)?1:0; + changed += (cv->flags & CONFIG_VALUE_CHANGED)?1:0; + count++; + } + + pthread_rwlock_unlock(&co->rwlock); + + if(!count) continue; + if(only_changed && !changed) continue; + + if(!used) { + buffer_sprintf(wb, "\n# node '%s' is not used.", co->name); + } + + buffer_sprintf(wb, "\n[%s]\n", co->name); + + pthread_rwlock_wrlock(&co->rwlock); + for(cv = co->values; cv ; cv = cv->next) { + + if(used && !(cv->flags & CONFIG_VALUE_USED)) { + buffer_sprintf(wb, "\n\t# option '%s' is not used.\n", cv->name); + } + buffer_sprintf(wb, "\t%s%s = %s\n", ((!(cv->flags & CONFIG_VALUE_CHANGED)) && (cv->flags & CONFIG_VALUE_USED))?"# ":"", cv->name, cv->value); + } + pthread_rwlock_unlock(&co->rwlock); + } + } + pthread_rwlock_unlock(&config_rwlock); + } +} + diff --git a/src/appconfig.h b/src/appconfig.h new file mode 100755 index 000000000..41d1e19bb --- /dev/null +++ b/src/appconfig.h @@ -0,0 +1,31 @@ +#include "web_buffer.h" + +#ifndef NETDATA_CONFIG_H +#define NETDATA_CONFIG_H 1 + +#define CONFIG_FILENAME "netdata.conf" + +// these are used to limit the configuration names and values lengths +// they are not enforced by config.c functions (they will strdup() all strings, no matter of their length) +#define CONFIG_MAX_NAME 1024 +#define CONFIG_MAX_VALUE 2048 + +extern int load_config(char *filename, int overwrite_used); + +extern char *config_get(const char *section, const char *name, const char *default_value); +extern long long config_get_number(const char *section, const char *name, long long value); +extern int config_get_boolean(const char *section, const char *name, int value); + +#define CONFIG_ONDEMAND_NO 0 +#define CONFIG_ONDEMAND_YES 1 +#define CONFIG_ONDEMAND_ONDEMAND 2 +extern int config_get_boolean_ondemand(const char *section, const char *name, int value); + +extern const char *config_set(const char *section, const char *name, const char *value); +extern const char *config_set_default(const char *section, const char *name, const char *value); +extern long long config_set_number(const char *section, const char *name, long long value); +extern int config_set_boolean(const char *section, const char *name, int value); + +extern void generate_config(BUFFER *wb, int only_changed); + +#endif /* NETDATA_CONFIG_H */ diff --git a/src/apps_plugin.c b/src/apps_plugin.c new file mode 100755 index 000000000..153687196 --- /dev/null +++ b/src/apps_plugin.c @@ -0,0 +1,2002 @@ +// TODO +// +// 1. disable RESET_OR_OVERFLOW check in charts + +#ifdef HAVE_CONFIG_H +#include <config.h> +#endif +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <time.h> +#include <unistd.h> +#include <sys/types.h> +#include <sys/time.h> +#include <sys/wait.h> + +#include <sys/resource.h> +#include <sys/stat.h> + +#include <errno.h> +#include <stdarg.h> +#include <locale.h> +#include <ctype.h> +#include <fcntl.h> + +#include <malloc.h> +#include <dirent.h> +#include <arpa/inet.h> + +#include "common.h" +#include "log.h" +#include "avl.h" +#include "procfile.h" + +#define MAX_COMPARE_NAME 15 +#define MAX_NAME 100 + +unsigned long long Hertz = 1; + +long processors = 1; +long pid_max = 32768; +int debug = 0; + +int update_every = 1; +unsigned long long file_counter = 0; + +char *host_prefix = ""; + +// ---------------------------------------------------------------------------- +// memory debugger + +struct allocations { + size_t allocations; + size_t allocated; + size_t allocated_max; +} allocations = { 0, 0, 0 }; + +#define MALLOC_MARK (uint32_t)(0x0BADCAFE) +#define MALLOC_PREFIX (sizeof(uint32_t) * 2) +#define MALLOC_SUFFIX (sizeof(uint32_t)) +#define MALLOC_OVERHEAD (MALLOC_PREFIX + MALLOC_SUFFIX) + +void *mark_allocation(void *allocated_ptr, size_t size_without_overheads) { + uint32_t *real_ptr = (uint32_t *)allocated_ptr; + real_ptr[0] = MALLOC_MARK; + real_ptr[1] = (uint32_t) size_without_overheads; + + uint32_t *end_ptr = (uint32_t *)(allocated_ptr + MALLOC_PREFIX + size_without_overheads); + end_ptr[0] = MALLOC_MARK; + + // fprintf(stderr, "MEMORY_POINTER: Allocated at %p, returning %p.\n", allocated_ptr, (void *)(allocated_ptr + MALLOC_PREFIX)); + + return allocated_ptr + MALLOC_PREFIX; +} + +void *check_allocation(const char *file, int line, const char *function, void *marked_ptr, size_t *size_without_overheads_ptr) { + uint32_t *real_ptr = (uint32_t *)(marked_ptr - MALLOC_PREFIX); + + // fprintf(stderr, "MEMORY_POINTER: Checking pointer at %p, real %p for %s/%u@%s.\n", marked_ptr, (void *)(marked_ptr - MALLOC_PREFIX), function, line, file); + + if(real_ptr[0] != MALLOC_MARK) fatal("MEMORY: prefix MARK is not valid for %s/%u@%s.", function, line, file); + + size_t size = real_ptr[1]; + + uint32_t *end_ptr = (uint32_t *)(marked_ptr + size); + if(end_ptr[0] != MALLOC_MARK) fatal("MEMORY: suffix MARK of allocation with size %zu is not valid for %s/%u@%s.", size, function, line, file); + + if(size_without_overheads_ptr) *size_without_overheads_ptr = size; + + return real_ptr; +} + +void *malloc_debug(const char *file, int line, const char *function, size_t size) { + void *ptr = malloc(size + MALLOC_OVERHEAD); + if(!ptr) fatal("MEMORY: Cannot allocate %zu bytes for %s/%u@%s.", size, function, line, file); + + allocations.allocated += size; + allocations.allocations++; + + debug(D_MEMORY, "MEMORY: Allocated %zu bytes for %s/%u@%s." + " Status: allocated %zu in %zu allocs." + , size + , function, line, file + , allocations.allocated + , allocations.allocations + ); + + if(allocations.allocated > allocations.allocated_max) { + debug(D_MEMORY, "MEMORY: total allocation peak increased from %zu to %zu", allocations.allocated_max, allocations.allocated); + allocations.allocated_max = allocations.allocated; + } + + size_t csize; + check_allocation(file, line, function, mark_allocation(ptr, size), &csize); + if(size != csize) { + fatal("Invalid size."); + } + + return mark_allocation(ptr, size); +} + +void *calloc_debug(const char *file, int line, const char *function, size_t nmemb, size_t size) { + void *ptr = malloc_debug(file, line, function, (nmemb * size)); + bzero(ptr, nmemb * size); + return ptr; +} + +void free_debug(const char *file, int line, const char *function, void *ptr) { + size_t size; + void *real_ptr = check_allocation(file, line, function, ptr, &size); + + bzero(real_ptr, size + MALLOC_OVERHEAD); + + free(real_ptr); + allocations.allocated -= size; + allocations.allocations--; + + debug(D_MEMORY, "MEMORY: freed %zu bytes for %s/%u@%s." + " Status: allocated %zu in %zu allocs." + , size + , function, line, file + , allocations.allocated + , allocations.allocations + ); +} + +void *realloc_debug(const char *file, int line, const char *function, void *ptr, size_t size) { + if(!ptr) return malloc_debug(file, line, function, size); + if(!size) { free_debug(file, line, function, ptr); return NULL; } + + size_t old_size; + void *real_ptr = check_allocation(file, line, function, ptr, &old_size); + + void *new_ptr = realloc(real_ptr, size + MALLOC_OVERHEAD); + if(!new_ptr) fatal("MEMORY: Cannot allocate %zu bytes for %s/%u@%s.", size, function, line, file); + + allocations.allocated += size; + allocations.allocated -= old_size; + + debug(D_MEMORY, "MEMORY: Re-allocated from %zu to %zu bytes for %s/%u@%s." + " Status: allocated %z in %zu allocs." + , old_size, size + , function, line, file + , allocations.allocated + , allocations.allocations + ); + + if(allocations.allocated > allocations.allocated_max) { + debug(D_MEMORY, "MEMORY: total allocation peak increased from %zu to %zu", allocations.allocated_max, allocations.allocated); + allocations.allocated_max = allocations.allocated; + } + + return mark_allocation(new_ptr, size); +} + +char *strdup_debug(const char *file, int line, const char *function, const char *ptr) { + size_t size = 0; + const char *s = ptr; + + while(*s++) size++; + size++; + + char *p = malloc_debug(file, line, function, size); + if(!p) fatal("Cannot allocate %zu bytes.", size); + + memcpy(p, ptr, size); + return p; +} + +#define malloc(size) malloc_debug(__FILE__, __LINE__, __FUNCTION__, (size)) +#define calloc(nmemb, size) calloc_debug(__FILE__, __LINE__, __FUNCTION__, (nmemb), (size)) +#define realloc(ptr, size) realloc_debug(__FILE__, __LINE__, __FUNCTION__, (ptr), (size)) +#define free(ptr) free_debug(__FILE__, __LINE__, __FUNCTION__, (ptr)) + +#ifdef strdup +#undef strdup +#endif +#define strdup(ptr) strdup_debug(__FILE__, __LINE__, __FUNCTION__, (ptr)) + +// ---------------------------------------------------------------------------- +// helper functions + +procfile *ff = NULL; + +long get_processors(void) { + int processors = 0; + + char filename[FILENAME_MAX + 1]; + snprintf(filename, FILENAME_MAX, "%s/proc/stat", host_prefix); + + ff = procfile_reopen(ff, filename, "", PROCFILE_FLAG_DEFAULT); + if(!ff) return 1; + + ff = procfile_readall(ff); + if(!ff) { + // procfile_close(ff); + return 1; + } + + unsigned int i; + for(i = 0; i < procfile_lines(ff); i++) { + if(!procfile_linewords(ff, i)) continue; + + if(strncmp(procfile_lineword(ff, i, 0), "cpu", 3) == 0) processors++; + } + processors--; + if(processors < 1) processors = 1; + + // procfile_close(ff); + return processors; +} + +long get_pid_max(void) { + long mpid = 32768; + + char filename[FILENAME_MAX + 1]; + snprintf(filename, FILENAME_MAX, "%s/proc/sys/kernel/pid_max", host_prefix); + ff = procfile_reopen(ff, filename, "", PROCFILE_FLAG_DEFAULT); + if(!ff) return mpid; + + ff = procfile_readall(ff); + if(!ff) { + // procfile_close(ff); + return mpid; + } + + mpid = atol(procfile_lineword(ff, 0, 0)); + if(!mpid) mpid = 32768; + + // procfile_close(ff); + return mpid; +} + +unsigned long long get_hertz(void) +{ + unsigned long long myhz = 1; + +#ifdef _SC_CLK_TCK + if((myhz = (unsigned long long int) sysconf(_SC_CLK_TCK)) > 0) { + return myhz; + } +#endif + +#ifdef HZ + myhz = HZ; /* <asm/param.h> */ +#else + /* If 32-bit or big-endian (not Alpha or ia64), assume HZ is 100. */ + hz = (sizeof(long)==sizeof(int) || htons(999)==999) ? 100UL : 1024UL; +#endif + + error("Unknown HZ value. Assuming %llu.", myhz); + return myhz; +} + + +// ---------------------------------------------------------------------------- +// target +// target is the point to aggregate a process tree values + +struct target { + char compare[MAX_COMPARE_NAME + 1]; + char id[MAX_NAME + 1]; + char name[MAX_NAME + 1]; + + unsigned long long minflt; + unsigned long long cminflt; + unsigned long long majflt; + unsigned long long cmajflt; + unsigned long long utime; + unsigned long long stime; + unsigned long long cutime; + unsigned long long cstime; + unsigned long long num_threads; + unsigned long long rss; + + unsigned long long fix_minflt; + unsigned long long fix_cminflt; + unsigned long long fix_majflt; + unsigned long long fix_cmajflt; + unsigned long long fix_utime; + unsigned long long fix_stime; + unsigned long long fix_cutime; + unsigned long long fix_cstime; + + unsigned long long statm_size; + unsigned long long statm_resident; + unsigned long long statm_share; + unsigned long long statm_text; + unsigned long long statm_lib; + unsigned long long statm_data; + unsigned long long statm_dirty; + + unsigned long long io_logical_bytes_read; + unsigned long long io_logical_bytes_written; + unsigned long long io_read_calls; + unsigned long long io_write_calls; + unsigned long long io_storage_bytes_read; + unsigned long long io_storage_bytes_written; + unsigned long long io_cancelled_write_bytes; + + unsigned long long fix_io_logical_bytes_read; + unsigned long long fix_io_logical_bytes_written; + unsigned long long fix_io_read_calls; + unsigned long long fix_io_write_calls; + unsigned long long fix_io_storage_bytes_read; + unsigned long long fix_io_storage_bytes_written; + unsigned long long fix_io_cancelled_write_bytes; + + int *fds; + unsigned long long openfiles; + unsigned long long openpipes; + unsigned long long opensockets; + unsigned long long openinotifies; + unsigned long long openeventfds; + unsigned long long opentimerfds; + unsigned long long opensignalfds; + unsigned long long openeventpolls; + unsigned long long openother; + + unsigned long processes; // how many processes have been merged to this + int exposed; // if set, we have sent this to netdata + int hidden; // if set, we set the hidden flag on the dimension + int debug; + + struct target *target; // the one that will be reported to netdata + struct target *next; +} *target_root = NULL, *default_target = NULL; + +long targets = 0; + +// find or create a target +// there are targets that are just agregated to other target (the second argument) +struct target *get_target(const char *id, struct target *target) +{ + const char *nid = id; + if(nid[0] == '-') nid++; + + struct target *w; + for(w = target_root ; w ; w = w->next) + if(strncmp(nid, w->id, MAX_NAME) == 0) return w; + + w = calloc(sizeof(struct target), 1); + if(!w) { + error("Cannot allocate %lu bytes of memory", (unsigned long)sizeof(struct target)); + return NULL; + } + + strncpy(w->id, nid, MAX_NAME); + strncpy(w->name, nid, MAX_NAME); + strncpy(w->compare, nid, MAX_COMPARE_NAME); + if(id[0] == '-') w->hidden = 1; + + w->target = target; + + w->next = target_root; + target_root = w; + + if(debug) fprintf(stderr, "apps.plugin: adding hook for process '%s', compare '%s' on target '%s'\n", w->id, w->compare, w->target?w->target->id:""); + + return w; +} + +// read the process groups file +int read_process_groups(const char *name) +{ + char buffer[4096+1]; + char filename[FILENAME_MAX + 1]; + + snprintf(filename, FILENAME_MAX, "%s/apps_%s.conf", CONFIG_DIR, name); + + if(debug) fprintf(stderr, "apps.plugin: process groups file: '%s'\n", filename); + FILE *fp = fopen(filename, "r"); + if(!fp) { + error("Cannot open file '%s'", filename); + return 1; + } + + long line = 0; + while(fgets(buffer, 4096, fp) != NULL) { + int whidden = 0, wdebug = 0; + line++; + + // if(debug) fprintf(stderr, "apps.plugin: \tread %s\n", buffer); + + char *s = buffer, *t, *p; + s = trim(s); + if(!s || !*s || *s == '#') continue; + + if(debug) fprintf(stderr, "apps.plugin: \tread %s\n", s); + + // the target name + t = strsep(&s, ":"); + if(t) t = trim(t); + if(!t || !*t) continue; + + while(t[0]) { + int stop = 1; + + switch(t[0]) { + case '-': + stop = 0; + whidden = 1; + t++; + break; + + case '+': + stop = 0; + wdebug = 1; + t++; + break; + } + + if(stop) break; + } + + if(debug) fprintf(stderr, "apps.plugin: \t\ttarget %s\n", t); + + struct target *w = NULL; + long count = 0; + int blen = 0; + char buffer[4097] = ""; + buffer[4096] = '\0'; + + // the process names + while((p = strsep(&s, " "))) { + p = trim(p); + if(!p || !*p) continue; + + strncpy(&buffer[blen], p, 4096 - blen); + blen = strlen(buffer); + + while(buffer[blen - 1] == '\\') { + buffer[blen - 1] = ' '; + + if((p = strsep(&s, " "))) + p = trim(p); + + if(!p || !*p) p = " "; + strncpy(&buffer[blen], p, 4096 - blen); + blen = strlen(buffer); + } + + struct target *n = get_target(buffer, w); + n->hidden = whidden; + n->debug = wdebug; + if(!w) w = n; + + buffer[0] = '\0'; + blen = 0; + + count++; + } + + if(w) strncpy(w->name, t, MAX_NAME); + if(!count) error("The line %ld on file '%s', for group '%s' does not state any process names.", line, filename, t); + } + fclose(fp); + + default_target = get_target("+p!o@w#e$i^r&7*5(-i)l-o_", NULL); // match nothing + strncpy(default_target->name, "other", MAX_NAME); + + return 0; +} + + +// ---------------------------------------------------------------------------- +// data to store for each pid +// see: man proc + +struct pid_stat { + int32_t pid; + char comm[MAX_COMPARE_NAME + 1]; + // char state; + int32_t ppid; + // int32_t pgrp; + // int32_t session; + // int32_t tty_nr; + // int32_t tpgid; + // uint64_t flags; + unsigned long long minflt; + unsigned long long cminflt; + unsigned long long majflt; + unsigned long long cmajflt; + unsigned long long utime; + unsigned long long stime; + unsigned long long cutime; + unsigned long long cstime; + // int64_t priority; + // int64_t nice; + int32_t num_threads; + // int64_t itrealvalue; + // unsigned long long starttime; + // unsigned long long vsize; + unsigned long long rss; + // unsigned long long rsslim; + // unsigned long long starcode; + // unsigned long long endcode; + // unsigned long long startstack; + // unsigned long long kstkesp; + // unsigned long long kstkeip; + // uint64_t signal; + // uint64_t blocked; + // uint64_t sigignore; + // uint64_t sigcatch; + // uint64_t wchan; + // uint64_t nswap; + // uint64_t cnswap; + // int32_t exit_signal; + // int32_t processor; + // uint32_t rt_priority; + // uint32_t policy; + // unsigned long long delayacct_blkio_ticks; + // uint64_t guest_time; + // int64_t cguest_time; + + unsigned long long statm_size; + unsigned long long statm_resident; + unsigned long long statm_share; + unsigned long long statm_text; + unsigned long long statm_lib; + unsigned long long statm_data; + unsigned long long statm_dirty; + + unsigned long long io_logical_bytes_read; + unsigned long long io_logical_bytes_written; + unsigned long long io_read_calls; + unsigned long long io_write_calls; + unsigned long long io_storage_bytes_read; + unsigned long long io_storage_bytes_written; + unsigned long long io_cancelled_write_bytes; + +#ifdef INCLUDE_CHILDS + unsigned long long old_utime; + unsigned long long old_stime; + unsigned long long old_minflt; + unsigned long long old_majflt; + + unsigned long long old_cutime; + unsigned long long old_cstime; + unsigned long long old_cminflt; + unsigned long long old_cmajflt; + + unsigned long long fix_cutime; + unsigned long long fix_cstime; + unsigned long long fix_cminflt; + unsigned long long fix_cmajflt; + + unsigned long long diff_cutime; + unsigned long long diff_cstime; + unsigned long long diff_cminflt; + unsigned long long diff_cmajflt; +#endif + + int *fds; // array of fds it uses + int fds_size; // the size of the fds array + + int childs; // number of processes directly referencing this + int updated; // 1 when update + int merged; // 1 when it has been merged to its parent + int new_entry; + struct target *target; + struct pid_stat *parent; + struct pid_stat *prev; + struct pid_stat *next; +} *root_of_pids = NULL, **all_pids; + +long all_pids_count = 0; + +struct pid_stat *get_pid_entry(pid_t pid) +{ + if(all_pids[pid]) { + all_pids[pid]->new_entry = 0; + return all_pids[pid]; + } + + all_pids[pid] = calloc(sizeof(struct pid_stat), 1); + if(!all_pids[pid]) { + error("Cannot allocate %lu bytes of memory", (unsigned long)sizeof(struct pid_stat)); + return NULL; + } + + all_pids[pid]->fds = calloc(sizeof(int), 100); + if(!all_pids[pid]->fds) + error("Cannot allocate %ld bytes of memory", (unsigned long)(sizeof(int) * 100)); + else all_pids[pid]->fds_size = 100; + + if(root_of_pids) root_of_pids->prev = all_pids[pid]; + all_pids[pid]->next = root_of_pids; + root_of_pids = all_pids[pid]; + + all_pids[pid]->pid = pid; + all_pids[pid]->new_entry = 1; + + return all_pids[pid]; +} + +void del_pid_entry(pid_t pid) +{ + if(!all_pids[pid]) return; + + if(debug) fprintf(stderr, "apps.plugin: process %d %s exited, deleting it.\n", pid, all_pids[pid]->comm); + + if(root_of_pids == all_pids[pid]) root_of_pids = all_pids[pid]->next; + if(all_pids[pid]->next) all_pids[pid]->next->prev = all_pids[pid]->prev; + if(all_pids[pid]->prev) all_pids[pid]->prev->next = all_pids[pid]->next; + + if(all_pids[pid]->fds) free(all_pids[pid]->fds); + free(all_pids[pid]); + all_pids[pid] = NULL; +} + + +// ---------------------------------------------------------------------------- +// update pids from proc + +int read_proc_pid_stat(struct pid_stat *p) { + char filename[FILENAME_MAX + 1]; + + snprintf(filename, FILENAME_MAX, "%s/proc/%d/stat", host_prefix, p->pid); + + ff = procfile_reopen(ff, filename, NULL, PROCFILE_FLAG_NO_ERROR_ON_FILE_IO); + if(!ff) return 1; + + ff = procfile_readall(ff); + if(!ff) { + // procfile_close(ff); + return 1; + } + + file_counter++; + + p->comm[0] = '\0'; + p->comm[MAX_COMPARE_NAME] = '\0'; + size_t blen = 0; + + char *s = procfile_lineword(ff, 0, 1); + if(*s == '(') s++; + size_t len = strlen(s); + unsigned int i = 0; + while(len && s[len - 1] != ')') { + if(blen < MAX_COMPARE_NAME) { + strncpy(&p->comm[blen], s, MAX_COMPARE_NAME - blen); + blen = strlen(p->comm); + } + + i++; + s = procfile_lineword(ff, 0, 1+i); + len = strlen(s); + } + if(len && s[len - 1] == ')') s[len - 1] = '\0'; + if(blen < MAX_COMPARE_NAME) + strncpy(&p->comm[blen], s, MAX_COMPARE_NAME - blen); + + // p->pid = atol(procfile_lineword(ff, 0, 0+i)); + // comm is at 1 + // p->state = *(procfile_lineword(ff, 0, 2+i)); + p->ppid = (int32_t) atol(procfile_lineword(ff, 0, 3 + i)); + // p->pgrp = atol(procfile_lineword(ff, 0, 4+i)); + // p->session = atol(procfile_lineword(ff, 0, 5+i)); + // p->tty_nr = atol(procfile_lineword(ff, 0, 6+i)); + // p->tpgid = atol(procfile_lineword(ff, 0, 7+i)); + // p->flags = strtoull(procfile_lineword(ff, 0, 8+i), NULL, 10); + p->minflt = strtoull(procfile_lineword(ff, 0, 9+i), NULL, 10); + p->cminflt = strtoull(procfile_lineword(ff, 0, 10+i), NULL, 10); + p->majflt = strtoull(procfile_lineword(ff, 0, 11+i), NULL, 10); + p->cmajflt = strtoull(procfile_lineword(ff, 0, 12+i), NULL, 10); + p->utime = strtoull(procfile_lineword(ff, 0, 13+i), NULL, 10); + p->stime = strtoull(procfile_lineword(ff, 0, 14+i), NULL, 10); + p->cutime = strtoull(procfile_lineword(ff, 0, 15+i), NULL, 10); + p->cstime = strtoull(procfile_lineword(ff, 0, 16+i), NULL, 10); + // p->priority = strtoull(procfile_lineword(ff, 0, 17+i), NULL, 10); + // p->nice = strtoull(procfile_lineword(ff, 0, 18+i), NULL, 10); + p->num_threads = (int32_t) atol(procfile_lineword(ff, 0, 19 + i)); + // p->itrealvalue = strtoull(procfile_lineword(ff, 0, 20+i), NULL, 10); + // p->starttime = strtoull(procfile_lineword(ff, 0, 21+i), NULL, 10); + // p->vsize = strtoull(procfile_lineword(ff, 0, 22+i), NULL, 10); + p->rss = strtoull(procfile_lineword(ff, 0, 23+i), NULL, 10); + // p->rsslim = strtoull(procfile_lineword(ff, 0, 24+i), NULL, 10); + // p->starcode = strtoull(procfile_lineword(ff, 0, 25+i), NULL, 10); + // p->endcode = strtoull(procfile_lineword(ff, 0, 26+i), NULL, 10); + // p->startstack = strtoull(procfile_lineword(ff, 0, 27+i), NULL, 10); + // p->kstkesp = strtoull(procfile_lineword(ff, 0, 28+i), NULL, 10); + // p->kstkeip = strtoull(procfile_lineword(ff, 0, 29+i), NULL, 10); + // p->signal = strtoull(procfile_lineword(ff, 0, 30+i), NULL, 10); + // p->blocked = strtoull(procfile_lineword(ff, 0, 31+i), NULL, 10); + // p->sigignore = strtoull(procfile_lineword(ff, 0, 32+i), NULL, 10); + // p->sigcatch = strtoull(procfile_lineword(ff, 0, 33+i), NULL, 10); + // p->wchan = strtoull(procfile_lineword(ff, 0, 34+i), NULL, 10); + // p->nswap = strtoull(procfile_lineword(ff, 0, 35+i), NULL, 10); + // p->cnswap = strtoull(procfile_lineword(ff, 0, 36+i), NULL, 10); + // p->exit_signal = atol(procfile_lineword(ff, 0, 37+i)); + // p->processor = atol(procfile_lineword(ff, 0, 38+i)); + // p->rt_priority = strtoul(procfile_lineword(ff, 0, 39+i), NULL, 10); + // p->policy = strtoul(procfile_lineword(ff, 0, 40+i), NULL, 10); + // p->delayacct_blkio_ticks = strtoull(procfile_lineword(ff, 0, 41+i), NULL, 10); + // p->guest_time = strtoull(procfile_lineword(ff, 0, 42+i), NULL, 10); + // p->cguest_time = strtoull(procfile_lineword(ff, 0, 43), NULL, 10); + + if(debug || (p->target && p->target->debug)) fprintf(stderr, "apps.plugin: VALUES: %s utime=%llu, stime=%llu, cutime=%llu, cstime=%llu, minflt=%llu, majflt=%llu, cminflt=%llu, cmajflt=%llu, threads=%d\n", p->comm, p->utime, p->stime, p->cutime, p->cstime, p->minflt, p->majflt, p->cminflt, p->cmajflt, p->num_threads); + + // procfile_close(ff); + return 0; +} + +int read_proc_pid_statm(struct pid_stat *p) { + char filename[FILENAME_MAX + 1]; + + snprintf(filename, FILENAME_MAX, "%s/proc/%d/statm", host_prefix, p->pid); + + ff = procfile_reopen(ff, filename, NULL, PROCFILE_FLAG_NO_ERROR_ON_FILE_IO); + if(!ff) return 1; + + ff = procfile_readall(ff); + if(!ff) { + // procfile_close(ff); + return 1; + } + + file_counter++; + + p->statm_size = strtoull(procfile_lineword(ff, 0, 0), NULL, 10); + p->statm_resident = strtoull(procfile_lineword(ff, 0, 1), NULL, 10); + p->statm_share = strtoull(procfile_lineword(ff, 0, 2), NULL, 10); + p->statm_text = strtoull(procfile_lineword(ff, 0, 3), NULL, 10); + p->statm_lib = strtoull(procfile_lineword(ff, 0, 4), NULL, 10); + p->statm_data = strtoull(procfile_lineword(ff, 0, 5), NULL, 10); + p->statm_dirty = strtoull(procfile_lineword(ff, 0, 6), NULL, 10); + + // procfile_close(ff); + return 0; +} + +int read_proc_pid_io(struct pid_stat *p) { + char filename[FILENAME_MAX + 1]; + + snprintf(filename, FILENAME_MAX, "%s/proc/%d/io", host_prefix, p->pid); + + ff = procfile_reopen(ff, filename, NULL, PROCFILE_FLAG_NO_ERROR_ON_FILE_IO); + if(!ff) return 1; + + ff = procfile_readall(ff); + if(!ff) { + // procfile_close(ff); + return 1; + } + + file_counter++; + + p->io_logical_bytes_read = strtoull(procfile_lineword(ff, 0, 1), NULL, 10); + p->io_logical_bytes_written = strtoull(procfile_lineword(ff, 1, 1), NULL, 10); + p->io_read_calls = strtoull(procfile_lineword(ff, 2, 1), NULL, 10); + p->io_write_calls = strtoull(procfile_lineword(ff, 3, 1), NULL, 10); + p->io_storage_bytes_read = strtoull(procfile_lineword(ff, 4, 1), NULL, 10); + p->io_storage_bytes_written = strtoull(procfile_lineword(ff, 5, 1), NULL, 10); + p->io_cancelled_write_bytes = strtoull(procfile_lineword(ff, 6, 1), NULL, 10); + + // procfile_close(ff); + return 0; +} + + +// ---------------------------------------------------------------------------- + +#ifdef INCLUDE_CHILDS +// print a tree view of all processes +int walk_down(pid_t pid, int level) { + struct pid_stat *p = NULL; + char b[level+3]; + int i, ret = 0; + + for(i = 0; i < level; i++) b[i] = '\t'; + b[level] = '|'; + b[level+1] = '-'; + b[level+2] = '\0'; + + for(p = root_of_pids; p ; p = p->next) { + if(p->ppid == pid) { + ret += walk_down(p->pid, level+1); + } + } + + p = all_pids[pid]; + if(p) { + if(!p->updated) ret += 1; + if(ret) fprintf(stderr, "%s %s %d [%s, %s] c=%d u=%llu+%llu, s=%llu+%llu, cu=%llu+%llu, cs=%llu+%llu, n=%llu+%llu, j=%llu+%llu, cn=%llu+%llu, cj=%llu+%llu\n" + , b, p->comm, p->pid, p->updated?"OK":"KILLED", p->target->name, p->childs + , p->utime, p->utime - p->old_utime + , p->stime, p->stime - p->old_stime + , p->cutime, p->cutime - p->old_cutime + , p->cstime, p->cstime - p->old_cstime + , p->minflt, p->minflt - p->old_minflt + , p->majflt, p->majflt - p->old_majflt + , p->cminflt, p->cminflt - p->old_cminflt + , p->cmajflt, p->cmajflt - p->old_cmajflt + ); + } + + return ret; +} +#endif + + +// ---------------------------------------------------------------------------- +// file descriptor +// this is used to keep a global list of all open files of the system +// it is needed in order to figure out the unique files a process tree has open + +#define FILE_DESCRIPTORS_INCREASE_STEP 100 + +struct file_descriptor { + avl avl; + uint32_t magic; + uint32_t hash; + const char *name; + int type; + int count; + int pos; +} *all_files = NULL; + +int all_files_len = 0; +int all_files_size = 0; + +int file_descriptor_compare(void* a, void* b) { + if(((struct file_descriptor *)a)->magic != 0x0BADCAFE || ((struct file_descriptor *)b)->magic != 0x0BADCAFE) + error("Corrupted index data detected. Please report this."); + + if(((struct file_descriptor *)a)->hash < ((struct file_descriptor *)b)->hash) + return -1; + else if(((struct file_descriptor *)a)->hash > ((struct file_descriptor *)b)->hash) + return 1; + else return strcmp(((struct file_descriptor *)a)->name, ((struct file_descriptor *)b)->name); +} +int file_descriptor_iterator(avl *a) { if(a) {}; return 0; } + +avl_tree all_files_index = { + NULL, + file_descriptor_compare, +#ifdef AVL_LOCK_WITH_MUTEX + PTHREAD_MUTEX_INITIALIZER +#else + PTHREAD_RWLOCK_INITIALIZER +#endif +}; + +static struct file_descriptor *file_descriptor_find(const char *name, uint32_t hash) { + struct file_descriptor *result = NULL, tmp; + tmp.hash = (hash)?hash:simple_hash(name); + tmp.name = name; + tmp.count = 0; + tmp.pos = 0; + tmp.magic = 0x0BADCAFE; + + avl_search(&all_files_index, (avl *)&tmp, file_descriptor_iterator, (avl **)&result); + return result; +} + +#define file_descriptor_add(fd) avl_insert(&all_files_index, (avl *)(fd)) +#define file_descriptor_remove(fd) avl_remove(&all_files_index, (avl *)(fd)) + +#define FILETYPE_OTHER 0 +#define FILETYPE_FILE 1 +#define FILETYPE_PIPE 2 +#define FILETYPE_SOCKET 3 +#define FILETYPE_INOTIFY 4 +#define FILETYPE_EVENTFD 5 +#define FILETYPE_EVENTPOLL 6 +#define FILETYPE_TIMERFD 7 +#define FILETYPE_SIGNALFD 8 + +void file_descriptor_not_used(int id) +{ + if(id > 0 && id < all_files_size) { + if(all_files[id].magic != 0x0BADCAFE) { + error("Ignoring request to remove empty file id %d.", id); + return; + } + + if(debug) fprintf(stderr, "apps.plugin: decreasing slot %d (count = %d).\n", id, all_files[id].count); + + if(all_files[id].count > 0) { + all_files[id].count--; + + if(!all_files[id].count) { + if(debug) fprintf(stderr, "apps.plugin: >> slot %d is empty.\n", id); + file_descriptor_remove(&all_files[id]); + all_files[id].magic = 0x00000000; + all_files_len--; + } + } + else + error("Request to decrease counter of fd %d (%s), while the use counter is 0", id, all_files[id].name); + } + else error("Request to decrease counter of fd %d, which is outside the array size (1 to %d)", id, all_files_size); +} + +int file_descriptor_find_or_add(const char *name) +{ + static int last_pos = 0; + uint32_t hash = simple_hash(name); + + if(debug) fprintf(stderr, "apps.plugin: adding or finding name '%s' with hash %u\n", name, hash); + + struct file_descriptor *fd = file_descriptor_find(name, hash); + if(fd) { + // found + if(debug) fprintf(stderr, "apps.plugin: >> found on slot %d\n", fd->pos); + fd->count++; + return fd->pos; + } + // not found + + // check we have enough memory to add it + if(!all_files || all_files_len == all_files_size) { + void *old = all_files; + int i; + + // there is no empty slot + if(debug) fprintf(stderr, "apps.plugin: extending fd array to %d entries\n", all_files_size + FILE_DESCRIPTORS_INCREASE_STEP); + all_files = realloc(all_files, (all_files_size + FILE_DESCRIPTORS_INCREASE_STEP) * sizeof(struct file_descriptor)); + + // if the address changed, we have to rebuild the index + // since all pointers are now invalid + if(old && old != (void *)all_files) { + if(debug) fprintf(stderr, "apps.plugin: >> re-indexing.\n"); + all_files_index.root = NULL; + for(i = 0; i < all_files_size; i++) { + if(!all_files[i].count) continue; + file_descriptor_add(&all_files[i]); + } + if(debug) fprintf(stderr, "apps.plugin: >> re-indexing done.\n"); + } + + for(i = all_files_size; i < (all_files_size + FILE_DESCRIPTORS_INCREASE_STEP); i++) { + all_files[i].count = 0; + all_files[i].name = NULL; + all_files[i].magic = 0x00000000; + all_files[i].pos = i; + } + + if(!all_files_size) all_files_len = 1; + all_files_size += FILE_DESCRIPTORS_INCREASE_STEP; + } + + if(debug) fprintf(stderr, "apps.plugin: >> searching for empty slot.\n"); + + // search for an empty slot + int i, c; + for(i = 0, c = last_pos ; i < all_files_size ; i++, c++) { + if(c >= all_files_size) c = 0; + if(c == 0) continue; + + if(!all_files[c].count) { + if(debug) fprintf(stderr, "apps.plugin: >> Examining slot %d.\n", c); + + if(all_files[c].magic == 0x0BADCAFE && all_files[c].name && file_descriptor_find(all_files[c].name, all_files[c].hash)) + error("fd on position %d is not cleared properly. It still has %s in it.\n", c, all_files[c].name); + + if(debug) fprintf(stderr, "apps.plugin: >> %s fd position %d for %s (last name: %s)\n", all_files[c].name?"re-using":"using", c, name, all_files[c].name); + if(all_files[c].name) free((void *)all_files[c].name); + all_files[c].name = NULL; + last_pos = c; + break; + } + } + if(i == all_files_size) { + fatal("We should find an empty slot, but there isn't any"); + exit(1); + } + if(debug) fprintf(stderr, "apps.plugin: >> updating slot %d.\n", c); + + all_files_len++; + + // else we have an empty slot in 'c' + + int type; + if(name[0] == '/') type = FILETYPE_FILE; + else if(strncmp(name, "pipe:", 5) == 0) type = FILETYPE_PIPE; + else if(strncmp(name, "socket:", 7) == 0) type = FILETYPE_SOCKET; + else if(strcmp(name, "anon_inode:inotify") == 0 || strcmp(name, "inotify") == 0) type = FILETYPE_INOTIFY; + else if(strcmp(name, "anon_inode:[eventfd]") == 0) type = FILETYPE_EVENTFD; + else if(strcmp(name, "anon_inode:[eventpoll]") == 0) type = FILETYPE_EVENTPOLL; + else if(strcmp(name, "anon_inode:[timerfd]") == 0) type = FILETYPE_TIMERFD; + else if(strcmp(name, "anon_inode:[signalfd]") == 0) type = FILETYPE_SIGNALFD; + else if(strncmp(name, "anon_inode:", 11) == 0) { + if(debug) fprintf(stderr, "apps.plugin: FIXME: unknown anonymous inode: %s\n", name); + type = FILETYPE_OTHER; + } + else { + if(debug) fprintf(stderr, "apps.plugin: FIXME: cannot understand linkname: %s\n", name); + type = FILETYPE_OTHER; + } + + all_files[c].name = strdup(name); + all_files[c].hash = hash; + all_files[c].type = type; + all_files[c].pos = c; + all_files[c].count = 1; + all_files[c].magic = 0x0BADCAFE; + + file_descriptor_add(&all_files[c]); + + if(debug) fprintf(stderr, "apps.plugin: using fd position %d (name: %s)\n", c, all_files[c].name); + + return c; +} + + +// 1. read all files in /proc +// 2. for each numeric directory: +// i. read /proc/pid/stat +// ii. read /proc/pid/statm +// iii. read /proc/pid/io (requires root access) +// iii. read the entries in directory /proc/pid/fd (requires root access) +// for each entry: +// a. find or create a struct file_descriptor +// b. cleanup any old/unused file_descriptors + +// after all these, some pids may be linked to targets, while others may not + +// in case of errors, only 1 every 1000 errors is printed +// to avoid filling up all disk space +// if debug is enabled, all errors are printed + +int update_from_proc(void) +{ + static long count_errors = 0; + + char filename[FILENAME_MAX+1]; + char dirname[FILENAME_MAX + 1]; + + snprintf(dirname, FILENAME_MAX, "%s/proc", host_prefix); + DIR *dir = opendir(dirname); + if(!dir) return 0; + + struct dirent *file = NULL; + struct pid_stat *p = NULL; + + // mark them all as un-updated + all_pids_count = 0; + for(p = root_of_pids; p ; p = p->next) { + all_pids_count++; + p->parent = NULL; + p->updated = 0; + p->childs = 0; + p->merged = 0; + p->new_entry = 0; + } + + while((file = readdir(dir))) { + char *endptr = file->d_name; + pid_t pid = (pid_t) strtoul(file->d_name, &endptr, 10); + if(pid <= 0 || pid > pid_max || endptr == file->d_name || *endptr != '\0') continue; + + p = get_pid_entry(pid); + if(!p) continue; + + // -------------------------------------------------------------------- + // /proc/<pid>/stat + + if(read_proc_pid_stat(p)) { + if(!count_errors++ || debug || (p->target && p->target->debug)) + error("Cannot process %s/proc/%d/stat", host_prefix, pid); + + continue; + } + if(p->ppid < 0 || p->ppid > pid_max) p->ppid = 0; + + + // -------------------------------------------------------------------- + // /proc/<pid>/statm + + if(read_proc_pid_statm(p)) { + if(!count_errors++ || debug || (p->target && p->target->debug)) + error("Cannot process %s/proc/%d/statm", host_prefix, pid); + + continue; + } + + + // -------------------------------------------------------------------- + // /proc/<pid>/io + + if(read_proc_pid_io(p)) { + if(!count_errors++ || debug || (p->target && p->target->debug)) + error("Cannot process %s/proc/%d/io", host_prefix, pid); + + continue; + } + + // -------------------------------------------------------------------- + // link it + + // check if it is target + // we do this only once, the first time this pid is loaded + if(p->new_entry) { + if(debug) fprintf(stderr, "apps.plugin: \tJust added %s\n", p->comm); + + struct target *w; + for(w = target_root; w ; w = w->next) { + // if(debug || (p->target && p->target->debug)) fprintf(stderr, "apps.plugin: \t\tcomparing '%s' with '%s'\n", w->compare, p->comm); + + if(strcmp(w->compare, p->comm) == 0) { + if(w->target) p->target = w->target; + else p->target = w; + + if(debug || (p->target && p->target->debug)) fprintf(stderr, "apps.plugin: \t\t%s linked to target %s\n", p->comm, p->target->name); + } + } + } + + // -------------------------------------------------------------------- + // /proc/<pid>/fd + + snprintf(filename, FILENAME_MAX, "%s/proc/%s/fd", host_prefix, file->d_name); + DIR *fds = opendir(filename); + if(fds) { + int c; + struct dirent *de; + char fdname[FILENAME_MAX + 1]; + char linkname[FILENAME_MAX + 1]; + + // make the array negative + for(c = 0 ; c < p->fds_size ; c++) p->fds[c] = -p->fds[c]; + + while((de = readdir(fds))) { + if(strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") == 0) continue; + + // check if the fds array is small + int fdid = atoi(de->d_name); + if(fdid < 0) continue; + if(fdid >= p->fds_size) { + // it is small, extend it + if(debug) fprintf(stderr, "apps.plugin: extending fd memory slots for %s from %d to %d\n", p->comm, p->fds_size, fdid + 100); + p->fds = realloc(p->fds, (fdid + 100) * sizeof(int)); + if(!p->fds) { + error("Cannot re-allocate fds for %s", p->comm); + break; + } + + // and initialize it + for(c = p->fds_size ; c < (fdid + 100) ; c++) p->fds[c] = 0; + p->fds_size = fdid + 100; + } + + if(p->fds[fdid] == 0) { + // we don't know this fd, get it + + sprintf(fdname, "%s/proc/%s/fd/%s", host_prefix, file->d_name, de->d_name); + ssize_t l = readlink(fdname, linkname, FILENAME_MAX); + if(l == -1) { + if(debug || (p->target && p->target->debug)) { + if(!count_errors++ || debug || (p->target && p->target->debug)) + error("Cannot read link %s", fdname); + } + continue; + } + linkname[l] = '\0'; + file_counter++; + + // if another process already has this, we will get + // the same id + p->fds[fdid] = file_descriptor_find_or_add(linkname); + } + + // else make it positive again, we need it + // of course, the actual file may have changed, but we don't care so much + // FIXME: we could compare the inode as returned by readdir direct structure + else p->fds[fdid] = -p->fds[fdid]; + } + closedir(fds); + + // remove all the negative file descriptors + for(c = 0 ; c < p->fds_size ; c++) if(p->fds[c] < 0) { + file_descriptor_not_used(-p->fds[c]); + p->fds[c] = 0; + } + } + + // -------------------------------------------------------------------- + // done! + + // mark it as updated + p->updated = 1; + } + if(count_errors > 1000) { + error("%ld more errors encountered\n", count_errors - 1); + count_errors = 0; + } + + closedir(dir); + + return 1; +} + + +// ---------------------------------------------------------------------------- +// update statistics on the targets + +// 1. link all childs to their parents +// 2. go from bottom to top, marking as merged all childs to their parents +// this step links all parents without a target to the child target, if any +// 3. link all top level processes (the ones not merged) to the default target +// 4. go from top to bottom, linking all childs without a target, to their parent target +// after this step, all processes have a target +// [5. for each killed pid (updated = 0), remove its usage from its target] +// 6. zero all targets +// 7. concentrate all values on the targets +// 8. remove all killed processes +// 9. find the unique file count for each target + +void update_statistics(void) +{ + int c; + struct pid_stat *p = NULL; + + // link all parents and update childs count + for(p = root_of_pids; p ; p = p->next) { + if(p->ppid > 0 && p->ppid <= pid_max && all_pids[p->ppid]) { + if(debug || (p->target && p->target->debug)) fprintf(stderr, "apps.plugin: \tparent of %d %s is %d %s\n", p->pid, p->comm, p->ppid, all_pids[p->ppid]->comm); + + p->parent = all_pids[p->ppid]; + p->parent->childs++; + } + else if(p->ppid != 0) error("pid %d %s states parent %d, but the later does not exist.", p->pid, p->comm, p->ppid); + } + + // find all the procs with 0 childs and merge them to their parents + // repeat, until nothing more can be done. + int found = 1; + while(found) { + found = 0; + for(p = root_of_pids; p ; p = p->next) { + // if this process does not have any childs, and + // is not already merged, and + // its parent has childs waiting to be merged, and + // the target of this process and its parent is the same, or the parent does not have a target, or this process does not have a parent + // and its parent is not init + // then... merge them! + if(!p->childs && !p->merged && p->parent && p->parent->childs && (p->target == p->parent->target || !p->parent->target || !p->target) && p->ppid != 1) { + p->parent->childs--; + p->merged = 1; + + // the parent inherits the child's target, if it does not have a target itself + if(p->target && !p->parent->target) { + p->parent->target = p->target; + if(debug || (p->target && p->target->debug)) fprintf(stderr, "apps.plugin: \t\ttarget %s is inherited by %d %s from its child %d %s.\n", p->target->name, p->parent->pid, p->parent->comm, p->pid, p->comm); + } + + found++; + } + } + if(debug) fprintf(stderr, "apps.plugin: merged %d processes\n", found); + } + + // give a default target on all top level processes + // init goes always to default target + if(all_pids[1]) all_pids[1]->target = default_target; + + for(p = root_of_pids; p ; p = p->next) { + // if the process is not merged itself + // then is is a top level process + if(!p->merged && !p->target) p->target = default_target; + +#ifdef INCLUDE_CHILDS + // by the way, update the diffs + // will be used later for substracting killed process times + p->diff_cutime = p->utime - p->cutime; + p->diff_cstime = p->stime - p->cstime; + p->diff_cminflt = p->minflt - p->cminflt; + p->diff_cmajflt = p->majflt - p->cmajflt; +#endif + } + + // give a target to all merged child processes + found = 1; + while(found) { + found = 0; + for(p = root_of_pids; p ; p = p->next) { + if(!p->target && p->merged && p->parent && p->parent->target) { + p->target = p->parent->target; + found++; + } + } + } + +#ifdef INCLUDE_CHILDS + // for each killed process, remove its values from the parents + // sums (we had already added them in a previous loop) + for(p = root_of_pids; p ; p = p->next) { + if(p->updated) continue; + + if(debug) fprintf(stderr, "apps.plugin: UNMERGING %d %s\n", p->pid, p->comm); + + unsigned long long diff_utime = p->utime + p->cutime + p->fix_cutime; + unsigned long long diff_stime = p->stime + p->cstime + p->fix_cstime; + unsigned long long diff_minflt = p->minflt + p->cminflt + p->fix_cminflt; + unsigned long long diff_majflt = p->majflt + p->cmajflt + p->fix_cmajflt; + + struct pid_stat *t = p; + while((t = t->parent)) { + if(!t->updated) continue; + + unsigned long long x; + if(diff_utime && t->diff_cutime) { + x = (t->diff_cutime < diff_utime)?t->diff_cutime:diff_utime; + diff_utime -= x; + t->diff_cutime -= x; + t->fix_cutime += x; + if(debug) fprintf(stderr, "apps.plugin: \t cutime %llu from %d %s %s\n", x, t->pid, t->comm, t->target->name); + } + if(diff_stime && t->diff_cstime) { + x = (t->diff_cstime < diff_stime)?t->diff_cstime:diff_stime; + diff_stime -= x; + t->diff_cstime -= x; + t->fix_cstime += x; + if(debug) fprintf(stderr, "apps.plugin: \t cstime %llu from %d %s %s\n", x, t->pid, t->comm, t->target->name); + } + if(diff_minflt && t->diff_cminflt) { + x = (t->diff_cminflt < diff_minflt)?t->diff_cminflt:diff_minflt; + diff_minflt -= x; + t->diff_cminflt -= x; + t->fix_cminflt += x; + if(debug) fprintf(stderr, "apps.plugin: \t cminflt %llu from %d %s %s\n", x, t->pid, t->comm, t->target->name); + } + if(diff_majflt && t->diff_cmajflt) { + x = (t->diff_cmajflt < diff_majflt)?t->diff_cmajflt:diff_majflt; + diff_majflt -= x; + t->diff_cmajflt -= x; + t->fix_cmajflt += x; + if(debug) fprintf(stderr, "apps.plugin: \t cmajflt %llu from %d %s %s\n", x, t->pid, t->comm, t->target->name); + } + } + + if(diff_utime) error("Cannot fix up utime %llu", diff_utime); + if(diff_stime) error("Cannot fix up stime %llu", diff_stime); + if(diff_minflt) error("Cannot fix up minflt %llu", diff_minflt); + if(diff_majflt) error("Cannot fix up majflt %llu", diff_majflt); + } +#endif + + // zero all the targets + targets = 0; + struct target *w; + for (w = target_root; w ; w = w->next) { + targets++; + + w->fds = calloc(sizeof(int), (size_t) all_files_size); + if(!w->fds) + error("Cannot allocate memory for fds in %s", w->name); + + w->minflt = 0; + w->majflt = 0; + w->utime = 0; + w->stime = 0; + w->cminflt = 0; + w->cmajflt = 0; + w->cutime = 0; + w->cstime = 0; + w->num_threads = 0; + w->rss = 0; + w->processes = 0; + + w->statm_size = 0; + w->statm_resident = 0; + w->statm_share = 0; + w->statm_text = 0; + w->statm_lib = 0; + w->statm_data = 0; + w->statm_dirty = 0; + + w->io_logical_bytes_read = 0; + w->io_logical_bytes_written = 0; + w->io_read_calls = 0; + w->io_write_calls = 0; + w->io_storage_bytes_read = 0; + w->io_storage_bytes_written = 0; + w->io_cancelled_write_bytes = 0; + } + +#ifdef INCLUDE_CHILDS + if(debug) walk_down(0, 1); +#endif + + // concentrate everything on the targets + for(p = root_of_pids; p ; p = p->next) { + if(!p->target) { + error("pid %d %s was left without a target!", p->pid, p->comm); + continue; + } + + if(p->updated) { + p->target->cutime += p->cutime; // - p->fix_cutime; + p->target->cstime += p->cstime; // - p->fix_cstime; + p->target->cminflt += p->cminflt; // - p->fix_cminflt; + p->target->cmajflt += p->cmajflt; // - p->fix_cmajflt; + + p->target->utime += p->utime; //+ (p->pid != 1)?(p->cutime - p->fix_cutime):0; + p->target->stime += p->stime; //+ (p->pid != 1)?(p->cstime - p->fix_cstime):0; + p->target->minflt += p->minflt; //+ (p->pid != 1)?(p->cminflt - p->fix_cminflt):0; + p->target->majflt += p->majflt; //+ (p->pid != 1)?(p->cmajflt - p->fix_cmajflt):0; + + //if(p->num_threads < 0) + // error("Negative threads number for pid '%s' (%d): %d", p->comm, p->pid, p->num_threads); + + //if(p->num_threads > 10000) + // error("Excessive threads number for pid '%s' (%d): %d", p->comm, p->pid, p->num_threads); + + p->target->num_threads += p->num_threads; + p->target->rss += p->rss; + + p->target->statm_size += p->statm_size; + p->target->statm_resident += p->statm_resident; + p->target->statm_share += p->statm_share; + p->target->statm_text += p->statm_text; + p->target->statm_lib += p->statm_lib; + p->target->statm_data += p->statm_data; + p->target->statm_dirty += p->statm_dirty; + + p->target->io_logical_bytes_read += p->io_logical_bytes_read; + p->target->io_logical_bytes_written += p->io_logical_bytes_written; + p->target->io_read_calls += p->io_read_calls; + p->target->io_write_calls += p->io_write_calls; + p->target->io_storage_bytes_read += p->io_storage_bytes_read; + p->target->io_storage_bytes_written += p->io_storage_bytes_written; + p->target->io_cancelled_write_bytes += p->io_cancelled_write_bytes; + + p->target->processes++; + + for(c = 0; c < p->fds_size ;c++) { + if(p->fds[c] == 0) continue; + if(p->fds[c] < all_files_size) { + if(p->target->fds) p->target->fds[p->fds[c]]++; + } + else + error("Invalid fd number %d", p->fds[c]); + } + + if(debug || p->target->debug) fprintf(stderr, "apps.plugin: \tAgregating %s pid %d on %s utime=%llu, stime=%llu, cutime=%llu, cstime=%llu, minflt=%llu, majflt=%llu, cminflt=%llu, cmajflt=%llu\n", p->comm, p->pid, p->target->name, p->utime, p->stime, p->cutime, p->cstime, p->minflt, p->majflt, p->cminflt, p->cmajflt); + +/* if(p->utime - p->old_utime > 100) fprintf(stderr, "BIG CHANGE: %d %s utime increased by %llu from %llu to %llu\n", p->pid, p->comm, p->utime - p->old_utime, p->old_utime, p->utime); + if(p->cutime - p->old_cutime > 100) fprintf(stderr, "BIG CHANGE: %d %s cutime increased by %llu from %llu to %llu\n", p->pid, p->comm, p->cutime - p->old_cutime, p->old_cutime, p->cutime); + if(p->stime - p->old_stime > 100) fprintf(stderr, "BIG CHANGE: %d %s stime increased by %llu from %llu to %llu\n", p->pid, p->comm, p->stime - p->old_stime, p->old_stime, p->stime); + if(p->cstime - p->old_cstime > 100) fprintf(stderr, "BIG CHANGE: %d %s cstime increased by %llu from %llu to %llu\n", p->pid, p->comm, p->cstime - p->old_cstime, p->old_cstime, p->cstime); + if(p->minflt - p->old_minflt > 5000) fprintf(stderr, "BIG CHANGE: %d %s minflt increased by %llu from %llu to %llu\n", p->pid, p->comm, p->minflt - p->old_minflt, p->old_minflt, p->minflt); + if(p->majflt - p->old_majflt > 5000) fprintf(stderr, "BIG CHANGE: %d %s majflt increased by %llu from %llu to %llu\n", p->pid, p->comm, p->majflt - p->old_majflt, p->old_majflt, p->majflt); + if(p->cminflt - p->old_cminflt > 15000) fprintf(stderr, "BIG CHANGE: %d %s cminflt increased by %llu from %llu to %llu\n", p->pid, p->comm, p->cminflt - p->old_cminflt, p->old_cminflt, p->cminflt); + if(p->cmajflt - p->old_cmajflt > 15000) fprintf(stderr, "BIG CHANGE: %d %s cmajflt increased by %llu from %llu to %llu\n", p->pid, p->comm, p->cmajflt - p->old_cmajflt, p->old_cmajflt, p->cmajflt); +*/ +#ifdef INCLUDE_CHILDS + p->old_utime = p->utime; + p->old_cutime = p->cutime; + p->old_stime = p->stime; + p->old_cstime = p->cstime; + p->old_minflt = p->minflt; + p->old_majflt = p->majflt; + p->old_cminflt = p->cminflt; + p->old_cmajflt = p->cmajflt; +#endif + } + else { + // since the process has exited, the user + // will see a drop in our charts, because the incremental + // values of this process will not be there + + // add them to the fix_* values and they will be added to + // the reported values, so that the report goes steady + p->target->fix_minflt += p->minflt; + p->target->fix_majflt += p->majflt; + p->target->fix_utime += p->utime; + p->target->fix_stime += p->stime; + p->target->fix_cminflt += p->cminflt; + p->target->fix_cmajflt += p->cmajflt; + p->target->fix_cutime += p->cutime; + p->target->fix_cstime += p->cstime; + + p->target->fix_io_logical_bytes_read += p->io_logical_bytes_read; + p->target->fix_io_logical_bytes_written += p->io_logical_bytes_written; + p->target->fix_io_read_calls += p->io_read_calls; + p->target->fix_io_write_calls += p->io_write_calls; + p->target->fix_io_storage_bytes_read += p->io_storage_bytes_read; + p->target->fix_io_storage_bytes_written += p->io_storage_bytes_written; + p->target->fix_io_cancelled_write_bytes += p->io_cancelled_write_bytes; + } + } + +// fprintf(stderr, "\n"); + // cleanup all un-updated processed (exited, killed, etc) + for(p = root_of_pids; p ;) { + if(!p->updated) { +// fprintf(stderr, "\tEXITED %d %s [parent %d %s, target %s] utime=%llu, stime=%llu, cutime=%llu, cstime=%llu, minflt=%llu, majflt=%llu, cminflt=%llu, cmajflt=%llu\n", p->pid, p->comm, p->parent->pid, p->parent->comm, p->target->name, p->utime, p->stime, p->cutime, p->cstime, p->minflt, p->majflt, p->cminflt, p->cmajflt); + + for(c = 0 ; c < p->fds_size ; c++) if(p->fds[c] > 0) { + file_descriptor_not_used(p->fds[c]); + p->fds[c] = 0; + } + + pid_t r = p->pid; + p = p->next; + del_pid_entry(r); + } + else p = p->next; + } + + for (w = target_root; w ; w = w->next) { + w->openfiles = 0; + w->openpipes = 0; + w->opensockets = 0; + w->openinotifies = 0; + w->openeventfds = 0; + w->opentimerfds = 0; + w->opensignalfds = 0; + w->openeventpolls = 0; + w->openother = 0; + + for(c = 1; c < all_files_size ;c++) { + if(w->fds && w->fds[c] > 0) switch(all_files[c].type) { + case FILETYPE_FILE: + w->openfiles++; + break; + + case FILETYPE_PIPE: + w->openpipes++; + break; + + case FILETYPE_SOCKET: + w->opensockets++; + break; + + case FILETYPE_INOTIFY: + w->openinotifies++; + break; + + case FILETYPE_EVENTFD: + w->openeventfds++; + break; + + case FILETYPE_TIMERFD: + w->opentimerfds++; + break; + + case FILETYPE_SIGNALFD: + w->opensignalfds++; + break; + + case FILETYPE_EVENTPOLL: + w->openeventpolls++; + break; + + default: + w->openother++; + } + } + + free(w->fds); + w->fds = NULL; + } +} + +// ---------------------------------------------------------------------------- +// update chart dimensions + +void show_dimensions(void) +{ + static struct timeval last = { 0, 0 }; + static struct rusage me_last; + + struct target *w; + struct timeval now; + struct rusage me; + + unsigned long long usec; + unsigned long long cpuuser; + unsigned long long cpusyst; + + if(!last.tv_sec) { + gettimeofday(&last, NULL); + getrusage(RUSAGE_SELF, &me_last); + + // the first time, give a zero to allow + // netdata calibrate to the current time + // usec = update_every * 1000000ULL; + usec = 0ULL; + cpuuser = 0; + cpusyst = 0; + } + else { + gettimeofday(&now, NULL); + getrusage(RUSAGE_SELF, &me); + + usec = usecdiff(&now, &last); + cpuuser = me.ru_utime.tv_sec * 1000000ULL + me.ru_utime.tv_usec; + cpusyst = me.ru_stime.tv_sec * 1000000ULL + me.ru_stime.tv_usec; + + bcopy(&now, &last, sizeof(struct timeval)); + bcopy(&me, &me_last, sizeof(struct rusage)); + } + + fprintf(stdout, "BEGIN apps.cpu %llu\n", usec); + for (w = target_root; w ; w = w->next) { + if(w->target || (!w->processes && !w->exposed)) continue; + + fprintf(stdout, "SET %s = %llu\n", w->name, w->utime + w->stime + w->fix_utime + w->fix_stime); + } + fprintf(stdout, "END\n"); + + fprintf(stdout, "BEGIN apps.cpu_user %llu\n", usec); + for (w = target_root; w ; w = w->next) { + if(w->target || (!w->processes && !w->exposed)) continue; + + fprintf(stdout, "SET %s = %llu\n", w->name, w->utime + w->fix_utime); + } + fprintf(stdout, "END\n"); + + fprintf(stdout, "BEGIN apps.cpu_system %llu\n", usec); + for (w = target_root; w ; w = w->next) { + if(w->target || (!w->processes && !w->exposed)) continue; + + fprintf(stdout, "SET %s = %llu\n", w->name, w->stime + w->fix_stime); + } + fprintf(stdout, "END\n"); + + fprintf(stdout, "BEGIN apps.threads %llu\n", usec); + for (w = target_root; w ; w = w->next) { + if(w->target || (!w->processes && !w->exposed)) continue; + + fprintf(stdout, "SET %s = %llu\n", w->name, w->num_threads); + } + fprintf(stdout, "END\n"); + + fprintf(stdout, "BEGIN apps.processes %llu\n", usec); + for (w = target_root; w ; w = w->next) { + if(w->target || (!w->processes && !w->exposed)) continue; + + fprintf(stdout, "SET %s = %lu\n", w->name, w->processes); + } + fprintf(stdout, "END\n"); + + fprintf(stdout, "BEGIN apps.mem %llu\n", usec); + for (w = target_root; w ; w = w->next) { + if(w->target || (!w->processes && !w->exposed)) continue; + + fprintf(stdout, "SET %s = %lld\n", w->name, (long long)w->statm_resident - (long long)w->statm_share); + } + fprintf(stdout, "END\n"); + + fprintf(stdout, "BEGIN apps.minor_faults %llu\n", usec); + for (w = target_root; w ; w = w->next) { + if(w->target || (!w->processes && !w->exposed)) continue; + + fprintf(stdout, "SET %s = %llu\n", w->name, w->minflt + w->fix_minflt); + } + fprintf(stdout, "END\n"); + + fprintf(stdout, "BEGIN apps.major_faults %llu\n", usec); + for (w = target_root; w ; w = w->next) { + if(w->target || (!w->processes && !w->exposed)) continue; + + fprintf(stdout, "SET %s = %llu\n", w->name, w->majflt + w->fix_majflt); + } + fprintf(stdout, "END\n"); + + fprintf(stdout, "BEGIN apps.lreads %llu\n", usec); + for (w = target_root; w ; w = w->next) { + if(w->target || (!w->processes && !w->exposed)) continue; + + fprintf(stdout, "SET %s = %llu\n", w->name, w->io_logical_bytes_read); + } + fprintf(stdout, "END\n"); + + fprintf(stdout, "BEGIN apps.lwrites %llu\n", usec); + for (w = target_root; w ; w = w->next) { + if(w->target || (!w->processes && !w->exposed)) continue; + + fprintf(stdout, "SET %s = %llu\n", w->name, w->io_logical_bytes_written); + } + fprintf(stdout, "END\n"); + + fprintf(stdout, "BEGIN apps.preads %llu\n", usec); + for (w = target_root; w ; w = w->next) { + if(w->target || (!w->processes && !w->exposed)) continue; + + fprintf(stdout, "SET %s = %llu\n", w->name, w->io_storage_bytes_read); + } + fprintf(stdout, "END\n"); + + fprintf(stdout, "BEGIN apps.pwrites %llu\n", usec); + for (w = target_root; w ; w = w->next) { + if(w->target || (!w->processes && !w->exposed)) continue; + + fprintf(stdout, "SET %s = %llu\n", w->name, w->io_storage_bytes_written); + } + fprintf(stdout, "END\n"); + + fprintf(stdout, "BEGIN apps.files %llu\n", usec); + for (w = target_root; w ; w = w->next) { + if(w->target || (!w->processes && !w->exposed)) continue; + + fprintf(stdout, "SET %s = %llu\n", w->name, w->openfiles); + } + fprintf(stdout, "END\n"); + + fprintf(stdout, "BEGIN apps.sockets %llu\n", usec); + for (w = target_root; w ; w = w->next) { + if(w->target || (!w->processes && !w->exposed)) continue; + + fprintf(stdout, "SET %s = %llu\n", w->name, w->opensockets); + } + fprintf(stdout, "END\n"); + + fprintf(stdout, "BEGIN apps.pipes %llu\n", usec); + for (w = target_root; w ; w = w->next) { + if(w->target || (!w->processes && !w->exposed)) continue; + + fprintf(stdout, "SET %s = %llu\n", w->name, w->openpipes); + } + fprintf(stdout, "END\n"); + + fprintf(stdout, "BEGIN netdata.apps_cpu %llu\n", usec); + fprintf(stdout, "SET user = %llu\n", cpuuser); + fprintf(stdout, "SET system = %llu\n", cpusyst); + fprintf(stdout, "END\n"); + + fprintf(stdout, "BEGIN netdata.apps_files %llu\n", usec); + fprintf(stdout, "SET files = %llu\n", file_counter); + fprintf(stdout, "SET pids = %ld\n", all_pids_count); + fprintf(stdout, "SET fds = %d\n", all_files_len); + fprintf(stdout, "SET targets = %ld\n", targets); + fprintf(stdout, "END\n"); + + fflush(stdout); +} + + +// ---------------------------------------------------------------------------- +// generate the charts + +void show_charts(void) +{ + struct target *w; + int newly_added = 0; + + for(w = target_root ; w ; w = w->next) + if(!w->exposed && w->processes) { + newly_added++; + w->exposed = 1; + if(debug || w->debug) fprintf(stderr, "apps.plugin: %s just added - regenerating charts.\n", w->name); + } + + // nothing more to show + if(!newly_added) return; + + // we have something new to show + // update the charts + fprintf(stdout, "CHART apps.cpu '' 'Apps CPU Time (%ld%% = %ld core%s)' 'cpu time %%' cpu apps.cpu stacked 20001 %d\n", (processors * 100), processors, (processors>1)?"s":"", update_every); + for (w = target_root; w ; w = w->next) { + if(w->target || (!w->processes && !w->exposed)) continue; + + fprintf(stdout, "DIMENSION %s '' incremental 100 %llu %s\n", w->name, Hertz, w->hidden ? "hidden,noreset" : "noreset"); + } + + fprintf(stdout, "CHART apps.mem '' 'Apps Dedicated Memory (w/o shared)' 'MB' mem apps.mem stacked 20003 %d\n", update_every); + for (w = target_root; w ; w = w->next) { + if(w->target || (!w->processes && !w->exposed)) continue; + + fprintf(stdout, "DIMENSION %s '' absolute %ld %ld noreset\n", w->name, sysconf(_SC_PAGESIZE), 1024L*1024L); + } + + fprintf(stdout, "CHART apps.threads '' 'Apps Threads' 'threads' processes apps.threads stacked 20005 %d\n", update_every); + for (w = target_root; w ; w = w->next) { + if(w->target || (!w->processes && !w->exposed)) continue; + + fprintf(stdout, "DIMENSION %s '' absolute 1 1 noreset\n", w->name); + } + + fprintf(stdout, "CHART apps.processes '' 'Apps Processes' 'processes' processes apps.processes stacked 20004 %d\n", update_every); + for (w = target_root; w ; w = w->next) { + if(w->target || (!w->processes && !w->exposed)) continue; + + fprintf(stdout, "DIMENSION %s '' absolute 1 1 noreset\n", w->name); + } + + fprintf(stdout, "CHART apps.cpu_user '' 'Apps CPU User Time (%ld%% = %ld core%s)' 'cpu time %%' cpu apps.cpu_user stacked 20020 %d\n", (processors * 100), processors, (processors>1)?"s":"", update_every); + for (w = target_root; w ; w = w->next) { + if(w->target || (!w->processes && !w->exposed)) continue; + + fprintf(stdout, "DIMENSION %s '' incremental 100 %llu noreset\n", w->name, Hertz * processors); + } + + fprintf(stdout, "CHART apps.cpu_system '' 'Apps CPU System Time (%ld%% = %ld core%s)' 'cpu time %%' cpu apps.cpu_system stacked 20021 %d\n", (processors * 100), processors, (processors>1)?"s":"", update_every); + for (w = target_root; w ; w = w->next) { + if(w->target || (!w->processes && !w->exposed)) continue; + + fprintf(stdout, "DIMENSION %s '' incremental 100 %llu noreset\n", w->name, Hertz * processors); + } + + fprintf(stdout, "CHART apps.major_faults '' 'Apps Major Page Faults (swap read)' 'page faults/s' swap apps.major_faults stacked 20010 %d\n", update_every); + for (w = target_root; w ; w = w->next) { + if(w->target || (!w->processes && !w->exposed)) continue; + + fprintf(stdout, "DIMENSION %s '' incremental 1 1 noreset\n", w->name); + } + + fprintf(stdout, "CHART apps.minor_faults '' 'Apps Minor Page Faults' 'page faults/s' mem apps.minor_faults stacked 20011 %d\n", update_every); + for (w = target_root; w ; w = w->next) { + if(w->target || (!w->processes && !w->exposed)) continue; + + fprintf(stdout, "DIMENSION %s '' incremental 1 1 noreset\n", w->name); + } + + fprintf(stdout, "CHART apps.lreads '' 'Apps Disk Logical Reads' 'kilobytes/s' disk apps.lreads stacked 20042 %d\n", update_every); + for (w = target_root; w ; w = w->next) { + if(w->target || (!w->processes && !w->exposed)) continue; + + fprintf(stdout, "DIMENSION %s '' incremental 1 %d noreset\n", w->name, 1024); + } + + fprintf(stdout, "CHART apps.lwrites '' 'Apps I/O Logical Writes' 'kilobytes/s' disk apps.lwrites stacked 20042 %d\n", update_every); + for (w = target_root; w ; w = w->next) { + if(w->target || (!w->processes && !w->exposed)) continue; + + fprintf(stdout, "DIMENSION %s '' incremental 1 %d noreset\n", w->name, 1024); + } + + fprintf(stdout, "CHART apps.preads '' 'Apps Disk Reads' 'kilobytes/s' disk apps.preads stacked 20002 %d\n", update_every); + for (w = target_root; w ; w = w->next) { + if(w->target || (!w->processes && !w->exposed)) continue; + + fprintf(stdout, "DIMENSION %s '' incremental 1 %d noreset\n", w->name, 1024); + } + + fprintf(stdout, "CHART apps.pwrites '' 'Apps Disk Writes' 'kilobytes/s' disk apps.pwrites stacked 20002 %d\n", update_every); + for (w = target_root; w ; w = w->next) { + if(w->target || (!w->processes && !w->exposed)) continue; + + fprintf(stdout, "DIMENSION %s '' incremental 1 %d noreset\n", w->name, 1024); + } + + fprintf(stdout, "CHART apps.files '' 'Apps Open Files' 'open files' disk apps.files stacked 20050 %d\n", update_every); + for (w = target_root; w ; w = w->next) { + if(w->target || (!w->processes && !w->exposed)) continue; + + fprintf(stdout, "DIMENSION %s '' absolute 1 1 noreset\n", w->name); + } + + fprintf(stdout, "CHART apps.sockets '' 'Apps Open Sockets' 'open sockets' net apps.sockets stacked 20051 %d\n", update_every); + for (w = target_root; w ; w = w->next) { + if(w->target || (!w->processes && !w->exposed)) continue; + + fprintf(stdout, "DIMENSION %s '' absolute 1 1 noreset\n", w->name); + } + + fprintf(stdout, "CHART apps.pipes '' 'Apps Pipes' 'open pipes' processes apps.pipes stacked 20053 %d\n", update_every); + for (w = target_root; w ; w = w->next) { + if(w->target || (!w->processes && !w->exposed)) continue; + + fprintf(stdout, "DIMENSION %s '' absolute 1 1 noreset\n", w->name); + } + + fprintf(stdout, "CHART netdata.apps_cpu '' 'Apps Plugin CPU' 'milliseconds/s' apps.plugin netdata.apps_cpu stacked 140000 %d\n", update_every); + fprintf(stdout, "DIMENSION user '' incremental 1 %d\n", 1000); + fprintf(stdout, "DIMENSION system '' incremental 1 %d\n", 1000); + + fprintf(stdout, "CHART netdata.apps_files '' 'Apps Plugin Files' 'files/s' apps.plugin netdata.apps_files line 140001 %d\n", update_every); + fprintf(stdout, "DIMENSION files '' incremental 1 1\n"); + fprintf(stdout, "DIMENSION pids '' absolute 1 1\n"); + fprintf(stdout, "DIMENSION fds '' absolute 1 1\n"); + fprintf(stdout, "DIMENSION targets '' absolute 1 1\n"); + + fflush(stdout); +} + + +// ---------------------------------------------------------------------------- +// parse command line arguments + +void parse_args(int argc, char **argv) +{ + int i, freq = 0; + char *name = NULL; + + for(i = 1; i < argc; i++) { + if(!freq) { + int n = atoi(argv[i]); + if(n > 0) { + freq = n; + continue; + } + } + + if(strcmp("debug", argv[i]) == 0) { + debug = 1; + debug_flags = 0xffffffff; + continue; + } + + if(!name) { + name = argv[i]; + continue; + } + + error("Cannot understand option %s", argv[i]); + exit(1); + } + + if(freq > 0) update_every = freq; + if(!name) name = "groups"; + + if(read_process_groups(name)) { + error("Cannot read process groups %s", name); + exit(1); + } +} + +int main(int argc, char **argv) +{ + // debug_flags = D_PROCFILE; + + // set the name for logging + program_name = "apps.plugin"; + + host_prefix = getenv("NETDATA_HOST_PREFIX"); + if(host_prefix == NULL) { + info("NETDATA_HOST_PREFIX is not passed from netdata"); + host_prefix = ""; + } + else info("Found NETDATA_HOST_PREFIX='%s'", host_prefix); + + info("starting..."); + + procfile_adaptive_initial_allocation = 1; + + time_t started_t = time(NULL); + time_t current_t; + Hertz = get_hertz(); + pid_max = get_pid_max(); + processors = get_processors(); + + parse_args(argc, argv); + + all_pids = calloc(sizeof(struct pid_stat *), (size_t) pid_max); + if(!all_pids) { + error("Cannot allocate %lu bytes of memory.", sizeof(struct pid_stat *) * pid_max); + printf("DISABLE\n"); + exit(1); + } + + unsigned long long counter = 1; + unsigned long long usec = 0, susec = 0; + struct timeval last, now; + gettimeofday(&last, NULL); + + for(;1; counter++) { + if(!update_from_proc()) { + error("Cannot allocate %lu bytes of memory.", sizeof(struct pid_stat *) * pid_max); + printf("DISABLE\n"); + exit(1); + } + + update_statistics(); + show_charts(); // this is smart enough to show only newly added apps, when needed + show_dimensions(); + + if(debug) fprintf(stderr, "apps.plugin: done Loop No %llu\n", counter); + fflush(NULL); + + gettimeofday(&now, NULL); + usec = usecdiff(&now, &last) - susec; + if(debug) fprintf(stderr, "apps.plugin: last loop took %llu usec (worked for %llu, sleeped for %llu).\n", usec + susec, usec, susec); + + // if the last loop took less than half the time + // wait the rest of the time + if(usec < (update_every * 1000000ULL / 2)) susec = (update_every * 1000000ULL) - usec; + else susec = update_every * 1000000ULL / 2; + + usleep((__useconds_t) susec); + bcopy(&now, &last, sizeof(struct timeval)); + + // restart once per day (14400 seconds) + current_t = time(NULL); + if(current_t - started_t > 14400) exit(0); + } +} diff --git a/src/avl.c b/src/avl.c new file mode 100755 index 000000000..4eb0ce0e4 --- /dev/null +++ b/src/avl.c @@ -0,0 +1,398 @@ +/* + * ANSI C Library for maintainance of AVL Balanced Trees + * + * ref.: + * G. M. Adelson-Velskij & E. M. Landis + * Doklady Akad. Nauk SSSR 146 (1962), 263-266 + * + * see also: + * D. E. Knuth: The Art of Computer Programming Vol.3 (Sorting and Searching) + * + * (C) 2000 Daniel Nagy, Budapest University of Technology and Economics + * Released under GNU General Public License (GPL) version 2 + * + */ + +#ifdef HAVE_CONFIG_H +#include <config.h> +#endif +#include "avl.h" + +/* Private methods */ +int _avl_removeroot(avl_tree* t); + +/* Swing to the left + * Warning: no balance maintainance + */ +void avl_swl(avl** root) { + avl* a = *root; + avl* b = a->right; + *root = b; + a->right = b->left; + b->left = a; +} + +/* Swing to the right + * Warning: no balance maintainance + */ +void avl_swr(avl** root) { + avl* a = *root; + avl* b = a->left; + *root = b; + a->left = b->right; + b->right = a; +} + +/* Balance maintainance after especially nasty swings + */ +void avl_nasty(avl* root) { + switch (root->balance) { + case -1: + root->left->balance = 0; + root->right->balance = 1; + break; + case 1: + root->left->balance = -1; + root->right->balance = 0; + break; + case 0: + root->left->balance = 0; + root->right->balance = 0; + } + root->balance = 0; +} + +/* Public methods */ + +/* Insert element a into the AVL tree t + * returns 1 if the depth of the tree has grown + * Warning: do not insert elements already present + */ +int _avl_insert(avl_tree* t, avl* a) { + /* initialize */ + a->left = 0; + a->right = 0; + a->balance = 0; + /* insert into an empty tree */ + if (!t->root) { + t->root = a; + return 1; + } + + if (t->compar(t->root, a) > 0) { + /* insert into the left subtree */ + if (t->root->left) { + avl_tree left_subtree; + left_subtree.root = t->root->left; + left_subtree.compar = t->compar; + if (_avl_insert(&left_subtree, a)) { + switch (t->root->balance--) { + case 1: + return 0; + case 0: + return 1; + } + if (t->root->left->balance < 0) { + avl_swr(&(t->root)); + t->root->balance = 0; + t->root->right->balance = 0; + } else { + avl_swl(&(t->root->left)); + avl_swr(&(t->root)); + avl_nasty(t->root); + } + } else + t->root->left = left_subtree.root; + return 0; + } else { + t->root->left = a; + if (t->root->balance--) + return 0; + return 1; + } + } else { + /* insert into the right subtree */ + if (t->root->right) { + avl_tree right_subtree; + right_subtree.root = t->root->right; + right_subtree.compar = t->compar; + if (_avl_insert(&right_subtree, a)) { + switch (t->root->balance++) { + case -1: + return 0; + case 0: + return 1; + } + if (t->root->right->balance > 0) { + avl_swl(&(t->root)); + t->root->balance = 0; + t->root->left->balance = 0; + } else { + avl_swr(&(t->root->right)); + avl_swl(&(t->root)); + avl_nasty(t->root); + } + } else + t->root->right = right_subtree.root; + return 0; + } else { + t->root->right = a; + if (t->root->balance++) + return 0; + return 1; + } + } +} +int avl_insert(avl_tree* t, avl* a) { +#ifdef AVL_LOCK_WITH_MUTEX + pthread_mutex_lock(&t->mutex); +#else + pthread_rwlock_wrlock(&t->rwlock); +#endif + + int ret = _avl_insert(t, a); + +#ifdef AVL_LOCK_WITH_MUTEX + pthread_mutex_unlock(&t->mutex); +#else + pthread_rwlock_unlock(&t->rwlock); +#endif + return ret; +} + +/* Remove an element a from the AVL tree t + * returns -1 if the depth of the tree has shrunk + * Warning: if the element is not present in the tree, + * returns 0 as if it had been removed succesfully. + */ +int _avl_remove(avl_tree* t, avl* a) { + int b; + if (t->root == a) + return _avl_removeroot(t); + b = t->compar(t->root, a); + if (b >= 0) { + /* remove from the left subtree */ + int ch; + avl_tree left_subtree; + if ((left_subtree.root = t->root->left)) { + left_subtree.compar = t->compar; + ch = _avl_remove(&left_subtree, a); + t->root->left = left_subtree.root; + if (ch) { + switch (t->root->balance++) { + case -1: + return -1; + case 0: + return 0; + } + switch (t->root->right->balance) { + case 0: + avl_swl(&(t->root)); + t->root->balance = -1; + t->root->left->balance = 1; + return 0; + case 1: + avl_swl(&(t->root)); + t->root->balance = 0; + t->root->left->balance = 0; + return -1; + } + avl_swr(&(t->root->right)); + avl_swl(&(t->root)); + avl_nasty(t->root); + return -1; + } + } + } + if (b <= 0) { + /* remove from the right subtree */ + int ch; + avl_tree right_subtree; + if ((right_subtree.root = t->root->right)) { + right_subtree.compar = t->compar; + ch = _avl_remove(&right_subtree, a); + t->root->right = right_subtree.root; + if (ch) { + switch (t->root->balance--) { + case 1: + return -1; + case 0: + return 0; + } + switch (t->root->left->balance) { + case 0: + avl_swr(&(t->root)); + t->root->balance = 1; + t->root->right->balance = -1; + return 0; + case -1: + avl_swr(&(t->root)); + t->root->balance = 0; + t->root->right->balance = 0; + return -1; + } + avl_swl(&(t->root->left)); + avl_swr(&(t->root)); + avl_nasty(t->root); + return -1; + } + } + } + return 0; +} + +int avl_remove(avl_tree* t, avl* a) { +#ifdef AVL_LOCK_WITH_MUTEX + pthread_mutex_lock(&t->mutex); +#else + pthread_rwlock_wrlock(&t->rwlock); +#endif + + int ret = _avl_remove(t, a); + +#ifdef AVL_LOCK_WITH_MUTEX + pthread_mutex_unlock(&t->mutex); +#else + pthread_rwlock_unlock(&t->rwlock); +#endif + return ret; +} + +/* Remove the root of the AVL tree t + * Warning: dumps core if t is empty + */ +int _avl_removeroot(avl_tree* t) { + int ch; + avl* a; + if (!t->root->left) { + if (!t->root->right) { + t->root = 0; + return -1; + } + t->root = t->root->right; + return -1; + } + if (!t->root->right) { + t->root = t->root->left; + return -1; + } + if (t->root->balance < 0) { + /* remove from the left subtree */ + a = t->root->left; + while (a->right) + a = a->right; + } else { + /* remove from the right subtree */ + a = t->root->right; + while (a->left) + a = a->left; + } + ch = _avl_remove(t, a); + a->left = t->root->left; + a->right = t->root->right; + a->balance = t->root->balance; + t->root = a; + if (a->balance == 0) + return ch; + return 0; +} + +int avl_removeroot(avl_tree* t) { +#ifdef AVL_LOCK_WITH_MUTEX + pthread_mutex_lock(&t->mutex); +#else + pthread_rwlock_wrlock(&t->rwlock); +#endif + + int ret = _avl_removeroot(t); + +#ifdef AVL_LOCK_WITH_MUTEX + pthread_mutex_unlock(&t->mutex); +#else + pthread_rwlock_unlock(&t->rwlock); +#endif + return ret; +} + +/* Iterate through elements in t from a range between a and b (inclusive) + * for each element calls iter(a) until it returns 0 + * returns the last value returned by iterator or 0 if there were no calls + * Warning: a<=b must hold + */ +int _avl_range(avl_tree* t, avl* a, avl* b, int (*iter)(avl*), avl** ret) { + int x, c = 0; + if (!t->root) + return 0; + x = t->compar(t->root, a); + if (a != b) { + if (x < 0) { + x = t->compar(t->root, b); + if (x > 0) + x = 0; + } + } + if (x >= 0) { + /* search in the left subtree */ + avl_tree left_subtree; + if ((left_subtree.root = t->root->left)) { + left_subtree.compar = t->compar; + if (!(c = _avl_range(&left_subtree, a, b, iter, ret))) + if (x > 0) + return 0; + } + } + if (x == 0) { + if (!(c = iter(t->root))) { + if (ret) + *ret = t->root; + return 0; + } + } + if (x <= 0) { + /* search in the right subtree */ + avl_tree right_subtree; + if ((right_subtree.root = t->root->right)) { + right_subtree.compar = t->compar; + if (!(c = _avl_range(&right_subtree, a, b, iter, ret))) + if (x < 0) + return 0; + } + } + return c; +} + +int avl_range(avl_tree* t, avl* a, avl* b, int (*iter)(avl*), avl** ret) { +#ifdef AVL_LOCK_WITH_MUTEX + pthread_mutex_lock(&t->mutex); +#else + pthread_rwlock_wrlock(&t->rwlock); +#endif + + int ret2 = _avl_range(t, a, b, iter, ret); + +#ifdef AVL_LOCK_WITH_MUTEX + pthread_mutex_unlock(&t->mutex); +#else + pthread_rwlock_unlock(&t->rwlock); +#endif + + return ret2; +} + +/* Iterate through elements in t equal to a + * for each element calls iter(a) until it returns 0 + * returns the last value returned by iterator or 0 if there were no calls + */ +int avl_search(avl_tree* t, avl* a, int (*iter)(avl* a), avl** ret) { + return avl_range(t, a, a, iter, ret); +} + +void avl_init(avl_tree* t, int (*compar)(void* a, void* b)) { + t->root = NULL; + t->compar = compar; +#ifdef AVL_LOCK_WITH_MUTEX + pthread_mutex_init(&t->mutex, NULL); +#else + pthread_rwlock_init(&t->rwlock, NULL); +#endif +} diff --git a/src/avl.h b/src/avl.h new file mode 100755 index 000000000..cbcc41211 --- /dev/null +++ b/src/avl.h @@ -0,0 +1,80 @@ +/* + * ANSI C Library for maintainance of AVL Balanced Trees + * + * ref.: + * G. M. Adelson-Velskij & E. M. Landis + * Doklady Akad. Nauk SSSR 146 (1962), 263-266 + * + * see also: + * D. E. Knuth: The Art of Computer Programming Vol.3 (Sorting and Searching) + * + * (C) 2000 Daniel Nagy, Budapest University of Technology and Economics + * Released under GNU General Public License (GPL) version 2 + * + */ +#ifndef _AVL_H +#define _AVL_H 1 + +#include <pthread.h> + +// #define AVL_LOCK_WITH_MUTEX 1 + +/* Data structures */ + +/* One element of the AVL tree */ +typedef struct avl { + struct avl* left; + struct avl* right; + signed char balance; +} avl; + +/* An AVL tree */ +typedef struct avl_tree { + avl* root; + int (*compar)(void* a, void* b); + +#ifdef AVL_LOCK_WITH_MUTEX + pthread_mutex_t mutex; +#else + pthread_rwlock_t rwlock; +#endif +} avl_tree; + +/* Public methods */ + +/* Insert element a into the AVL tree t + * returns 1 if the depth of the tree has grown + * Warning: do not insert elements already present + */ +int avl_insert(avl_tree* t, avl* a); + +/* Remove an element a from the AVL tree t + * returns -1 if the depth of the tree has shrunk + * Warning: if the element is not present in the tree, + * returns 0 as if it had been removed succesfully. + */ +int avl_remove(avl_tree* t, avl* a); + +/* Remove the root of the AVL tree t + * Warning: dumps core if t is empty + */ +int avl_removeroot(avl_tree* t); + +/* Iterate through elements in t from a range between a and b (inclusive) + * for each element calls iter(a) until it returns 0 + * returns the last value returned by iterator or 0 if there were no calls + * Warning: a<=b must hold + */ +int avl_range(avl_tree* t, avl* a, avl* b, int (*iter)(avl*), avl** ret); + +/* Iterate through elements in t equal to a + * for each element calls iter(a) until it returns 0 + * returns the last value returned by iterator or 0 if there were no calls + */ +int avl_search(avl_tree* t, avl* a, int (*iter)(avl*), avl** ret); + +/* Initialize the avl_tree + */ +void avl_init(avl_tree* t, int (*compar)(void* a, void* b)); + +#endif /* avl.h */ diff --git a/src/common.c b/src/common.c new file mode 100755 index 000000000..e3c3afe3f --- /dev/null +++ b/src/common.c @@ -0,0 +1,226 @@ +#ifdef HAVE_CONFIG_H +#include <config.h> +#endif +#include <sys/syscall.h> +#include <string.h> +#include <ctype.h> +#include <errno.h> +#include <unistd.h> +#include <sys/types.h> +#include <sys/stat.h> +#include <fcntl.h> +#include <sys/mman.h> + + +#include "log.h" +#include "common.h" +#include "appconfig.h" + +char *global_host_prefix = ""; +int enable_ksm = 1; + +/* +// http://stackoverflow.com/questions/7666509/hash-function-for-string +uint32_t simple_hash(const char *name) +{ + const char *s = name; + uint32_t hash = 5381; + int i; + + while((i = *s++)) hash = ((hash << 5) + hash) + i; + + // fprintf(stderr, "HASH: %lu %s\n", hash, name); + + return hash; +} +*/ + + +// http://isthe.com/chongo/tech/comp/fnv/#FNV-1a +uint32_t simple_hash(const char *name) { + unsigned char *s = (unsigned char *)name; + uint32_t hval = 0x811c9dc5; + + // FNV-1a algorithm + while (*s) { + // multiply by the 32 bit FNV magic prime mod 2^32 + // gcc optimized + hval += (hval<<1) + (hval<<4) + (hval<<7) + (hval<<8) + (hval<<24); + + // xor the bottom with the current octet + hval ^= (uint32_t)*s++; + } + + // fprintf(stderr, "HASH: %u = %s\n", hval, name); + return hval; +} + +/* +// http://eternallyconfuzzled.com/tuts/algorithms/jsw_tut_hashing.aspx +// one at a time hash +uint32_t simple_hash(const char *name) { + unsigned char *s = (unsigned char *)name; + uint32_t h = 0; + + while(*s) { + h += *s++; + h += (h << 10); + h ^= (h >> 6); + } + + h += (h << 3); + h ^= (h >> 11); + h += (h << 15); + + // fprintf(stderr, "HASH: %u = %s\n", h, name); + + return h; +} +*/ + +void strreverse(char* begin, char* end) +{ + char aux; + while (end > begin) + aux = *end, *end-- = *begin, *begin++ = aux; +} + +char *mystrsep(char **ptr, char *s) +{ + char *p = ""; + while ( p && !p[0] && *ptr ) p = strsep(ptr, s); + return(p); +} + +char *trim(char *s) +{ + // skip leading spaces + while(*s && isspace(*s)) s++; + if(!*s || *s == '#') return NULL; + + // skip tailing spaces + long c = (long) strlen(s) - 1; + while(c >= 0 && isspace(s[c])) { + s[c] = '\0'; + c--; + } + if(c < 0) return NULL; + if(!*s) return NULL; + return s; +} + +void *mymmap(const char *filename, size_t size, int flags, int ksm) +{ + int fd; + void *mem = NULL; + + errno = 0; + fd = open(filename, O_RDWR|O_CREAT|O_NOATIME, 0664); + if(fd != -1) { + if(lseek(fd, size, SEEK_SET) == (long)size) { + if(write(fd, "", 1) == 1) { + if(ftruncate(fd, size)) + error("Cannot truncate file '%s' to size %ld. Will use the larger file.", filename, size); + +#ifdef MADV_MERGEABLE + if(flags & MAP_SHARED || !enable_ksm || !ksm) { +#endif + mem = mmap(NULL, size, PROT_READ|PROT_WRITE, flags, fd, 0); + if(mem) { + int advise = MADV_SEQUENTIAL|MADV_DONTFORK; + if(flags & MAP_SHARED) advise |= MADV_WILLNEED; + + if(madvise(mem, size, advise) != 0) + error("Cannot advise the kernel about the memory usage of file '%s'.", filename); + } +#ifdef MADV_MERGEABLE + } + else { + mem = mmap(NULL, size, PROT_READ|PROT_WRITE, flags|MAP_ANONYMOUS, -1, 0); + if(mem) { + if(lseek(fd, 0, SEEK_SET) == 0) { + if(read(fd, mem, size) != (ssize_t)size) + error("Cannot read from file '%s'", filename); + } + else + error("Cannot seek to beginning of file '%s'.", filename); + + // don't use MADV_SEQUENTIAL|MADV_DONTFORK, they disable MADV_MERGEABLE + if(madvise(mem, size, MADV_SEQUENTIAL|MADV_DONTFORK) != 0) + error("Cannot advise the kernel about the memory usage (MADV_SEQUENTIAL|MADV_DONTFORK) of file '%s'.", filename); + + if(madvise(mem, size, MADV_MERGEABLE) != 0) + error("Cannot advise the kernel about the memory usage (MADV_MERGEABLE) of file '%s'.", filename); + } + else + error("Cannot allocate PRIVATE ANONYMOUS memory for KSM for file '%s'.", filename); + } +#endif + } + else error("Cannot write to file '%s' at position %ld.", filename, size); + } + else error("Cannot seek file '%s' to size %ld.", filename, size); + + close(fd); + } + else error("Cannot create/open file '%s'.", filename); + + return mem; +} + +int savememory(const char *filename, void *mem, unsigned long size) +{ + char tmpfilename[FILENAME_MAX + 1]; + + snprintf(tmpfilename, FILENAME_MAX, "%s.%ld.tmp", filename, (long)getpid()); + + int fd = open(tmpfilename, O_RDWR|O_CREAT|O_NOATIME, 0664); + if(fd < 0) { + error("Cannot create/open file '%s'.", filename); + return -1; + } + + if(write(fd, mem, size) != (long)size) { + error("Cannot write to file '%s' %ld bytes.", filename, (long)size); + close(fd); + return -1; + } + + close(fd); + + int ret = 0; + if(rename(tmpfilename, filename)) { + error("Cannot rename '%s' to '%s'", tmpfilename, filename); + ret = -1; + } + + return ret; +} + +int fd_is_valid(int fd) { + return fcntl(fd, F_GETFD) != -1 || errno != EBADF; +} + +/* + *************************************************************************** + * Get number of clock ticks per second. + *************************************************************************** + */ +unsigned int hz; + +void get_HZ(void) +{ + long ticks; + + if ((ticks = sysconf(_SC_CLK_TCK)) == -1) { + perror("sysconf"); + } + + hz = (unsigned int) ticks; +} + +pid_t gettid(void) +{ + return syscall(SYS_gettid); +} + diff --git a/src/common.h b/src/common.h new file mode 100755 index 000000000..9d8836f8b --- /dev/null +++ b/src/common.h @@ -0,0 +1,35 @@ +#ifndef NETDATA_COMMON_H +#define NETDATA_COMMON_H 1 + +#if defined(HAVE_INTTYPES_H) +#include <inttypes.h> +#elif defined(HAVE_STDINT_H) +#include <stdint.h> +#endif +#include <sys/types.h> +#include <unistd.h> + +#define abs(x) ((x < 0)? -x : x) +#define usecdiff(now, last) (((((now)->tv_sec * 1000000ULL) + (now)->tv_usec) - (((last)->tv_sec * 1000000ULL) + (last)->tv_usec))) + +extern uint32_t simple_hash(const char *name); +extern void strreverse(char* begin, char* end); +extern char *mystrsep(char **ptr, char *s); +extern char *trim(char *s); + +extern void *mymmap(const char *filename, size_t size, int flags, int ksm); +extern int savememory(const char *filename, void *mem, unsigned long size); + +extern int fd_is_valid(int fd); + +extern char *global_host_prefix; +extern int enable_ksm; + +/* Number of ticks per second */ +#define HZ myhz +extern unsigned int hz; +extern void get_HZ(void); + +extern pid_t gettid(void); + +#endif /* NETDATA_COMMON_H */ diff --git a/src/daemon.c b/src/daemon.c new file mode 100755 index 000000000..268814798 --- /dev/null +++ b/src/daemon.c @@ -0,0 +1,317 @@ +#ifdef HAVE_CONFIG_H +#include <config.h> +#endif +#include <stdio.h> +#include <stdlib.h> +#include <unistd.h> +#include <fcntl.h> +#include <errno.h> +#include <signal.h> +#include <string.h> +#include <sys/types.h> +#include <pwd.h> +#include <pthread.h> +#include <sys/wait.h> +#include <sys/stat.h> +#include <execinfo.h> + +#include "common.h" +#include "appconfig.h" +#include "log.h" +#include "web_client.h" +#include "plugins_d.h" +#include "rrd.h" +#include "popen.h" +#include "main.h" +#include "daemon.h" + +void sig_handler(int signo) +{ + switch(signo) { + case SIGILL: + case SIGABRT: + case SIGFPE: + case SIGSEGV: + case SIGBUS: + case SIGSYS: + case SIGTRAP: + case SIGXCPU: + case SIGXFSZ: + infoerr("Death signaled exit (signal %d).", signo); + signal(signo, SIG_DFL); + break; + + case SIGKILL: + case SIGTERM: + case SIGQUIT: + case SIGINT: + case SIGHUP: + case SIGUSR1: + case SIGUSR2: + infoerr("Signaled exit (signal %d).", signo); + signal(SIGPIPE, SIG_IGN); + signal(SIGTERM, SIG_IGN); + signal(SIGQUIT, SIG_IGN); + signal(SIGHUP, SIG_IGN); + signal(SIGINT, SIG_IGN); + signal(SIGCHLD, SIG_IGN); + netdata_cleanup_and_exit(1); + break; + + case SIGPIPE: + infoerr("Signaled PIPE (signal %d).", signo); + // this is received when web clients send a reset + // no need to log it. + // infoerr("Ignoring signal %d.", signo); + break; + + default: + info("Signal %d received. Falling back to default action for it.", signo); + signal(signo, SIG_DFL); + break; + } +} + +char rundir[FILENAME_MAX + 1] = "/var/run/netdata"; +char pidfile[FILENAME_MAX + 1] = ""; +void prepare_rundir() { + if(getuid() != 0) { + mkdir("/run/user", 0775); + snprintf(rundir, FILENAME_MAX, "/run/user/%d", getuid()); + mkdir(rundir, 0775); + snprintf(rundir, FILENAME_MAX, "/run/user/%d/netdata", getuid()); + } + + snprintf(pidfile, FILENAME_MAX, "%s/netdata.pid", rundir); + + if(mkdir(rundir, 0775) != 0) { + if(errno != EEXIST) error("Cannot create directory '%s'.", rundir); + } +} + +int become_user(const char *username) +{ + struct passwd *pw = getpwnam(username); + if(!pw) { + error("User %s is not present.", username); + return -1; + } + + if(chown(rundir, pw->pw_uid, pw->pw_gid) != 0) { + error("Cannot chown directory '%s' to user %s.", rundir, username); + return -1; + } + + if(setresgid(pw->pw_gid, pw->pw_gid, pw->pw_gid) != 0) { + error("Cannot switch to user's %s group (gid: %d).", username, pw->pw_gid); + return -1; + } + + if(setresuid(pw->pw_uid, pw->pw_uid, pw->pw_uid) != 0) { + error("Cannot switch to user %s (uid: %d).", username, pw->pw_uid); + return -1; + } + + if(setgid(pw->pw_gid) != 0) { + error("Cannot switch to user's %s group (gid: %d).", username, pw->pw_gid); + return -1; + } + if(setegid(pw->pw_gid) != 0) { + error("Cannot effectively switch to user's %s group (gid: %d).", username, pw->pw_gid); + return -1; + } + if(setuid(pw->pw_uid) != 0) { + error("Cannot switch to user %s (uid: %d).", username, pw->pw_uid); + return -1; + } + if(seteuid(pw->pw_uid) != 0) { + error("Cannot effectively switch to user %s (uid: %d).", username, pw->pw_uid); + return -1; + } + + return(0); +} + +int become_daemon(int dont_fork, int close_all_files, const char *user, const char *input, const char *output, const char *error, const char *access, int *access_fd, FILE **access_fp) +{ + fflush(NULL); + + // open the files before forking + int input_fd = -1, output_fd = -1, error_fd = -1, dev_null; + + if(input && *input) { + if((input_fd = open(input, O_RDONLY, 0666)) == -1) { + error("Cannot open input file '%s'.", input); + return -1; + } + } + + if(output && *output) { + if((output_fd = open(output, O_RDWR | O_APPEND | O_CREAT, 0666)) == -1) { + error("Cannot open output log file '%s'", output); + if(input_fd != -1) close(input_fd); + return -1; + } + } + + if(error && *error) { + if((error_fd = open(error, O_RDWR | O_APPEND | O_CREAT, 0666)) == -1) { + error("Cannot open error log file '%s'.", error); + if(input_fd != -1) close(input_fd); + if(output_fd != -1) close(output_fd); + return -1; + } + } + + if(access && *access && access_fd) { + if((*access_fd = open(access, O_RDWR | O_APPEND | O_CREAT, 0666)) == -1) { + error("Cannot open access log file '%s'", access); + if(input_fd != -1) close(input_fd); + if(output_fd != -1) close(output_fd); + if(error_fd != -1) close(error_fd); + return -1; + } + + if(access_fp) { + *access_fp = fdopen(*access_fd, "w"); + if(!*access_fp) { + error("Cannot migrate file's '%s' fd %d.", access, *access_fd); + if(input_fd != -1) close(input_fd); + if(output_fd != -1) close(output_fd); + if(error_fd != -1) close(error_fd); + close(*access_fd); + *access_fd = -1; + return -1; + } + } + } + + if((dev_null = open("/dev/null", O_RDWR, 0666)) == -1) { + perror("Cannot open /dev/null"); + if(input_fd != -1) close(input_fd); + if(output_fd != -1) close(output_fd); + if(error_fd != -1) close(error_fd); + if(access && access_fd && *access_fd != -1) { + close(*access_fd); + *access_fd = -1; + if(access_fp) { + fclose(*access_fp); + *access_fp = NULL; + } + } + return -1; + } + + // all files opened + // lets do it + + if(!dont_fork) { + int i = fork(); + if(i == -1) { + perror("cannot fork"); + exit(1); + } + if(i != 0) { + exit(0); // the parent + } + + // become session leader + if (setsid() < 0) { + perror("Cannot become session leader."); + exit(2); + } + } + + signal(SIGCHLD, SIG_IGN); + signal(SIGHUP, SIG_IGN); + signal(SIGWINCH, SIG_IGN); + + // fork() again + if(!dont_fork) { + int i = fork(); + if(i == -1) { + perror("cannot fork"); + exit(1); + } + if(i != 0) { + exit(0); // the parent + } + } + + // Set new file permissions + umask(0); + + // close all files + if(close_all_files) { + int i; + for(i = (int) (sysconf(_SC_OPEN_MAX) - 1); i > 0; i--) + if( + ((access_fd && i != *access_fd) || !access_fd) + && i != dev_null + && i != input_fd + && i != output_fd + && i != error_fd + && fd_is_valid(i) + ) close(i); + } + else { + close(STDIN_FILENO); + close(STDOUT_FILENO); + close(STDERR_FILENO); + } + + // put the opened files + // to our standard file descriptors + if(input_fd != -1) { + if(input_fd != STDIN_FILENO) { + dup2(input_fd, STDIN_FILENO); + close(input_fd); + } + input_fd = -1; + } + else dup2(dev_null, STDIN_FILENO); + + if(output_fd != -1) { + if(output_fd != STDOUT_FILENO) { + dup2(output_fd, STDOUT_FILENO); + close(output_fd); + } + output_fd = -1; + } + else dup2(dev_null, STDOUT_FILENO); + + if(error_fd != -1) { + if(error_fd != STDERR_FILENO) { + dup2(error_fd, STDERR_FILENO); + close(error_fd); + } + error_fd = -1; + } + else dup2(dev_null, STDERR_FILENO); + + // close /dev/null + if(dev_null != STDIN_FILENO && dev_null != STDOUT_FILENO && dev_null != STDERR_FILENO) + close(dev_null); + + // generate our pid file + { + unlink(pidfile); + int fd = open(pidfile, O_RDWR | O_CREAT, 0666); + if(fd >= 0) { + char b[100]; + sprintf(b, "%d\n", getpid()); + ssize_t i = write(fd, b, strlen(b)); + if(i <= 0) perror("Cannot write pid to file."); + close(fd); + } + } + + if(user && *user) { + if(become_user(user) != 0) { + error("Cannot become user '%s'. Continuing as we are.", user); + } + else info("Successfully became user '%s'.", user); + } + + return(0); +} diff --git a/src/daemon.h b/src/daemon.h new file mode 100755 index 000000000..77186daae --- /dev/null +++ b/src/daemon.h @@ -0,0 +1,14 @@ +#ifndef NETDATA_DAEMON_H +#define NETDATA_DAEMON_H 1 + +extern void sig_handler(int signo); + +extern void prepare_rundir(); + +extern int become_user(const char *username); + +extern int become_daemon(int dont_fork, int close_all_files, const char *user, const char *input, const char *output, const char *error, const char *access, int *access_fd, FILE **access_fp); + +extern void netdata_cleanup_and_exit(int i); + +#endif /* NETDATA_DAEMON_H */ diff --git a/src/dictionary.c b/src/dictionary.c new file mode 100755 index 000000000..31f4d52e1 --- /dev/null +++ b/src/dictionary.c @@ -0,0 +1,160 @@ +#ifdef HAVE_CONFIG_H +#include <config.h> +#endif +#include <pthread.h> +#include <stdlib.h> +#include <string.h> + +#include "avl.h" +#include "common.h" +#include "log.h" + +#include "dictionary.h" + +// ---------------------------------------------------------------------------- +// name_value index + +static int name_value_iterator(avl *a) { if(a) {}; return 0; } + +static int name_value_compare(void* a, void* b) { + if(((NAME_VALUE *)a)->hash < ((NAME_VALUE *)b)->hash) return -1; + else if(((NAME_VALUE *)a)->hash > ((NAME_VALUE *)b)->hash) return 1; + else return strcmp(((NAME_VALUE *)a)->name, ((NAME_VALUE *)b)->name); +} + +#define name_value_index_add(dict, cv) avl_insert(&((dict)->values_index), (avl *)(cv)) +#define name_value_index_del(dict, cv) avl_remove(&((dict)->values_index), (avl *)(cv)) + +static NAME_VALUE *dictionary_name_value_index_find(DICTIONARY *dict, const char *name, uint32_t hash) { + NAME_VALUE *result = NULL, tmp; + tmp.hash = (hash)?hash:simple_hash(name); + tmp.name = (char *)name; + + avl_search(&(dict->values_index), (avl *)&tmp, name_value_iterator, (avl **)&result); + return result; +} + +// ---------------------------------------------------------------------------- + +static NAME_VALUE *dictionary_name_value_create(DICTIONARY *dict, const char *name, void *value, size_t value_len) { + debug(D_DICTIONARY, "Creating name value entry for name '%s', value '%s'.", name, value); + + NAME_VALUE *nv = calloc(1, sizeof(NAME_VALUE)); + if(!nv) { + fatal("Cannot allocate name_value of size %z", sizeof(NAME_VALUE)); + exit(1); + } + + nv->name = strdup(name); + if(!nv->name) fatal("Cannot allocate name_value.name of size %z", strlen(name)); + nv->hash = simple_hash(nv->name); + + nv->value = malloc(value_len); + if(!nv->value) fatal("Cannot allocate name_value.value of size %z", value_len); + memcpy(nv->value, value, value_len); + + // link it + pthread_rwlock_wrlock(&dict->rwlock); + nv->next = dict->values; + dict->values = nv; + pthread_rwlock_unlock(&dict->rwlock); + + // index it + name_value_index_add(dict, nv); + + return nv; +} + +static void dictionary_name_value_destroy(DICTIONARY *dict, NAME_VALUE *nv) { + debug(D_DICTIONARY, "Destroying name value entry for name '%s'.", nv->name); + + pthread_rwlock_wrlock(&dict->rwlock); + if(dict->values == nv) dict->values = nv->next; + else { + NAME_VALUE *n = dict->values; + while(n && n->next && n->next != nv) nv = nv->next; + if(!n || n->next != nv) { + fatal("Cannot find name_value with name '%s' in dictionary.", nv->name); + exit(1); + } + n->next = nv->next; + nv->next = NULL; + } + pthread_rwlock_unlock(&dict->rwlock); + + free(nv->value); + free(nv); +} + +// ---------------------------------------------------------------------------- + +DICTIONARY *dictionary_create(void) { + debug(D_DICTIONARY, "Creating dictionary."); + + DICTIONARY *dict = calloc(1, sizeof(DICTIONARY)); + if(!dict) { + fatal("Cannot allocate DICTIONARY"); + exit(1); + } + + avl_init(&dict->values_index, name_value_compare); + pthread_rwlock_init(&dict->rwlock, NULL); + + return dict; +} + +void dictionary_destroy(DICTIONARY *dict) { + debug(D_DICTIONARY, "Destroying dictionary."); + + pthread_rwlock_wrlock(&dict->rwlock); + while(dict->values) dictionary_name_value_destroy(dict, dict->values); + pthread_rwlock_unlock(&dict->rwlock); + + free(dict); +} + +// ---------------------------------------------------------------------------- + +void *dictionary_set(DICTIONARY *dict, const char *name, void *value, size_t value_len) { + debug(D_DICTIONARY, "SET dictionary entry with name '%s'.", name); + + pthread_rwlock_rdlock(&dict->rwlock); + NAME_VALUE *nv = dictionary_name_value_index_find(dict, name, 0); + pthread_rwlock_unlock(&dict->rwlock); + if(!nv) { + debug(D_DICTIONARY, "Dictionary entry with name '%s' not found. Creating a new one.", name); + nv = dictionary_name_value_create(dict, name, value, value_len); + if(!nv) { + fatal("Cannot create name_value."); + exit(1); + } + return nv->value; + } + else { + debug(D_DICTIONARY, "Dictionary entry with name '%s' found. Changing its value.", name); + pthread_rwlock_wrlock(&dict->rwlock); + void *old = nv->value; + nv->value = malloc(value_len); + if(!nv->value) fatal("Cannot allocate value of size %z", value_len); + memcpy(nv->value, value, value_len); + pthread_rwlock_unlock(&dict->rwlock); + free(old); + } + + return nv->value; +} + +void *dictionary_get(DICTIONARY *dict, const char *name) { + debug(D_DICTIONARY, "GET dictionary entry with name '%s'.", name); + + pthread_rwlock_rdlock(&dict->rwlock); + NAME_VALUE *nv = dictionary_name_value_index_find(dict, name, 0); + pthread_rwlock_unlock(&dict->rwlock); + if(!nv) { + debug(D_DICTIONARY, "Not found dictionary entry with name '%s'.", name); + return NULL; + } + + debug(D_DICTIONARY, "Found dictionary entry with name '%s'.", name); + return nv->value; +} diff --git a/src/dictionary.h b/src/dictionary.h new file mode 100755 index 000000000..9822b23c2 --- /dev/null +++ b/src/dictionary.h @@ -0,0 +1,29 @@ +#include "web_buffer.h" + +#ifndef NETDATA_DICTIONARY_H +#define NETDATA_DICTIONARY_H 1 + +typedef struct name_value { + avl avl; // the index - this has to be first! + + uint32_t hash; // a simple hash to speed up searching + // we first compare hashes, and only if the hashes are equal we do string comparisons + + char *name; + char *value; + + struct name_value *next; +} NAME_VALUE; + +typedef struct dictionary { + NAME_VALUE *values; + avl_tree values_index; + pthread_rwlock_t rwlock; +} DICTIONARY; + +extern DICTIONARY *dictionary_create(void); +extern void dictionary_destroy(DICTIONARY *dict); +extern void *dictionary_set(DICTIONARY *dict, const char *name, void *value, size_t value_len); +extern void *dictionary_get(DICTIONARY *dict, const char *name); + +#endif /* NETDATA_DICTIONARY_H */ diff --git a/src/global_statistics.c b/src/global_statistics.c new file mode 100755 index 000000000..d4a04efd2 --- /dev/null +++ b/src/global_statistics.c @@ -0,0 +1,18 @@ +#ifdef HAVE_CONFIG_H +#include <config.h> +#endif +#include <pthread.h> + +#include "global_statistics.h" + +struct global_statistics global_statistics = { 0ULL, 0ULL, 0ULL, 0ULL }; + +pthread_mutex_t global_statistics_mutex = PTHREAD_MUTEX_INITIALIZER; + +void global_statistics_lock(void) { + pthread_mutex_lock(&global_statistics_mutex); +} + +void global_statistics_unlock(void) { + pthread_mutex_unlock(&global_statistics_mutex); +} diff --git a/src/global_statistics.h b/src/global_statistics.h new file mode 100755 index 000000000..ce3c3490e --- /dev/null +++ b/src/global_statistics.h @@ -0,0 +1,20 @@ +#ifndef NETDATA_GLOBAL_STATISTICS_H +#define NETDATA_GLOBAL_STATISTICS_H 1 + +// ---------------------------------------------------------------------------- +// global statistics + +struct global_statistics { + unsigned long long connected_clients; + unsigned long long web_requests; + unsigned long long bytes_received; + unsigned long long bytes_sent; + +}; + +extern struct global_statistics global_statistics; + +extern void global_statistics_lock(void); +extern void global_statistics_unlock(void); + +#endif /* NETDATA_GLOBAL_STATISTICS_H */ diff --git a/src/log.c b/src/log.c new file mode 100755 index 000000000..7ab3f1a51 --- /dev/null +++ b/src/log.c @@ -0,0 +1,157 @@ +#ifdef HAVE_CONFIG_H +#include <config.h> +#endif +#include <time.h> +#include <syslog.h> +#include <errno.h> +#include <string.h> +#include <unistd.h> +#include <stdlib.h> + +#include "log.h" +#include "common.h" + + +// ---------------------------------------------------------------------------- +// LOG + +const char *program_name = ""; +unsigned long long debug_flags = DEBUG; + +int silent = 0; + +int access_fd = -1; +FILE *stdaccess = NULL; + +int access_log_syslog = 1; +int error_log_syslog = 1; +int output_log_syslog = 1; // debug log + +void log_date(FILE *out) +{ + char outstr[200]; + time_t t; + struct tm *tmp, tmbuf; + + t = time(NULL); + tmp = localtime_r(&t, &tmbuf); + + if (tmp == NULL) return; + if (strftime(outstr, sizeof(outstr), "%y-%m-%d %H:%M:%S", tmp) == 0) return; + + fprintf(out, "%s: ", outstr); +} + +void debug_int( const char *file, const char *function, const unsigned long line, const char *fmt, ... ) +{ + va_list args; + + log_date(stdout); + va_start( args, fmt ); + fprintf(stdout, "DEBUG (%04lu@%-10.10s:%-15.15s): %s: ", line, file, function, program_name); + vfprintf( stdout, fmt, args ); + va_end( args ); + fprintf(stdout, "\n"); + + if(output_log_syslog) { + va_start( args, fmt ); + vsyslog(LOG_ERR, fmt, args ); + va_end( args ); + } +} + +void info_int( const char *file, const char *function, const unsigned long line, const char *fmt, ... ) +{ + va_list args; + + log_date(stderr); + + va_start( args, fmt ); + if(debug_flags) fprintf(stderr, "INFO (%04lu@%-10.10s:%-15.15s): %s: ", line, file, function, program_name); + else fprintf(stderr, "INFO: %s: ", program_name); + vfprintf( stderr, fmt, args ); + va_end( args ); + + fprintf(stderr, "\n"); + + if(error_log_syslog) { + va_start( args, fmt ); + vsyslog(LOG_INFO, fmt, args ); + va_end( args ); + } +} + +void error_int( const char *prefix, const char *file, const char *function, const unsigned long line, const char *fmt, ... ) +{ + va_list args; + + log_date(stderr); + + va_start( args, fmt ); + if(debug_flags) fprintf(stderr, "%s (%04lu@%-10.10s:%-15.15s): %s: ", prefix, line, file, function, program_name); + else fprintf(stderr, "%s: %s: ", prefix, program_name); + vfprintf( stderr, fmt, args ); + va_end( args ); + + if(errno) { + char buf[200]; + char *s = strerror_r(errno, buf, 200); + fprintf(stderr, " (errno %d, %s)\n", errno, s); + errno = 0; + } + else fprintf(stderr, "\n"); + + if(error_log_syslog) { + va_start( args, fmt ); + vsyslog(LOG_ERR, fmt, args ); + va_end( args ); + } +} + +void fatal_int( const char *file, const char *function, const unsigned long line, const char *fmt, ... ) +{ + va_list args; + + log_date(stderr); + + va_start( args, fmt ); + if(debug_flags) fprintf(stderr, "FATAL (%04lu@%-10.10s:%-15.15s): %s: ", line, file, function, program_name); + else fprintf(stderr, "FATAL: %s: ", program_name); + vfprintf( stderr, fmt, args ); + va_end( args ); + + perror(" # "); + fprintf(stderr, "\n"); + + if(error_log_syslog) { + va_start( args, fmt ); + vsyslog(LOG_CRIT, fmt, args ); + va_end( args ); + } + + exit(1); +} + +void log_access( const char *fmt, ... ) +{ + va_list args; + + if(stdaccess) { + log_date(stdaccess); + + va_start( args, fmt ); + vfprintf( stdaccess, fmt, args ); + va_end( args ); + fprintf( stdaccess, "\n"); +#ifdef NETDATA_INTERNAL_CHECKS + fflush( stdaccess ); +#endif + } + + if(access_log_syslog) { + va_start( args, fmt ); + vsyslog(LOG_INFO, fmt, args ); + va_end( args ); + } +} + diff --git a/src/log.h b/src/log.h new file mode 100755 index 000000000..08f3c4fe3 --- /dev/null +++ b/src/log.h @@ -0,0 +1,58 @@ +#include <stdio.h> +#include <stdarg.h> + +#ifndef NETDATA_LOG_H +#define NETDATA_LOG_H 1 + +#define D_WEB_BUFFER 0x00000001 +#define D_WEB_CLIENT 0x00000002 +#define D_LISTENER 0x00000004 +#define D_WEB_DATA 0x00000008 +#define D_OPTIONS 0x00000010 +#define D_PROCNETDEV_LOOP 0x00000020 +#define D_RRD_STATS 0x00000040 +#define D_WEB_CLIENT_ACCESS 0x00000080 +#define D_TC_LOOP 0x00000100 +#define D_DEFLATE 0x00000200 +#define D_CONFIG 0x00000400 +#define D_PLUGINSD 0x00000800 +#define D_CHILDS 0x00001000 +#define D_EXIT 0x00002000 +#define D_CHECKS 0x00004000 +#define D_NFACCT_LOOP 0x00008000 +#define D_PROCFILE 0x00010000 +#define D_RRD_CALLS 0x00020000 +#define D_DICTIONARY 0x00040000 +#define D_MEMORY 0x00080000 + +//#define DEBUG (D_WEB_CLIENT_ACCESS|D_LISTENER|D_RRD_STATS) +//#define DEBUG 0xffffffff +#define DEBUG (0) + +extern unsigned long long debug_flags; + +extern const char *program_name; + +extern int silent; + +extern int access_fd; +extern FILE *stdaccess; + +extern int access_log_syslog; +extern int error_log_syslog; +extern int output_log_syslog; + +#define debug(type, args...) do { if(unlikely(!silent && (debug_flags & type))) debug_int(__FILE__, __FUNCTION__, __LINE__, ##args); } while(0) +#define info(args...) info_int(__FILE__, __FUNCTION__, __LINE__, ##args) +#define infoerr(args...) error_int("INFO", __FILE__, __FUNCTION__, __LINE__, ##args) +#define error(args...) error_int("ERROR", __FILE__, __FUNCTION__, __LINE__, ##args) +#define fatal(args...) fatal_int(__FILE__, __FUNCTION__, __LINE__, ##args) + +extern void log_date(FILE *out); +extern void debug_int( const char *file, const char *function, const unsigned long line, const char *fmt, ... ); +extern void info_int( const char *file, const char *function, const unsigned long line, const char *fmt, ... ); +extern void error_int( const char *prefix, const char *file, const char *function, const unsigned long line, const char *fmt, ... ); +extern void fatal_int( const char *file, const char *function, const unsigned long line, const char *fmt, ... ) __attribute__ ((noreturn)); +extern void log_access( const char *fmt, ... ); + +#endif /* NETDATA_LOG_H */ diff --git a/src/main.c b/src/main.c new file mode 100755 index 000000000..a29757358 --- /dev/null +++ b/src/main.c @@ -0,0 +1,479 @@ +#ifdef HAVE_CONFIG_H +#include <config.h> +#endif +#include <stdio.h> +#include <stdlib.h> +#include <unistd.h> +#include <signal.h> +#include <syslog.h> +#include <pthread.h> +#include <string.h> +#include <errno.h> +#include <sys/types.h> +#include <sys/wait.h> +#include <sys/time.h> +#include <sys/resource.h> +#include <sys/mman.h> + +#include "common.h" +#include "log.h" +#include "daemon.h" +#include "web_server.h" +#include "popen.h" +#include "appconfig.h" +#include "web_client.h" +#include "rrd.h" +#include "rrd2json.h" + +#include "unit_test.h" + +#include "plugins_d.h" +#include "plugin_idlejitter.h" +#include "plugin_tc.h" +#include "plugin_checks.h" +#include "plugin_proc.h" +#include "plugin_nfacct.h" + +#include "main.h" + +int netdata_exit = 0; + +void netdata_cleanup_and_exit(int ret) +{ + netdata_exit = 1; + rrdset_save_all(); + // kill_childs(); + unlink("/var/run/netdata.pid"); + info("NetData exiting. Bye bye..."); + exit(ret); +} + +struct netdata_static_thread { + char *name; + + char *config_section; + char *config_name; + + int enabled; + + pthread_t *thread; + + void (*init_routine) (void); + void *(*start_routine) (void *); +}; + +struct netdata_static_thread static_threads[] = { + {"tc", "plugins", "tc", 1, NULL, NULL, tc_main}, + {"idlejitter", "plugins", "idlejitter", 1, NULL, NULL, cpuidlejitter_main}, + {"proc", "plugins", "proc", 1, NULL, NULL, proc_main}, + +#ifdef INTERNAL_PLUGIN_NFACCT + // nfacct requires root access + // so, we build it as an external plugin with setuid to root + {"nfacct", "plugins", "nfacct", 1, NULL, NULL, nfacct_main}, +#endif + + {"plugins.d", NULL, NULL, 1, NULL, NULL, pluginsd_main}, + {"check", "plugins", "checks", 0, NULL, NULL, checks_main}, + {"web", NULL, NULL, 1, NULL, NULL, socket_listen_main}, + {NULL, NULL, NULL, 0, NULL, NULL, NULL} +}; + +int killpid(pid_t pid, int sig) +{ + int ret = -1; + debug(D_EXIT, "Request to kill pid %d", pid); + + errno = 0; + if(kill(pid, 0) == -1) { + switch(errno) { + case ESRCH: + error("Request to kill pid %d, but it is not running.", pid); + break; + + case EPERM: + error("Request to kill pid %d, but I do not have enough permissions.", pid); + break; + + default: + error("Request to kill pid %d, but I received an error.", pid); + break; + } + } + else { + errno = 0; + + void (*old)(int); + old = signal(sig, SIG_IGN); + if(old == SIG_ERR) { + error("Cannot overwrite signal handler for signal %d", sig); + old = sig_handler; + } + + ret = kill(pid, sig); + + if(signal(sig, old) == SIG_ERR) + error("Cannot restore signal handler for signal %d", sig); + + if(ret == -1) { + switch(errno) { + case ESRCH: + error("Cannot kill pid %d, but it is not running.", pid); + break; + + case EPERM: + error("Cannot kill pid %d, but I do not have enough permissions.", pid); + break; + + default: + error("Cannot kill pid %d, but I received an error.", pid); + break; + } + } + } + + return ret; +} + +void kill_childs() +{ + siginfo_t info; + + struct web_client *w; + for(w = web_clients; w ; w = w->next) { + debug(D_EXIT, "Stopping web client %s", w->client_ip); + pthread_cancel(w->thread); + pthread_join(w->thread, NULL); + } + + int i; + for (i = 0; static_threads[i].name != NULL ; i++) { + if(static_threads[i].thread) { + debug(D_EXIT, "Stopping %s thread", static_threads[i].name); + pthread_cancel(*static_threads[i].thread); + pthread_join(*static_threads[i].thread, NULL); + static_threads[i].thread = NULL; + } + } + + if(tc_child_pid) { + info("Killing tc-qos-helper procees"); + if(killpid(tc_child_pid, SIGTERM) != -1) + waitid(P_PID, (id_t) tc_child_pid, &info, WEXITED); + } + tc_child_pid = 0; + + struct plugind *cd; + for(cd = pluginsd_root ; cd ; cd = cd->next) { + debug(D_EXIT, "Stopping %s plugin thread", cd->id); + pthread_cancel(cd->thread); + pthread_join(cd->thread, NULL); + + if(cd->pid && !cd->obsolete) { + debug(D_EXIT, "killing %s plugin process", cd->id); + if(killpid(cd->pid, SIGTERM) != -1) + waitid(P_PID, (id_t) cd->pid, &info, WEXITED); + } + } + + // if, for any reason there is any child exited + // catch it here + waitid(P_PID, 0, &info, WEXITED|WNOHANG); + + debug(D_EXIT, "All threads/childs stopped."); +} + + +int main(int argc, char **argv) +{ + int i; + int config_loaded = 0; + int dont_fork = 0; + + // global initialization + get_HZ(); + + // set the name for logging + program_name = "netdata"; + + // parse the arguments + for(i = 1; i < argc ; i++) { + if(strcmp(argv[i], "-c") == 0 && (i+1) < argc) { + if(load_config(argv[i+1], 1) != 1) { + error("Cannot load configuration file %s.", argv[i+1]); + exit(1); + } + else { + debug(D_OPTIONS, "Configuration loaded from %s.", argv[i+1]); + config_loaded = 1; + } + i++; + } + else if(strcmp(argv[i], "-df") == 0 && (i+1) < argc) { config_set("global", "debug flags", argv[i+1]); debug_flags = strtoull(argv[i+1], NULL, 0); i++; } + else if(strcmp(argv[i], "-p") == 0 && (i+1) < argc) { config_set("global", "port", argv[i+1]); i++; } + else if(strcmp(argv[i], "-u") == 0 && (i+1) < argc) { config_set("global", "run as user", argv[i+1]); i++; } + else if(strcmp(argv[i], "-l") == 0 && (i+1) < argc) { config_set("global", "history", argv[i+1]); i++; } + else if(strcmp(argv[i], "-t") == 0 && (i+1) < argc) { config_set("global", "update every", argv[i+1]); i++; } + else if(strcmp(argv[i], "-ch") == 0 && (i+1) < argc) { config_set("global", "host access prefix", argv[i+1]); i++; } + else if(strcmp(argv[i], "-nodeamon") == 0 || strcmp(argv[i], "-nd") == 0) dont_fork = 1; + else if(strcmp(argv[i], "--unittest") == 0) { + rrd_update_every = 1; + if(run_all_mockup_tests()) exit(1); + if(unit_test_storage()) exit(1); + fprintf(stderr, "\n\nALL TESTS PASSED\n\n"); + exit(0); + } + else { + fprintf(stderr, "Cannot understand option '%s'.\n", argv[i]); + fprintf(stderr, "\nUSAGE: %s [-d] [-l LINES_TO_SAVE] [-u UPDATE_TIMER] [-p LISTEN_PORT] [-dl debug log file] [-df debug flags].\n\n", argv[0]); + fprintf(stderr, " -c CONFIG FILE the configuration file to load. Default: %s.\n", CONFIG_DIR "/" CONFIG_FILENAME); + fprintf(stderr, " -l LINES_TO_SAVE can be from 5 to %d lines in JSON data. Default: %d.\n", RRD_HISTORY_ENTRIES_MAX, RRD_DEFAULT_HISTORY_ENTRIES); + fprintf(stderr, " -t UPDATE_TIMER can be from 1 to %d seconds. Default: %d.\n", UPDATE_EVERY_MAX, UPDATE_EVERY); + fprintf(stderr, " -p LISTEN_PORT can be from 1 to %d. Default: %d.\n", 65535, LISTEN_PORT); + fprintf(stderr, " -u USERNAME can be any system username to run as. Default: none.\n"); + fprintf(stderr, " -ch path to access host /proc and /sys when running in a container. Default: empty.\n"); + fprintf(stderr, " -nd or -nodeamon to disable forking in the background. Default: unset.\n"); + fprintf(stderr, " -df FLAGS debug options. Default: 0x%08llx.\n", debug_flags); + exit(1); + } + } + + if(!config_loaded) load_config(NULL, 0); + + // prepare configuration environment variables for the plugins + setenv("NETDATA_CONFIG_DIR" , config_get("global", "config directory" , CONFIG_DIR) , 1); + setenv("NETDATA_PLUGINS_DIR", config_get("global", "plugins directory" , PLUGINS_DIR), 1); + setenv("NETDATA_WEB_DIR" , config_get("global", "web files directory", WEB_DIR) , 1); + setenv("NETDATA_CACHE_DIR" , config_get("global", "cache directory" , CACHE_DIR) , 1); + setenv("NETDATA_LOG_DIR" , config_get("global", "log directory" , LOG_DIR) , 1); + setenv("NETDATA_HOST_PREFIX", config_get("global", "host access prefix" , "") , 1); + + // avoid extended to stat(/etc/localtime) + // http://stackoverflow.com/questions/4554271/how-to-avoid-excessive-stat-etc-localtime-calls-in-strftime-on-linux + setenv("TZ", ":/etc/localtime", 0); + + // cd to /tmp to avoid any plugins writing files at random places + if(chdir("/tmp")) error("netdata: ERROR: Cannot cd to /tmp"); + + char *input_log_file = NULL; + char *output_log_file = NULL; + char *error_log_file = NULL; + char *access_log_file = NULL; + char *user = NULL; + { + char buffer[1024]; + + // -------------------------------------------------------------------- + + sprintf(buffer, "0x%08llx", 0ULL); + char *flags = config_get("global", "debug flags", buffer); + setenv("NETDATA_DEBUG_FLAGS", flags, 1); + + debug_flags = strtoull(flags, NULL, 0); + debug(D_OPTIONS, "Debug flags set to '0x%8llx'.", debug_flags); + + if(debug_flags != 0) { + struct rlimit rl = { RLIM_INFINITY, RLIM_INFINITY }; + if(setrlimit(RLIMIT_CORE, &rl) != 0) + info("Cannot request unlimited core dumps for debugging... Proceeding anyway..."); + } + + // -------------------------------------------------------------------- + +#ifdef MADV_MERGEABLE + enable_ksm = config_get_boolean("global", "memory deduplication (ksm)", enable_ksm); +#else +#warning "Kernel memory deduplication (KSM) is not available" +#endif + + // -------------------------------------------------------------------- + + + global_host_prefix = config_get("global", "host access prefix", ""); + setenv("NETDATA_HOST_PREFIX", global_host_prefix, 1); + + // -------------------------------------------------------------------- + + output_log_file = config_get("global", "debug log", LOG_DIR "/debug.log"); + if(strcmp(output_log_file, "syslog") == 0) { + output_log_syslog = 1; + output_log_file = NULL; + } + else if(strcmp(output_log_file, "none") == 0) { + output_log_syslog = 0; + output_log_file = NULL; + } + else output_log_syslog = 0; + + // -------------------------------------------------------------------- + + error_log_file = config_get("global", "error log", LOG_DIR "/error.log"); + if(strcmp(error_log_file, "syslog") == 0) { + error_log_syslog = 1; + error_log_file = NULL; + } + else if(strcmp(error_log_file, "none") == 0) { + error_log_syslog = 0; + error_log_file = NULL; + // optimization - do not even generate debug log entries + } + else error_log_syslog = 0; + + // -------------------------------------------------------------------- + + access_log_file = config_get("global", "access log", LOG_DIR "/access.log"); + if(strcmp(access_log_file, "syslog") == 0) { + access_log_syslog = 1; + access_log_file = NULL; + } + else if(strcmp(access_log_file, "none") == 0) { + access_log_syslog = 0; + access_log_file = NULL; + } + else access_log_syslog = 0; + + // -------------------------------------------------------------------- + + rrd_memory_mode = rrd_memory_mode_id(config_get("global", "memory mode", rrd_memory_mode_name(rrd_memory_mode))); + + // -------------------------------------------------------------------- + + if(gethostname(buffer, HOSTNAME_MAX) == -1) + error("WARNING: Cannot get machine hostname."); + hostname = config_get("global", "hostname", buffer); + debug(D_OPTIONS, "hostname set to '%s'", hostname); + + // -------------------------------------------------------------------- + + rrd_default_history_entries = (int) config_get_number("global", "history", RRD_DEFAULT_HISTORY_ENTRIES); + if(rrd_default_history_entries < 5 || rrd_default_history_entries > RRD_HISTORY_ENTRIES_MAX) { + fprintf(stderr, "Invalid save lines %d given. Defaulting to %d.\n", rrd_default_history_entries, RRD_DEFAULT_HISTORY_ENTRIES); + rrd_default_history_entries = RRD_DEFAULT_HISTORY_ENTRIES; + } + else { + debug(D_OPTIONS, "save lines set to %d.", rrd_default_history_entries); + } + + // -------------------------------------------------------------------- + + rrd_update_every = (int) config_get_number("global", "update every", UPDATE_EVERY); + if(rrd_update_every < 1 || rrd_update_every > 600) { + fprintf(stderr, "Invalid update timer %d given. Defaulting to %d.\n", rrd_update_every, UPDATE_EVERY_MAX); + rrd_update_every = UPDATE_EVERY; + } + else debug(D_OPTIONS, "update timer set to %d.", rrd_update_every); + + // let the plugins know the min update_every + { + char buf[50]; + snprintf(buf, 50, "%d", rrd_update_every); + setenv("NETDATA_UPDATE_EVERY", buf, 1); + } + + // -------------------------------------------------------------------- + + for (i = 0; static_threads[i].name != NULL ; i++) { + struct netdata_static_thread *st = &static_threads[i]; + + if(st->config_name) st->enabled = config_get_boolean(st->config_section, st->config_name, st->enabled); + if(st->enabled && st->init_routine) st->init_routine(); + } + + // -------------------------------------------------------------------- + + prepare_rundir(); + user = config_get("global", "run as user" , (getuid() == 0)?NETDATA_USER:""); + web_files_uid(); + + // -------------------------------------------------------------------- + + listen_backlog = (int) config_get_number("global", "http port listen backlog", LISTEN_BACKLOG); + + listen_port = (int) config_get_number("global", "port", LISTEN_PORT); + if(listen_port < 1 || listen_port > 65535) { + fprintf(stderr, "Invalid listen port %d given. Defaulting to %d.\n", listen_port, LISTEN_PORT); + listen_port = LISTEN_PORT; + } + else debug(D_OPTIONS, "Listen port set to %d.", listen_port); + + int ip = 0; + char *ipv = config_get("global", "ip version", "any"); + if(!strcmp(ipv, "any") || !strcmp(ipv, "both") || !strcmp(ipv, "all")) ip = 0; + else if(!strcmp(ipv, "ipv4") || !strcmp(ipv, "IPV4") || !strcmp(ipv, "IPv4") || !strcmp(ipv, "4")) ip = 4; + else if(!strcmp(ipv, "ipv6") || !strcmp(ipv, "IPV6") || !strcmp(ipv, "IPv6") || !strcmp(ipv, "6")) ip = 6; + else fprintf(stderr, "Cannot understand ip version '%s'. Assumming 'any'.", ipv); + + if(ip == 0 || ip == 6) listen_fd = create_listen_socket6(listen_port, listen_backlog); + if(listen_fd < 0) { + listen_fd = create_listen_socket4(listen_port, listen_backlog); + if(listen_fd >= 0 && ip != 4) fprintf(stderr, "Managed to open an IPv4 socket on port %d.", listen_port); + } + + if(listen_fd < 0) fatal("Cannot listen socket."); + } + + // never become a problem + if(nice(20) == -1) error("Cannot lower my CPU priority."); + + if(become_daemon(dont_fork, 0, user, input_log_file, output_log_file, error_log_file, access_log_file, &access_fd, &stdaccess) == -1) { + fatal("Cannot demonize myself."); + exit(1); + } + + if(output_log_syslog || error_log_syslog || access_log_syslog) + openlog("netdata", LOG_PID, LOG_DAEMON); + + info("NetData started on pid %d", getpid()); + + + // catch all signals + for (i = 1 ; i < 65 ;i++) { + switch(i) { + case SIGKILL: // not catchable + case SIGSTOP: // not catchable + break; + + case SIGSEGV: + case SIGFPE: + case SIGCHLD: + signal(i, SIG_DFL); + break; + + default: + signal(i, sig_handler); + break; + } + } + + for (i = 0; static_threads[i].name != NULL ; i++) { + struct netdata_static_thread *st = &static_threads[i]; + + if(st->enabled) { + st->thread = malloc(sizeof(pthread_t)); + + info("Starting thread %s.", st->name); + + if(pthread_create(st->thread, NULL, st->start_routine, NULL)) + error("failed to create new thread for %s.", st->name); + + else if(pthread_detach(*st->thread)) + error("Cannot request detach of newly created %s thread.", st->name); + } + else info("Not starting thread %s.", st->name); + } + + // for future use - the main thread + while(1) { + if(netdata_exit != 0) { + netdata_exit++; + + if(netdata_exit > 5) { + netdata_cleanup_and_exit(0); + exit(0); + } + } + sleep(2); + } + + exit(0); +} diff --git a/src/main.h b/src/main.h new file mode 100755 index 000000000..6a90efd9d --- /dev/null +++ b/src/main.h @@ -0,0 +1,10 @@ +#ifndef NETDATA_MAIN_H +#define NETDATA_MAIN_H 1 + +extern int netdata_exit; + +extern void kill_childs(void); +extern int killpid(pid_t pid, int signal); +extern void netdata_cleanup_and_exit(int ret); + +#endif /* NETDATA_MAIN_H */ diff --git a/src/plugin_checks.c b/src/plugin_checks.c new file mode 100755 index 000000000..379fb9a84 --- /dev/null +++ b/src/plugin_checks.c @@ -0,0 +1,97 @@ +#ifdef HAVE_CONFIG_H +#include <config.h> +#endif +#include <pthread.h> +#include <sys/time.h> +#include <sys/resource.h> +#include <strings.h> +#include <unistd.h> + +#include "common.h" +#include "appconfig.h" +#include "log.h" +#include "rrd.h" +#include "plugin_checks.h" + +void *checks_main(void *ptr) +{ + if(ptr) { ; } + + info("CHECKS thread created with task id %d", gettid()); + + if(pthread_setcanceltype(PTHREAD_CANCEL_DEFERRED, NULL) != 0) + error("Cannot set pthread cancel type to DEFERRED."); + + if(pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, NULL) != 0) + error("Cannot set pthread cancel state to ENABLE."); + + unsigned long long usec = 0, susec = rrd_update_every * 1000000ULL, loop_usec = 0, total_susec = 0; + struct timeval now, last, loop; + + RRDSET *check1, *check2, *check3, *apps_cpu = NULL; + + check1 = rrdset_create("netdata", "check1", NULL, "netdata", NULL, "Caller gives microseconds", "a million !", 99999, rrd_update_every, RRDSET_TYPE_LINE); + rrddim_add(check1, "absolute", NULL, -1, 1, RRDDIM_ABSOLUTE); + rrddim_add(check1, "incremental", NULL, 1, 1, RRDDIM_INCREMENTAL); + + check2 = rrdset_create("netdata", "check2", NULL, "netdata", NULL, "Netdata calcs microseconds", "a million !", 99999, rrd_update_every, RRDSET_TYPE_LINE); + rrddim_add(check2, "absolute", NULL, -1, 1, RRDDIM_ABSOLUTE); + rrddim_add(check2, "incremental", NULL, 1, 1, RRDDIM_INCREMENTAL); + + check3 = rrdset_create("netdata", "checkdt", NULL, "netdata", NULL, "Clock difference", "microseconds diff", 99999, rrd_update_every, RRDSET_TYPE_LINE); + rrddim_add(check3, "caller", NULL, 1, 1, RRDDIM_ABSOLUTE); + rrddim_add(check3, "netdata", NULL, 1, 1, RRDDIM_ABSOLUTE); + rrddim_add(check3, "apps.plugin", NULL, 1, 1, RRDDIM_ABSOLUTE); + + gettimeofday(&last, NULL); + while(1) { + usleep(susec); + + // find the time to sleep in order to wait exactly update_every seconds + gettimeofday(&now, NULL); + loop_usec = usecdiff(&now, &last); + usec = loop_usec - susec; + debug(D_PROCNETDEV_LOOP, "CHECK: last loop took %llu usec (worked for %llu, sleeped for %llu).", loop_usec, usec, susec); + + if(usec < (rrd_update_every * 1000000ULL / 2ULL)) susec = (rrd_update_every * 1000000ULL) - usec; + else susec = rrd_update_every * 1000000ULL / 2ULL; + + // -------------------------------------------------------------------- + // Calculate loop time + + last.tv_sec = now.tv_sec; + last.tv_usec = now.tv_usec; + total_susec += loop_usec; + + // -------------------------------------------------------------------- + // check chart 1 + + if(check1->counter_done) rrdset_next_usec(check1, loop_usec); + rrddim_set(check1, "absolute", 1000000); + rrddim_set(check1, "incremental", total_susec); + rrdset_done(check1); + + // -------------------------------------------------------------------- + // check chart 2 + + if(check2->counter_done) rrdset_next(check2); + rrddim_set(check2, "absolute", 1000000); + rrddim_set(check2, "incremental", total_susec); + rrdset_done(check2); + + // -------------------------------------------------------------------- + // check chart 3 + + if(!apps_cpu) apps_cpu = rrdset_find("apps.cpu"); + if(check3->counter_done) rrdset_next_usec(check3, loop_usec); + gettimeofday(&loop, NULL); + rrddim_set(check3, "caller", (long long)usecdiff(&loop, &check1->last_collected_time)); + rrddim_set(check3, "netdata", (long long)usecdiff(&loop, &check2->last_collected_time)); + if(apps_cpu) rrddim_set(check3, "apps.plugin", (long long)usecdiff(&loop, &apps_cpu->last_collected_time)); + rrdset_done(check3); + } + + pthread_exit(NULL); + return NULL; +} + diff --git a/src/plugin_checks.h b/src/plugin_checks.h new file mode 100755 index 000000000..c27685b84 --- /dev/null +++ b/src/plugin_checks.h @@ -0,0 +1,6 @@ +#ifndef NETDATA_PLUGIN_CHECKS_H +#define NETDATA_PLUGIN_CHECKS_H 1 + +void *checks_main(void *ptr); + +#endif /* NETDATA_PLUGIN_PROC_H */ diff --git a/src/plugin_idlejitter.c b/src/plugin_idlejitter.c new file mode 100755 index 000000000..56c22a160 --- /dev/null +++ b/src/plugin_idlejitter.c @@ -0,0 +1,68 @@ +#ifdef HAVE_CONFIG_H +#include <config.h> +#endif +#include <pthread.h> +#include <sys/time.h> +#include <sys/resource.h> +#include <strings.h> +#include <unistd.h> + +#include "global_statistics.h" +#include "common.h" +#include "appconfig.h" +#include "log.h" +#include "rrd.h" +#include "plugin_idlejitter.h" + +#define CPU_IDLEJITTER_SLEEP_TIME_MS 20 + +void *cpuidlejitter_main(void *ptr) +{ + if(ptr) { ; } + + info("CPU Idle Jitter thread created with task id %d", gettid()); + + if(pthread_setcanceltype(PTHREAD_CANCEL_DEFERRED, NULL) != 0) + error("Cannot set pthread cancel type to DEFERRED."); + + if(pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, NULL) != 0) + error("Cannot set pthread cancel state to ENABLE."); + + int sleep_ms = (int) config_get_number("plugin:idlejitter", "loop time in ms", CPU_IDLEJITTER_SLEEP_TIME_MS); + if(sleep_ms <= 0) { + config_set_number("plugin:idlejitter", "loop time in ms", CPU_IDLEJITTER_SLEEP_TIME_MS); + sleep_ms = CPU_IDLEJITTER_SLEEP_TIME_MS; + } + + RRDSET *st = rrdset_find("system.idlejitter"); + if(!st) { + st = rrdset_create("system", "idlejitter", NULL, "processes", NULL, "CPU Idle Jitter", "microseconds lost/s", 9999, rrd_update_every, RRDSET_TYPE_LINE); + rrddim_add(st, "jitter", NULL, 1, 1, RRDDIM_ABSOLUTE); + } + + struct timeval before, after; + unsigned long long counter; + for(counter = 0; 1 ;counter++) { + unsigned long long usec = 0, susec = 0; + + while(susec < (rrd_update_every * 1000000ULL)) { + + gettimeofday(&before, NULL); + usleep(sleep_ms * 1000); + gettimeofday(&after, NULL); + + // calculate the time it took for a full loop + usec = usecdiff(&after, &before); + susec += usec; + } + usec -= (sleep_ms * 1000); + + if(counter) rrdset_next(st); + rrddim_set(st, "jitter", usec); + rrdset_done(st); + } + + pthread_exit(NULL); + return NULL; +} + diff --git a/src/plugin_idlejitter.h b/src/plugin_idlejitter.h new file mode 100755 index 000000000..dc82f052f --- /dev/null +++ b/src/plugin_idlejitter.h @@ -0,0 +1,6 @@ +#ifndef NETDATA_PLUGIN_IDLEJITTER_H +#define NETDATA_PLUGIN_IDLEJITTER_H 1 + +extern void *cpuidlejitter_main(void *ptr); + +#endif /* NETDATA_PLUGIN_IDLEJITTER_H */ diff --git a/src/plugin_nfacct.c b/src/plugin_nfacct.c new file mode 100644 index 000000000..0c5b39457 --- /dev/null +++ b/src/plugin_nfacct.c @@ -0,0 +1,221 @@ +#ifdef HAVE_CONFIG_H +#include <config.h> +#endif +#ifdef INTERNAL_PLUGIN_NFACCT +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <sys/types.h> +#include <dirent.h> +#include <sys/stat.h> +#include <fcntl.h> +#include <unistd.h> +#include <time.h> +#include <errno.h> + +#include <libmnl/libmnl.h> +#include <libnetfilter_acct/libnetfilter_acct.h> + +#include "global_statistics.h" +#include "common.h" +#include "appconfig.h" +#include "log.h" +#include "rrd.h" +#include "plugin_proc.h" + +struct mynfacct { + const char *name; + uint64_t pkts; + uint64_t bytes; + struct nfacct *nfacct; +}; + +struct nfacct_list { + int size; + int len; + struct mynfacct data[]; +} *nfacct_list = NULL; + +static int nfacct_callback(const struct nlmsghdr *nlh, void *data) { + if(data) {}; + + if(!nfacct_list || nfacct_list->len == nfacct_list->size) { + int size = (nfacct_list) ? nfacct_list->size : 0; + int len = (nfacct_list) ? nfacct_list->len : 0; + size++; + + info("nfacct.plugin: increasing nfacct_list to size %d", size); + + nfacct_list = realloc(nfacct_list, sizeof(struct nfacct_list) + (sizeof(struct mynfacct) * size)); + if(!nfacct_list) { + error("nfacct.plugin: cannot allocate nfacct_list."); + return MNL_CB_OK; + } + + nfacct_list->data[len].nfacct = nfacct_alloc(); + if(!nfacct_list->data[size - 1].nfacct) { + error("nfacct.plugin: nfacct_alloc() failed."); + free(nfacct_list); + nfacct_list = NULL; + return MNL_CB_OK; + } + + nfacct_list->size = size; + nfacct_list->len = len; + } + + if(nfacct_nlmsg_parse_payload(nlh, nfacct_list->data[nfacct_list->len].nfacct) < 0) { + error("nfacct.plugin: nfacct_nlmsg_parse_payload() failed."); + return MNL_CB_OK; + } + + nfacct_list->data[nfacct_list->len].name = nfacct_attr_get_str(nfacct_list->data[nfacct_list->len].nfacct, NFACCT_ATTR_NAME); + nfacct_list->data[nfacct_list->len].pkts = nfacct_attr_get_u64(nfacct_list->data[nfacct_list->len].nfacct, NFACCT_ATTR_PKTS); + nfacct_list->data[nfacct_list->len].bytes = nfacct_attr_get_u64(nfacct_list->data[nfacct_list->len].nfacct, NFACCT_ATTR_BYTES); + + nfacct_list->len++; + return MNL_CB_OK; +} + +void *nfacct_main(void *ptr) { + if(ptr) { ; } + + info("NFACCT thread created with task id %d", gettid()); + + if(pthread_setcanceltype(PTHREAD_CANCEL_DEFERRED, NULL) != 0) + error("nfacct.plugin: Cannot set pthread cancel type to DEFERRED."); + + if(pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, NULL) != 0) + error("nfacct.plugin: Cannot set pthread cancel state to ENABLE."); + + char buf[MNL_SOCKET_BUFFER_SIZE]; + struct mnl_socket *nl = NULL; + struct nlmsghdr *nlh = NULL; + unsigned int seq = 0, portid = 0; + + seq = time(NULL) - 1; + + nl = mnl_socket_open(NETLINK_NETFILTER); + if(!nl) { + error("nfacct.plugin: mnl_socket_open() failed"); + pthread_exit(NULL); + return NULL; + } + + if(mnl_socket_bind(nl, 0, MNL_SOCKET_AUTOPID) < 0) { + mnl_socket_close(nl); + error("nfacct.plugin: mnl_socket_bind() failed"); + pthread_exit(NULL); + return NULL; + } + portid = mnl_socket_get_portid(nl); + + // ------------------------------------------------------------------------ + + struct timeval last, now; + unsigned long long usec = 0, susec = 0; + RRDSET *st = NULL; + + gettimeofday(&last, NULL); + + // ------------------------------------------------------------------------ + + while(1) { + if(unlikely(netdata_exit)) break; + + seq++; + + nlh = nfacct_nlmsg_build_hdr(buf, NFNL_MSG_ACCT_GET, NLM_F_DUMP, seq); + if(!nlh) { + mnl_socket_close(nl); + error("nfacct.plugin: nfacct_nlmsg_build_hdr() failed"); + pthread_exit(NULL); + return NULL; + } + + if(mnl_socket_sendto(nl, nlh, nlh->nlmsg_len) < 0) { + error("nfacct.plugin: mnl_socket_send"); + pthread_exit(NULL); + return NULL; + } + + if(nfacct_list) nfacct_list->len = 0; + + int ret; + while((ret = mnl_socket_recvfrom(nl, buf, sizeof(buf))) > 0) { + if((ret = mnl_cb_run(buf, ret, seq, portid, nfacct_callback, NULL)) <= 0) break; + } + + if (ret == -1) { + error("nfacct.plugin: error communicating with kernel."); + pthread_exit(NULL); + return NULL; + } + + // -------------------------------------------------------------------- + + gettimeofday(&now, NULL); + usec = usecdiff(&now, &last) - susec; + debug(D_NFACCT_LOOP, "nfacct.plugin: last loop took %llu usec (worked for %llu, sleeped for %llu).", usec + susec, usec, susec); + + if(usec < (rrd_update_every * 1000000ULL / 2ULL)) susec = (rrd_update_every * 1000000ULL) - usec; + else susec = rrd_update_every * 1000000ULL / 2ULL; + + + // -------------------------------------------------------------------- + + if(nfacct_list && nfacct_list->len) { + int i; + + st = rrdset_find_bytype("nfacct", "packets"); + if(!st) { + st = rrdset_create("nfacct", "packets", NULL, "netfilter", "Netfilter Accounting Packets", "packets/s", 1006, rrd_update_every, RRDSET_TYPE_STACKED); + + for(i = 0; i < nfacct_list->len ; i++) + rrddim_add(st, nfacct_list->data[i].name, NULL, 1, rrd_update_every, RRDDIM_INCREMENTAL); + } + else rrdset_next(st); + + for(i = 0; i < nfacct_list->len ; i++) { + RRDDIM *rd = rrddim_find(st, nfacct_list->data[i].name); + + if(!rd) rd = rrddim_add(st, nfacct_list->data[i].name, NULL, 1, rrd_update_every, RRDDIM_INCREMENTAL); + if(rd) rrddim_set_by_pointer(st, rd, nfacct_list->data[i].pkts); + } + + rrdset_done(st); + + // ---------------------------------------------------------------- + + st = rrdset_find_bytype("nfacct", "bytes"); + if(!st) { + st = rrdset_create("nfacct", "bytes", NULL, "netfilter", "Netfilter Accounting Bandwidth", "kilobytes/s", 1007, rrd_update_every, RRDSET_TYPE_STACKED); + + for(i = 0; i < nfacct_list->len ; i++) + rrddim_add(st, nfacct_list->data[i].name, NULL, 1, 1000 * rrd_update_every, RRDDIM_INCREMENTAL); + } + else rrdset_next(st); + + for(i = 0; i < nfacct_list->len ; i++) { + RRDDIM *rd = rrddim_find(st, nfacct_list->data[i].name); + + if(!rd) rd = rrddim_add(st, nfacct_list->data[i].name, NULL, 1, 1000 * rrd_update_every, RRDDIM_INCREMENTAL); + if(rd) rrddim_set_by_pointer(st, rd, nfacct_list->data[i].bytes); + } + + rrdset_done(st); + } + + // -------------------------------------------------------------------- + + usleep(susec); + + // copy current to last + bcopy(&now, &last, sizeof(struct timeval)); + } + + mnl_socket_close(nl); + pthread_exit(NULL); + return NULL; +} +#endif diff --git a/src/plugin_nfacct.h b/src/plugin_nfacct.h new file mode 100755 index 000000000..88a3a9230 --- /dev/null +++ b/src/plugin_nfacct.h @@ -0,0 +1,7 @@ +#ifndef NETDATA_NFACCT_H +#define NETDATA_NFACCT_H 1 + +extern void *nfacct_main(void *ptr); + +#endif /* NETDATA_NFACCT_H */ + diff --git a/src/plugin_proc.c b/src/plugin_proc.c new file mode 100755 index 000000000..af679de1e --- /dev/null +++ b/src/plugin_proc.c @@ -0,0 +1,300 @@ +#ifdef HAVE_CONFIG_H +#include <config.h> +#endif +#include <pthread.h> +#include <sys/time.h> +#include <sys/resource.h> +#include <strings.h> +#include <unistd.h> + +#include "global_statistics.h" +#include "common.h" +#include "appconfig.h" +#include "log.h" +#include "rrd.h" +#include "plugin_proc.h" +#include "main.h" + +unsigned long long sutime() { + struct timeval now; + gettimeofday(&now, NULL); + return now.tv_sec * 1000000ULL + now.tv_usec; +} + +void *proc_main(void *ptr) +{ + if(ptr) { ; } + + info("PROC Plugin thread created with task id %d", gettid()); + + if(pthread_setcanceltype(PTHREAD_CANCEL_DEFERRED, NULL) != 0) + error("Cannot set pthread cancel type to DEFERRED."); + + if(pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, NULL) != 0) + error("Cannot set pthread cancel state to ENABLE."); + + struct rusage me, thread; + + // disable (by default) various interface that are not needed + config_get_boolean("plugin:proc:/proc/net/dev:lo", "enabled", 0); + config_get_boolean("plugin:proc:/proc/net/dev:fireqos_monitor", "enabled", 0); + + // when ZERO, attempt to do it + int vdo_proc_net_dev = !config_get_boolean("plugin:proc", "/proc/net/dev", 1); + int vdo_proc_diskstats = !config_get_boolean("plugin:proc", "/proc/diskstats", 1); + int vdo_proc_net_snmp = !config_get_boolean("plugin:proc", "/proc/net/snmp", 1); + int vdo_proc_net_netstat = !config_get_boolean("plugin:proc", "/proc/net/netstat", 1); + int vdo_proc_net_stat_conntrack = !config_get_boolean("plugin:proc", "/proc/net/stat/conntrack", 1); + int vdo_proc_net_ip_vs_stats = !config_get_boolean("plugin:proc", "/proc/net/ip_vs/stats", 1); + int vdo_proc_stat = !config_get_boolean("plugin:proc", "/proc/stat", 1); + int vdo_proc_meminfo = !config_get_boolean("plugin:proc", "/proc/meminfo", 1); + int vdo_proc_vmstat = !config_get_boolean("plugin:proc", "/proc/vmstat", 1); + int vdo_proc_net_rpc_nfsd = !config_get_boolean("plugin:proc", "/proc/net/rpc/nfsd", 1); + int vdo_proc_sys_kernel_random_entropy_avail = !config_get_boolean("plugin:proc", "/proc/sys/kernel/random/entropy_avail", 1); + int vdo_proc_interrupts = !config_get_boolean("plugin:proc", "/proc/interrupts", 1); + int vdo_proc_softirqs = !config_get_boolean("plugin:proc", "/proc/softirqs", 1); + int vdo_proc_loadavg = !config_get_boolean("plugin:proc", "/proc/loadavg", 1); + int vdo_sys_kernel_mm_ksm = !config_get_boolean("plugin:proc", "/sys/kernel/mm/ksm", 1); + int vdo_cpu_netdata = !config_get_boolean("plugin:proc", "netdata server resources", 1); + + // keep track of the time each module was called + unsigned long long sutime_proc_net_dev = 0ULL; + unsigned long long sutime_proc_diskstats = 0ULL; + unsigned long long sutime_proc_net_snmp = 0ULL; + unsigned long long sutime_proc_net_netstat = 0ULL; + unsigned long long sutime_proc_net_stat_conntrack = 0ULL; + unsigned long long sutime_proc_net_ip_vs_stats = 0ULL; + unsigned long long sutime_proc_stat = 0ULL; + unsigned long long sutime_proc_meminfo = 0ULL; + unsigned long long sutime_proc_vmstat = 0ULL; + unsigned long long sutime_proc_net_rpc_nfsd = 0ULL; + unsigned long long sutime_proc_sys_kernel_random_entropy_avail = 0ULL; + unsigned long long sutime_proc_interrupts = 0ULL; + unsigned long long sutime_proc_softirqs = 0ULL; + unsigned long long sutime_proc_loadavg = 0ULL; + unsigned long long sutime_sys_kernel_mm_ksm = 0ULL; + + // the next time we will run - aligned properly + unsigned long long sunext = (time(NULL) - (time(NULL) % rrd_update_every) + rrd_update_every) * 1000000ULL; + unsigned long long sunow; + + RRDSET *stcpu = NULL, *stcpu_thread = NULL, *stclients = NULL, *streqs = NULL, *stbytes = NULL; + + for(;1;) { + if(unlikely(netdata_exit)) break; + + // delay until it is our time to run + while((sunow = sutime()) < sunext) + usleep((useconds_t)(sunext - sunow)); + + // find the next time we need to run + while(sutime() > sunext) + sunext += rrd_update_every * 1000000ULL; + + if(unlikely(netdata_exit)) break; + + // BEGIN -- the job to be done + + if(!vdo_sys_kernel_mm_ksm) { + debug(D_PROCNETDEV_LOOP, "PROCNETDEV: calling do_sys_kernel_mm_ksm()."); + + sunow = sutime(); + vdo_sys_kernel_mm_ksm = do_sys_kernel_mm_ksm(rrd_update_every, (sutime_sys_kernel_mm_ksm > 0)?sunow - sutime_sys_kernel_mm_ksm:0ULL); + sutime_sys_kernel_mm_ksm = sunow; + } + if(unlikely(netdata_exit)) break; + + if(!vdo_proc_loadavg) { + debug(D_PROCNETDEV_LOOP, "PROCNETDEV: calling do_proc_loadavg()."); + sunow = sutime(); + vdo_proc_loadavg = do_proc_loadavg(rrd_update_every, (sutime_proc_loadavg > 0)?sunow - sutime_proc_loadavg:0ULL); + sutime_proc_loadavg = sunow; + } + if(unlikely(netdata_exit)) break; + + if(!vdo_proc_interrupts) { + debug(D_PROCNETDEV_LOOP, "PROCNETDEV: calling do_proc_interrupts()."); + sunow = sutime(); + vdo_proc_interrupts = do_proc_interrupts(rrd_update_every, (sutime_proc_interrupts > 0)?sunow - sutime_proc_interrupts:0ULL); + sutime_proc_interrupts = sunow; + } + if(unlikely(netdata_exit)) break; + + if(!vdo_proc_softirqs) { + debug(D_PROCNETDEV_LOOP, "PROCNETDEV: calling do_proc_softirqs()."); + sunow = sutime(); + vdo_proc_softirqs = do_proc_softirqs(rrd_update_every, (sutime_proc_softirqs > 0)?sunow - sutime_proc_softirqs:0ULL); + sutime_proc_softirqs = sunow; + } + if(unlikely(netdata_exit)) break; + + if(!vdo_proc_sys_kernel_random_entropy_avail) { + debug(D_PROCNETDEV_LOOP, "PROCNETDEV: calling do_proc_sys_kernel_random_entropy_avail()."); + sunow = sutime(); + vdo_proc_sys_kernel_random_entropy_avail = do_proc_sys_kernel_random_entropy_avail(rrd_update_every, (sutime_proc_sys_kernel_random_entropy_avail > 0)?sunow - sutime_proc_sys_kernel_random_entropy_avail:0ULL); + sutime_proc_sys_kernel_random_entropy_avail = sunow; + } + if(unlikely(netdata_exit)) break; + + if(!vdo_proc_net_dev) { + debug(D_PROCNETDEV_LOOP, "PROCNETDEV: calling do_proc_net_dev()."); + sunow = sutime(); + vdo_proc_net_dev = do_proc_net_dev(rrd_update_every, (sutime_proc_net_dev > 0)?sunow - sutime_proc_net_dev:0ULL); + sutime_proc_net_dev = sunow; + } + if(unlikely(netdata_exit)) break; + + if(!vdo_proc_diskstats) { + debug(D_PROCNETDEV_LOOP, "PROCNETDEV: calling do_proc_diskstats()."); + sunow = sutime(); + vdo_proc_diskstats = do_proc_diskstats(rrd_update_every, (sutime_proc_diskstats > 0)?sunow - sutime_proc_diskstats:0ULL); + sutime_proc_diskstats = sunow; + } + if(unlikely(netdata_exit)) break; + + if(!vdo_proc_net_snmp) { + debug(D_PROCNETDEV_LOOP, "PROCNETDEV: calling do_proc_net_snmp()."); + sunow = sutime(); + vdo_proc_net_snmp = do_proc_net_snmp(rrd_update_every, (sutime_proc_net_snmp > 0)?sunow - sutime_proc_net_snmp:0ULL); + sutime_proc_net_snmp = sunow; + } + if(unlikely(netdata_exit)) break; + + if(!vdo_proc_net_netstat) { + debug(D_PROCNETDEV_LOOP, "PROCNETDEV: calling do_proc_net_netstat()."); + sunow = sutime(); + vdo_proc_net_netstat = do_proc_net_netstat(rrd_update_every, (sutime_proc_net_netstat > 0)?sunow - sutime_proc_net_netstat:0ULL); + sutime_proc_net_netstat = sunow; + } + if(unlikely(netdata_exit)) break; + + if(!vdo_proc_net_stat_conntrack) { + debug(D_PROCNETDEV_LOOP, "PROCNETDEV: calling do_proc_net_stat_conntrack()."); + sunow = sutime(); + vdo_proc_net_stat_conntrack = do_proc_net_stat_conntrack(rrd_update_every, (sutime_proc_net_stat_conntrack > 0)?sunow - sutime_proc_net_stat_conntrack:0ULL); + sutime_proc_net_stat_conntrack = sunow; + } + if(unlikely(netdata_exit)) break; + + if(!vdo_proc_net_ip_vs_stats) { + debug(D_PROCNETDEV_LOOP, "PROCNETDEV: calling vdo_proc_net_ip_vs_stats()."); + sunow = sutime(); + vdo_proc_net_ip_vs_stats = do_proc_net_ip_vs_stats(rrd_update_every, (sutime_proc_net_ip_vs_stats > 0)?sunow - sutime_proc_net_ip_vs_stats:0ULL); + sutime_proc_net_ip_vs_stats = sunow; + } + if(unlikely(netdata_exit)) break; + + if(!vdo_proc_stat) { + debug(D_PROCNETDEV_LOOP, "PROCNETDEV: calling do_proc_stat()."); + sunow = sutime(); + vdo_proc_stat = do_proc_stat(rrd_update_every, (sutime_proc_stat > 0)?sunow - sutime_proc_stat:0ULL); + sutime_proc_stat = sunow; + } + if(unlikely(netdata_exit)) break; + + if(!vdo_proc_meminfo) { + debug(D_PROCNETDEV_LOOP, "PROCNETDEV: calling vdo_proc_meminfo()."); + sunow = sutime(); + vdo_proc_meminfo = do_proc_meminfo(rrd_update_every, (sutime_proc_meminfo > 0)?sunow - sutime_proc_meminfo:0ULL); + sutime_proc_meminfo = sunow; + } + if(unlikely(netdata_exit)) break; + + if(!vdo_proc_vmstat) { + debug(D_PROCNETDEV_LOOP, "PROCNETDEV: calling vdo_proc_vmstat()."); + sunow = sutime(); + vdo_proc_vmstat = do_proc_vmstat(rrd_update_every, (sutime_proc_vmstat > 0)?sunow - sutime_proc_vmstat:0ULL); + sutime_proc_vmstat = sunow; + } + if(unlikely(netdata_exit)) break; + + if(!vdo_proc_net_rpc_nfsd) { + debug(D_PROCNETDEV_LOOP, "PROCNETDEV: calling do_proc_net_rpc_nfsd()."); + sunow = sutime(); + vdo_proc_net_rpc_nfsd = do_proc_net_rpc_nfsd(rrd_update_every, (sutime_proc_net_rpc_nfsd > 0)?sunow - sutime_proc_net_rpc_nfsd:0ULL); + sutime_proc_net_rpc_nfsd = sunow; + } + if(unlikely(netdata_exit)) break; + + // END -- the job is done + + // -------------------------------------------------------------------- + + if(!vdo_cpu_netdata) { + getrusage(RUSAGE_THREAD, &thread); + getrusage(RUSAGE_SELF, &me); + + if(!stcpu_thread) stcpu_thread = rrdset_find("netdata.plugin_proc_cpu"); + if(!stcpu_thread) { + stcpu_thread = rrdset_create("netdata", "plugin_proc_cpu", NULL, "proc.internal", NULL, "NetData Proc Plugin CPU usage", "milliseconds/s", 131000, rrd_update_every, RRDSET_TYPE_STACKED); + + rrddim_add(stcpu_thread, "user", NULL, 1, 1000, RRDDIM_INCREMENTAL); + rrddim_add(stcpu_thread, "system", NULL, 1, 1000, RRDDIM_INCREMENTAL); + } + else rrdset_next(stcpu_thread); + + rrddim_set(stcpu_thread, "user" , thread.ru_utime.tv_sec * 1000000ULL + thread.ru_utime.tv_usec); + rrddim_set(stcpu_thread, "system", thread.ru_stime.tv_sec * 1000000ULL + thread.ru_stime.tv_usec); + rrdset_done(stcpu_thread); + + // ---------------------------------------------------------------- + + if(!stcpu) stcpu = rrdset_find("netdata.server_cpu"); + if(!stcpu) { + stcpu = rrdset_create("netdata", "server_cpu", NULL, "netdata", NULL, "NetData CPU usage", "milliseconds/s", 130000, rrd_update_every, RRDSET_TYPE_STACKED); + + rrddim_add(stcpu, "user", NULL, 1, 1000, RRDDIM_INCREMENTAL); + rrddim_add(stcpu, "system", NULL, 1, 1000, RRDDIM_INCREMENTAL); + } + else rrdset_next(stcpu); + + rrddim_set(stcpu, "user" , me.ru_utime.tv_sec * 1000000ULL + me.ru_utime.tv_usec); + rrddim_set(stcpu, "system", me.ru_stime.tv_sec * 1000000ULL + me.ru_stime.tv_usec); + rrdset_done(stcpu); + + // ---------------------------------------------------------------- + + if(!stclients) stclients = rrdset_find("netdata.clients"); + if(!stclients) { + stclients = rrdset_create("netdata", "clients", NULL, "netdata", NULL, "NetData Web Clients", "connected clients", 131000, rrd_update_every, RRDSET_TYPE_LINE); + + rrddim_add(stclients, "clients", NULL, 1, 1, RRDDIM_ABSOLUTE); + } + else rrdset_next(stclients); + + rrddim_set(stclients, "clients", global_statistics.connected_clients); + rrdset_done(stclients); + + // ---------------------------------------------------------------- + + if(!streqs) streqs = rrdset_find("netdata.requests"); + if(!streqs) { + streqs = rrdset_create("netdata", "requests", NULL, "netdata", NULL, "NetData Web Requests", "requests/s", 131100, rrd_update_every, RRDSET_TYPE_LINE); + + rrddim_add(streqs, "requests", NULL, 1, 1, RRDDIM_INCREMENTAL); + } + else rrdset_next(streqs); + + rrddim_set(streqs, "requests", global_statistics.web_requests); + rrdset_done(streqs); + + // ---------------------------------------------------------------- + + if(!stbytes) stbytes = rrdset_find("netdata.net"); + if(!stbytes) { + stbytes = rrdset_create("netdata", "net", NULL, "netdata", NULL, "NetData Network Traffic", "kilobits/s", 131200, rrd_update_every, RRDSET_TYPE_AREA); + + rrddim_add(stbytes, "in", NULL, 8, 1024, RRDDIM_INCREMENTAL); + rrddim_add(stbytes, "out", NULL, -8, 1024, RRDDIM_INCREMENTAL); + } + else rrdset_next(stbytes); + + rrddim_set(stbytes, "in", global_statistics.bytes_received); + rrddim_set(stbytes, "out", global_statistics.bytes_sent); + rrdset_done(stbytes); + } + } + + pthread_exit(NULL); + return NULL; +} diff --git a/src/plugin_proc.h b/src/plugin_proc.h new file mode 100755 index 000000000..28b8c59ef --- /dev/null +++ b/src/plugin_proc.h @@ -0,0 +1,22 @@ +#ifndef NETDATA_PLUGIN_PROC_H +#define NETDATA_PLUGIN_PROC_H 1 + +void *proc_main(void *ptr); + +extern int do_proc_net_dev(int update_every, unsigned long long dt); +extern int do_proc_diskstats(int update_every, unsigned long long dt); +extern int do_proc_net_snmp(int update_every, unsigned long long dt); +extern int do_proc_net_netstat(int update_every, unsigned long long dt); +extern int do_proc_net_stat_conntrack(int update_every, unsigned long long dt); +extern int do_proc_net_ip_vs_stats(int update_every, unsigned long long dt); +extern int do_proc_stat(int update_every, unsigned long long dt); +extern int do_proc_meminfo(int update_every, unsigned long long dt); +extern int do_proc_vmstat(int update_every, unsigned long long dt); +extern int do_proc_net_rpc_nfsd(int update_every, unsigned long long dt); +extern int do_proc_sys_kernel_random_entropy_avail(int update_every, unsigned long long dt); +extern int do_proc_interrupts(int update_every, unsigned long long dt); +extern int do_proc_softirqs(int update_every, unsigned long long dt); +extern int do_sys_kernel_mm_ksm(int update_every, unsigned long long dt); +extern int do_proc_loadavg(int update_every, unsigned long long dt); + +#endif /* NETDATA_PLUGIN_PROC_H */ diff --git a/src/plugin_tc.c b/src/plugin_tc.c new file mode 100755 index 000000000..f02b70788 --- /dev/null +++ b/src/plugin_tc.c @@ -0,0 +1,737 @@ +#ifdef HAVE_CONFIG_H +#include <config.h> +#endif +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <sys/types.h> +#include <sys/resource.h> + +#include "avl.h" +#include "log.h" +#include "common.h" +#include "appconfig.h" +#include "rrd.h" +#include "popen.h" +#include "plugin_tc.h" +#include "main.h" + +#define RRD_TYPE_TC "tc" +#define RRD_TYPE_TC_LEN strlen(RRD_TYPE_TC) + +// ---------------------------------------------------------------------------- +// /sbin/tc processor +// this requires the script plugins.d/tc-qos-helper.sh + +#define TC_LINE_MAX 1024 + +struct tc_class { + avl avl; + + char *id; + uint32_t hash; + + char *name; + + char *leafid; + uint32_t leaf_hash; + + char *parentid; + uint32_t parent_hash; + + char hasparent; + char isleaf; + unsigned long long bytes; + unsigned long long packets; + unsigned long long dropped; + unsigned long long overlimits; + unsigned long long requeues; + unsigned long long lended; + unsigned long long borrowed; + unsigned long long giants; + unsigned long long tokens; + unsigned long long ctokens; + + char updated; // updated bytes + char seen; // seen in the tc list (even without bytes) + + struct tc_class *next; + struct tc_class *prev; +}; + +struct tc_device { + avl avl; + + char *id; + uint32_t hash; + + char *name; + char *family; + + avl_tree classes_index; + + struct tc_class *classes; + + struct tc_device *next; + struct tc_device *prev; +}; + + +struct tc_device *tc_device_root = NULL; + +// ---------------------------------------------------------------------------- +// tc_device index + +static int tc_device_iterator(avl *a) { if(a) {}; return 0; } + +static int tc_device_compare(void* a, void* b) { + if(((struct tc_device *)a)->hash < ((struct tc_device *)b)->hash) return -1; + else if(((struct tc_device *)a)->hash > ((struct tc_device *)b)->hash) return 1; + else return strcmp(((struct tc_device *)a)->id, ((struct tc_device *)b)->id); +} + +avl_tree tc_device_root_index = { + NULL, + tc_device_compare, +#ifdef AVL_LOCK_WITH_MUTEX + PTHREAD_MUTEX_INITIALIZER +#else + PTHREAD_RWLOCK_INITIALIZER +#endif +}; + +#define tc_device_index_add(st) avl_insert(&tc_device_root_index, (avl *)(st)) +#define tc_device_index_del(st) avl_remove(&tc_device_root_index, (avl *)(st)) + +static struct tc_device *tc_device_index_find(const char *id, uint32_t hash) { + struct tc_device *result = NULL, tmp; + tmp.id = (char *)id; + tmp.hash = (hash)?hash:simple_hash(tmp.id); + + avl_search(&(tc_device_root_index), (avl *)&tmp, tc_device_iterator, (avl **)&result); + return result; +} + + +// ---------------------------------------------------------------------------- +// tc_class index + +static int tc_class_iterator(avl *a) { if(a) {}; return 0; } + +static int tc_class_compare(void* a, void* b) { + if(((struct tc_class *)a)->hash < ((struct tc_class *)b)->hash) return -1; + else if(((struct tc_class *)a)->hash > ((struct tc_class *)b)->hash) return 1; + else return strcmp(((struct tc_class *)a)->id, ((struct tc_class *)b)->id); +} + +#define tc_class_index_add(st, rd) avl_insert(&((st)->classes_index), (avl *)(rd)) +#define tc_class_index_del(st, rd) avl_remove(&((st)->classes_index), (avl *)(rd)) + +static struct tc_class *tc_class_index_find(struct tc_device *st, const char *id, uint32_t hash) { + struct tc_class *result = NULL, tmp; + tmp.id = (char *)id; + tmp.hash = (hash)?hash:simple_hash(tmp.id); + + avl_search(&(st->classes_index), (avl *)&tmp, tc_class_iterator, (avl **)&result); + return result; +} + +// ---------------------------------------------------------------------------- + +static void tc_class_free(struct tc_device *n, struct tc_class *c) { + debug(D_TC_LOOP, "Removing from device '%s' class '%s', parentid '%s', leafid '%s', seen=%d", n->id, c->id, c->parentid?c->parentid:"", c->leafid?c->leafid:"", c->seen); + + if(c->next) c->next->prev = c->prev; + if(c->prev) c->prev->next = c->next; + if(n->classes == c) { + if(c->next) n->classes = c->next; + else n->classes = c->prev; + } + + tc_class_index_del(n, c); + + if(c->id) free(c->id); + if(c->name) free(c->name); + if(c->leafid) free(c->leafid); + if(c->parentid) free(c->parentid); + + free(c); +} + +static void tc_device_classes_cleanup(struct tc_device *d) { + static int cleanup_every = 999; + + if(cleanup_every > 0) { + cleanup_every = (int) -config_get_number("plugin:tc", "cleanup unused classes every", 60); + if(cleanup_every > 0) cleanup_every = -cleanup_every; + if(cleanup_every == 0) cleanup_every = -1; + } + + struct tc_class *c = d->classes; + while(c) { + if(c->seen < cleanup_every) { + struct tc_class *nc = c->next; + tc_class_free(d, c); + c = nc; + } + else c = c->next; + + if(c) { + c->updated = 0; + c->seen--; + } + } +} + +static void tc_device_commit(struct tc_device *d) +{ + static int enable_new_interfaces = -1; + + if(enable_new_interfaces == -1) enable_new_interfaces = config_get_boolean("plugin:tc", "enable new interfaces detected at runtime", 1); + + // we only need to add leaf classes + struct tc_class *c, *x; + + // set all classes + for(c = d->classes ; c ; c = c->next) { + c->isleaf = 1; + c->hasparent = 0; + } + + // mark the classes as leafs and parents + for(c = d->classes ; c ; c = c->next) { + if(!c->updated) continue; + + for(x = d->classes ; x ; x = x->next) { + if(!x->updated) continue; + + if(c == x) continue; + + if(x->parentid && ( + ( c->hash == x->parent_hash && strcmp(c->id, x->parentid) == 0) || + (c->leafid && c->leaf_hash == x->parent_hash && strcmp(c->leafid, x->parentid) == 0))) { + // debug(D_TC_LOOP, "TC: In device '%s', class '%s' (leafid: '%s') has as leaf class '%s' (parentid: '%s').", d->name?d->name:d->id, c->name?c->name:c->id, c->leafid?c->leafid:c->id, x->name?x->name:x->id, x->parentid?x->parentid:x->id); + c->isleaf = 0; + x->hasparent = 1; + } + } + } + + // debugging: + /* + for ( c = d->classes ; c ; c = c->next) { + if(c->isleaf && c->hasparent) debug(D_TC_LOOP, "TC: Device %s, class %s, OK", d->name, c->id); + else debug(D_TC_LOOP, "TC: Device %s, class %s, IGNORE (isleaf: %d, hasparent: %d, parent: %s)", d->name, c->id, c->isleaf, c->hasparent, c->parentid); + } + */ + + // we need at least a class + for(c = d->classes ; c ; c = c->next) { + // debug(D_TC_LOOP, "TC: Device '%s', class '%s', isLeaf=%d, HasParent=%d, Seen=%d", d->name?d->name:d->id, c->name?c->name:c->id, c->isleaf, c->hasparent, c->seen); + if(!c->updated) continue; + if(c->isleaf && c->hasparent) break; + } + if(!c) { + debug(D_TC_LOOP, "TC: Ignoring TC device '%s'. No leaf classes.", d->name?d->name:d->id); + tc_device_classes_cleanup(d); + return; + } + + char var_name[CONFIG_MAX_NAME + 1]; + snprintf(var_name, CONFIG_MAX_NAME, "qos for %s", d->id); + if(config_get_boolean("plugin:tc", var_name, enable_new_interfaces)) { + RRDSET *st = rrdset_find_bytype(RRD_TYPE_TC, d->id); + if(!st) { + debug(D_TC_LOOP, "TC: Creating new chart for device '%s'", d->name?d->name:d->id); + + st = rrdset_create(RRD_TYPE_TC, d->id, d->name?d->name:d->id, d->family?d->family:d->id, RRD_TYPE_TC ".qos", "Class Usage", "kilobits/s", 7000, rrd_update_every, RRDSET_TYPE_STACKED); + + for(c = d->classes ; c ; c = c->next) { + if(!c->updated) continue; + + if(c->isleaf && c->hasparent) + rrddim_add(st, c->id, c->name?c->name:c->id, 8, 1024, RRDDIM_INCREMENTAL); + } + } + else { + debug(D_TC_LOOP, "TC: Updating chart for device '%s'", d->name?d->name:d->id); + rrdset_next_plugins(st); + + if(d->name && strcmp(d->id, d->name) != 0) rrdset_set_name(st, d->name); + } + + for(c = d->classes ; c ; c = c->next) { + if(!c->updated) continue; + + if(c->isleaf && c->hasparent) { + RRDDIM *rd = rrddim_find(st, c->id); + + if(!rd) { + debug(D_TC_LOOP, "TC: Adding to chart '%s', dimension '%s'", st->id, c->id, c->name); + + // new class, we have to add it + rd = rrddim_add(st, c->id, c->name?c->name:c->id, 8, 1024, RRDDIM_INCREMENTAL); + } + else debug(D_TC_LOOP, "TC: Updating chart '%s', dimension '%s'", st->id, c->id); + + rrddim_set_by_pointer(st, rd, c->bytes); + + // if it has a name, different to the id + if(c->name) { + // update the rrd dimension with the new name + debug(D_TC_LOOP, "TC: Setting chart '%s', dimension '%s' name to '%s'", st->id, rd->id, c->name); + rrddim_set_name(st, rd, c->name); + + free(c->name); + c->name = NULL; + } + } + } + rrdset_done(st); + } + + tc_device_classes_cleanup(d); +} + +static void tc_device_set_class_name(struct tc_device *d, char *id, char *name) +{ + struct tc_class *c = tc_class_index_find(d, id, 0); + if(c) { + if(c->name) free(c->name); + c->name = NULL; + + if(name && *name && strcmp(c->id, name) != 0) { + debug(D_TC_LOOP, "TC: Setting device '%s', class '%s' name to '%s'", d->id, id, name); + c->name = strdup(name); + } + } +} + +static void tc_device_set_device_name(struct tc_device *d, char *name) { + if(d->name) free(d->name); + d->name = NULL; + + if(name && *name && strcmp(d->id, name) != 0) { + debug(D_TC_LOOP, "TC: Setting device '%s' name to '%s'", d->id, name); + d->name = strdup(name); + } +} + +static void tc_device_set_device_family(struct tc_device *d, char *family) { + if(d->family) free(d->family); + d->family = NULL; + + if(family && *family && strcmp(d->id, family) != 0) { + debug(D_TC_LOOP, "TC: Setting device '%s' family to '%s'", d->id, family); + d->family = strdup(family); + } + // no need for null termination - it is already null +} + +static struct tc_device *tc_device_create(char *id) +{ + struct tc_device *d = tc_device_index_find(id, 0); + + if(!d) { + debug(D_TC_LOOP, "TC: Creating device '%s'", id); + + d = calloc(1, sizeof(struct tc_device)); + if(!d) { + fatal("Cannot allocate memory for tc_device %s", id); + return NULL; + } + + d->id = strdup(id); + d->hash = simple_hash(d->id); + + d->classes_index.root = NULL; + d->classes_index.compar = tc_class_compare; +#ifdef AVL_LOCK_WITH_MUTEX + pthread_mutex_init(&d->classes_index.mutex, NULL); +#else + pthread_rwlock_init(&d->classes_index.rwlock, NULL); +#endif + + tc_device_index_add(d); + + if(!tc_device_root) { + tc_device_root = d; + } + else { + d->next = tc_device_root; + tc_device_root->prev = d; + tc_device_root = d; + } + } + + return(d); +} + +static struct tc_class *tc_class_add(struct tc_device *n, char *id, char *parentid, char *leafid) +{ + struct tc_class *c = tc_class_index_find(n, id, 0); + + if(!c) { + debug(D_TC_LOOP, "TC: Creating in device '%s', class id '%s', parentid '%s', leafid '%s'", n->id, id, parentid?parentid:"", leafid?leafid:""); + + c = calloc(1, sizeof(struct tc_class)); + if(!c) { + fatal("Cannot allocate memory for tc class"); + return NULL; + } + + if(n->classes) n->classes->prev = c; + c->next = n->classes; + n->classes = c; + + c->id = strdup(id); + if(!c->id) { + free(c); + return NULL; + } + c->hash = simple_hash(c->id); + + if(parentid && *parentid) { + c->parentid = strdup(parentid); + c->parent_hash = simple_hash(c->parentid); + } + + if(leafid && *leafid) { + c->leafid = strdup(leafid); + c->leaf_hash = simple_hash(c->leafid); + } + + tc_class_index_add(n, c); + } + + c->seen = 1; + + return(c); +} + +static void tc_device_free(struct tc_device *n) +{ + if(n->next) n->next->prev = n->prev; + if(n->prev) n->prev->next = n->next; + if(tc_device_root == n) { + if(n->next) tc_device_root = n->next; + else tc_device_root = n->prev; + } + + tc_device_index_del(n); + + while(n->classes) tc_class_free(n, n->classes); + + if(n->id) free(n->id); + if(n->name) free(n->name); + if(n->family) free(n->family); + + free(n); +} + +static void tc_device_free_all() +{ + while(tc_device_root) + tc_device_free(tc_device_root); +} + +#define MAX_WORDS 20 + +static inline int tc_space(char c) { + switch(c) { + case ' ': + case '\t': + case '\r': + case '\n': + return 1; + + default: + return 0; + } +} + +static void tc_split_words(char *str, char **words, int max_words) { + char *s = str; + int i = 0; + + // skip all white space + while(tc_space(*s)) s++; + + // store the first word + words[i++] = s; + + // while we have something + while(*s) { + // if it is a space + if(tc_space(*s)) { + + // terminate the word + *s++ = '\0'; + + // skip all white space + while(tc_space(*s)) s++; + + // if we reached the end, stop + if(!*s) break; + + // store the next word + if(i < max_words) words[i++] = s; + else break; + } + else s++; + } + + // terminate the words + while(i < max_words) words[i++] = NULL; +} + +pid_t tc_child_pid = 0; +void *tc_main(void *ptr) +{ + if(ptr) { ; } + + info("TC thread created with task id %d", gettid()); + + if(pthread_setcanceltype(PTHREAD_CANCEL_DEFERRED, NULL) != 0) + error("Cannot set pthread cancel type to DEFERRED."); + + if(pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, NULL) != 0) + error("Cannot set pthread cancel state to ENABLE."); + + struct rusage thread; + RRDSET *stcpu = NULL, *sttime = NULL; + + char buffer[TC_LINE_MAX+1] = ""; + char *words[MAX_WORDS] = { NULL }; + + uint32_t BEGIN_HASH = simple_hash("BEGIN"); + uint32_t END_HASH = simple_hash("END"); + uint32_t CLASS_HASH = simple_hash("class"); + uint32_t SENT_HASH = simple_hash("Sent"); + uint32_t LENDED_HASH = simple_hash("lended:"); + uint32_t TOKENS_HASH = simple_hash("tokens:"); + uint32_t SETDEVICENAME_HASH = simple_hash("SETDEVICENAME"); + uint32_t SETDEVICEGROUP_HASH = simple_hash("SETDEVICEGROUP"); + uint32_t SETCLASSNAME_HASH = simple_hash("SETCLASSNAME"); + uint32_t WORKTIME_HASH = simple_hash("WORKTIME"); +#ifdef DETACH_PLUGINS_FROM_NETDATA + uint32_t MYPID_HASH = simple_hash("MYPID"); +#endif + uint32_t first_hash; + + for(;1;) { + if(unlikely(netdata_exit)) break; + + FILE *fp; + struct tc_device *device = NULL; + struct tc_class *class = NULL; + + snprintf(buffer, TC_LINE_MAX, "exec %s %d", config_get("plugin:tc", "script to run to get tc values", PLUGINS_DIR "/tc-qos-helper.sh"), rrd_update_every); + debug(D_TC_LOOP, "executing '%s'", buffer); + // fp = popen(buffer, "r"); + fp = mypopen(buffer, &tc_child_pid); + if(!fp) { + error("TC: Cannot popen(\"%s\", \"r\").", buffer); + pthread_exit(NULL); + return NULL; + } + + while(fgets(buffer, TC_LINE_MAX, fp) != NULL) { + if(unlikely(netdata_exit)) break; + + buffer[TC_LINE_MAX] = '\0'; + // debug(D_TC_LOOP, "TC: read '%s'", buffer); + + tc_split_words(buffer, words, MAX_WORDS); + if(!words[0] || !*words[0]) { + // debug(D_TC_LOOP, "empty line"); + continue; + } + // else debug(D_TC_LOOP, "First word is '%s'", words[0]); + + first_hash = simple_hash(words[0]); + + if(device && first_hash == CLASS_HASH && strcmp(words[0], "class") == 0) { + // debug(D_TC_LOOP, "CLASS line on class id='%s', parent='%s', parentid='%s', leaf='%s', leafid='%s'", words[2], words[3], words[4], words[5], words[6]); + + // clear the last class + class = NULL; + + // words[1] : class type + // words[2] : N:XX + // words[3] : parent or root + if(words[1] && words[2] && words[3] && (strcmp(words[3], "parent") == 0 || strcmp(words[3], "root") == 0)) { + //char *type = words[1]; // the class: htb, fq_codel, etc + + // we are only interested for HTB classes + //if(strcmp(type, "htb") != 0) continue; + + char *id = words[2]; // the class major:minor + char *parent = words[3]; // 'parent' or 'root' + char *parentid = words[4]; // the parent's id + char *leaf = words[5]; // 'leaf' + char *leafid = words[6]; // leafid + + if(strcmp(parent, "root") == 0) { + parentid = NULL; + leafid = NULL; + } + else if(!leaf || strcmp(leaf, "leaf") != 0) + leafid = NULL; + + char leafbuf[20 + 1] = ""; + if(leafid && leafid[strlen(leafid) - 1] == ':') { + strncpy(leafbuf, leafid, 20 - 1); + strcat(leafbuf, "1"); + leafid = leafbuf; + } + + class = tc_class_add(device, id, parentid, leafid); + } + } + else if(first_hash == END_HASH && strcmp(words[0], "END") == 0) { + // debug(D_TC_LOOP, "END line"); + + if(device) { + if(pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, NULL) != 0) + error("Cannot set pthread cancel state to DISABLE."); + + tc_device_commit(device); + // tc_device_free(device); + device = NULL; + class = NULL; + + if(pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, NULL) != 0) + error("Cannot set pthread cancel state to ENABLE."); + } + } + else if(first_hash == BEGIN_HASH && strcmp(words[0], "BEGIN") == 0) { + // debug(D_TC_LOOP, "BEGIN line on device '%s'", words[1]); + + if(device) { + // tc_device_free(device); + device = NULL; + class = NULL; + } + + if(words[1] && *words[1]) { + device = tc_device_create(words[1]); + class = NULL; + } + } + else if(device && class && first_hash == SENT_HASH && strcmp(words[0], "Sent") == 0) { + // debug(D_TC_LOOP, "SENT line '%s'", words[1]); + if(words[1] && *words[1]) { + class->bytes = strtoull(words[1], NULL, 10); + class->updated = 1; + } + + if(words[3] && *words[3]) + class->packets = strtoull(words[3], NULL, 10); + + if(words[6] && *words[6]) + class->dropped = strtoull(words[6], NULL, 10); + + if(words[8] && *words[8]) + class->overlimits = strtoull(words[8], NULL, 10); + + if(words[10] && *words[10]) + class->requeues = strtoull(words[8], NULL, 10); + } + else if(device && class && class->updated && first_hash == LENDED_HASH && strcmp(words[0], "lended:") == 0) { + // debug(D_TC_LOOP, "LENDED line '%s'", words[1]); + if(words[1] && *words[1]) + class->lended = strtoull(words[1], NULL, 10); + + if(words[3] && *words[3]) + class->borrowed = strtoull(words[3], NULL, 10); + + if(words[5] && *words[5]) + class->giants = strtoull(words[5], NULL, 10); + } + else if(device && class && class->updated && first_hash == TOKENS_HASH && strcmp(words[0], "tokens:") == 0) { + // debug(D_TC_LOOP, "TOKENS line '%s'", words[1]); + if(words[1] && *words[1]) + class->tokens = strtoull(words[1], NULL, 10); + + if(words[3] && *words[3]) + class->ctokens = strtoull(words[3], NULL, 10); + } + else if(device && first_hash == SETDEVICENAME_HASH && strcmp(words[0], "SETDEVICENAME") == 0) { + // debug(D_TC_LOOP, "SETDEVICENAME line '%s'", words[1]); + if(words[1] && *words[1]) tc_device_set_device_name(device, words[1]); + } + else if(device && first_hash == SETDEVICEGROUP_HASH && strcmp(words[0], "SETDEVICEGROUP") == 0) { + // debug(D_TC_LOOP, "SETDEVICEGROUP line '%s'", words[1]); + if(words[1] && *words[1]) tc_device_set_device_family(device, words[1]); + } + else if(device && first_hash == SETCLASSNAME_HASH && strcmp(words[0], "SETCLASSNAME") == 0) { + // debug(D_TC_LOOP, "SETCLASSNAME line '%s' '%s'", words[1], words[2]); + char *id = words[1]; + char *path = words[2]; + if(id && *id && path && *path) tc_device_set_class_name(device, id, path); + } + else if(first_hash == WORKTIME_HASH && strcmp(words[0], "WORKTIME") == 0) { + // debug(D_TC_LOOP, "WORKTIME line '%s' '%s'", words[1], words[2]); + getrusage(RUSAGE_THREAD, &thread); + + if(!stcpu) stcpu = rrdset_find("netdata.plugin_tc_cpu"); + if(!stcpu) { + stcpu = rrdset_create("netdata", "plugin_tc_cpu", NULL, "tc.helper", NULL, "NetData TC CPU usage", "milliseconds/s", 135000, rrd_update_every, RRDSET_TYPE_STACKED); + rrddim_add(stcpu, "user", NULL, 1, 1000, RRDDIM_INCREMENTAL); + rrddim_add(stcpu, "system", NULL, 1, 1000, RRDDIM_INCREMENTAL); + } + else rrdset_next(stcpu); + + rrddim_set(stcpu, "user" , thread.ru_utime.tv_sec * 1000000ULL + thread.ru_utime.tv_usec); + rrddim_set(stcpu, "system", thread.ru_stime.tv_sec * 1000000ULL + thread.ru_stime.tv_usec); + rrdset_done(stcpu); + + if(!sttime) stcpu = rrdset_find("netdata.plugin_tc_time"); + if(!sttime) { + sttime = rrdset_create("netdata", "plugin_tc_time", NULL, "tc.helper", NULL, "NetData TC script execution", "milliseconds/run", 135001, rrd_update_every, RRDSET_TYPE_AREA); + rrddim_add(sttime, "run_time", "run time", 1, 1, RRDDIM_ABSOLUTE); + } + else rrdset_next(sttime); + + rrddim_set(sttime, "run_time", atoll(words[1])); + rrdset_done(sttime); + + } +#ifdef DETACH_PLUGINS_FROM_NETDATA + else if(first_hash == MYPID_HASH && (strcmp(words[0], "MYPID") == 0)) { + // debug(D_TC_LOOP, "MYPID line '%s'", words[1]); + char *id = words[1]; + pid_t pid = atol(id); + + if(pid) tc_child_pid = pid; + + debug(D_TC_LOOP, "TC: Child PID is %d.", tc_child_pid); + } +#endif + //else { + // debug(D_TC_LOOP, "IGNORED line"); + //} + } + mypclose(fp, tc_child_pid); + tc_child_pid = 0; + + if(device) { + // tc_device_free(device); + device = NULL; + class = NULL; + } + + if(netdata_exit) { + tc_device_free_all(); + pthread_exit(NULL); + return NULL; + } + + sleep((unsigned int) rrd_update_every); + } + + pthread_exit(NULL); + return NULL; +} + diff --git a/src/plugin_tc.h b/src/plugin_tc.h new file mode 100755 index 000000000..c3abbddd0 --- /dev/null +++ b/src/plugin_tc.h @@ -0,0 +1,8 @@ +#ifndef NETDATA_PLUGIN_TC_H +#define NETDATA_PLUGIN_TC_H 1 + +extern pid_t tc_child_pid; +extern void *tc_main(void *ptr); + +#endif /* NETDATA_PLUGIN_TC_H */ + diff --git a/src/plugins_d.c b/src/plugins_d.c new file mode 100755 index 000000000..eb8f141b8 --- /dev/null +++ b/src/plugins_d.c @@ -0,0 +1,532 @@ +#ifdef HAVE_CONFIG_H +#include <config.h> +#endif +#include <sys/types.h> +#include <dirent.h> +#include <pthread.h> +#include <string.h> +#include <stdio.h> +#include <stdlib.h> +#include <signal.h> + +#include "main.h" +#include "common.h" +#include "appconfig.h" +#include "log.h" +#include "rrd.h" +#include "popen.h" +#include "plugins_d.h" + +struct plugind *pluginsd_root = NULL; + +#define MAX_WORDS 20 + +static inline int pluginsd_space(char c) { + switch(c) { + case ' ': + case '\t': + case '\r': + case '\n': + case '=': + return 1; + + default: + return 0; + } +} + +static int pluginsd_split_words(char *str, char **words, int max_words) { + char *s = str, quote = 0; + int i = 0, j; + + // skip all white space + while(unlikely(pluginsd_space(*s))) s++; + + // check for quote + if(unlikely(*s == '\'' || *s == '"')) { + quote = *s; // remember the quote + s++; // skip the quote + } + + // store the first word + words[i++] = s; + + // while we have something + while(likely(*s)) { + // if it is escape + if(unlikely(*s == '\\' && s[1])) { + s += 2; + continue; + } + + // if it is quote + else if(unlikely(*s == quote)) { + quote = 0; + *s = ' '; + continue; + } + + // if it is a space + else if(unlikely(quote == 0 && pluginsd_space(*s))) { + + // terminate the word + *s++ = '\0'; + + // skip all white space + while(likely(pluginsd_space(*s))) s++; + + // check for quote + if(unlikely(*s == '\'' || *s == '"')) { + quote = *s; // remember the quote + s++; // skip the quote + } + + // if we reached the end, stop + if(unlikely(!*s)) break; + + // store the next word + if(likely(i < max_words)) words[i++] = s; + else break; + } + + // anything else + else s++; + } + + // terminate the words + j = i; + while(likely(j < max_words)) words[j++] = NULL; + + return i; +} + + +void *pluginsd_worker_thread(void *arg) +{ + struct plugind *cd = (struct plugind *)arg; + char line[PLUGINSD_LINE_MAX + 1]; + +#ifdef DETACH_PLUGINS_FROM_NETDATA + unsigned long long usec = 0, susec = 0; + struct timeval last = {0, 0} , now = {0, 0}; +#endif + + char *words[MAX_WORDS] = { NULL }; + uint32_t SET_HASH = simple_hash("SET"); + uint32_t BEGIN_HASH = simple_hash("BEGIN"); + uint32_t END_HASH = simple_hash("END"); + uint32_t FLUSH_HASH = simple_hash("FLUSH"); + uint32_t CHART_HASH = simple_hash("CHART"); + uint32_t DIMENSION_HASH = simple_hash("DIMENSION"); + uint32_t DISABLE_HASH = simple_hash("DISABLE"); +#ifdef DETACH_PLUGINS_FROM_NETDATA + uint32_t MYPID_HASH = simple_hash("MYPID"); + uint32_t STOPPING_WAKE_ME_UP_PLEASE_HASH = simple_hash("STOPPING_WAKE_ME_UP_PLEASE"); +#endif + + while(likely(1)) { + if(unlikely(netdata_exit)) break; + + FILE *fp = mypopen(cd->cmd, &cd->pid); + if(unlikely(!fp)) { + error("Cannot popen(\"%s\", \"r\").", cd->cmd); + break; + } + + info("PLUGINSD: '%s' running on pid %d", cd->fullfilename, cd->pid); + + RRDSET *st = NULL; + unsigned long long count = 0; + char *s; + uint32_t hash; + + while(likely(fgets(line, PLUGINSD_LINE_MAX, fp) != NULL)) { + if(unlikely(netdata_exit)) break; + + line[PLUGINSD_LINE_MAX] = '\0'; + + // debug(D_PLUGINSD, "PLUGINSD: %s: %s", cd->filename, line); + + int w = pluginsd_split_words(line, words, MAX_WORDS); + s = words[0]; + if(unlikely(!s || !*s || !w)) { + // debug(D_PLUGINSD, "PLUGINSD: empty line"); + continue; + } + + // debug(D_PLUGINSD, "PLUGINSD: words 0='%s' 1='%s' 2='%s' 3='%s' 4='%s' 5='%s' 6='%s' 7='%s' 8='%s' 9='%s'", words[0], words[1], words[2], words[3], words[4], words[5], words[6], words[7], words[8], words[9]); + + hash = simple_hash(s); + + if(likely(hash == SET_HASH && !strcmp(s, "SET"))) { + char *dimension = words[1]; + char *value = words[2]; + + if(unlikely(!dimension || !*dimension)) { + error("PLUGINSD: '%s' is requesting a SET on chart '%s', without a dimension. Disabling it.", cd->fullfilename, st->id); + cd->enabled = 0; + killpid(cd->pid, SIGTERM); + break; + } + + if(unlikely(!value || !*value)) value = NULL; + + if(unlikely(!st)) { + error("PLUGINSD: '%s' is requesting a SET on dimension %s with value %s, without a BEGIN. Disabling it.", cd->fullfilename, dimension, value?value:"<nothing>"); + cd->enabled = 0; + killpid(cd->pid, SIGTERM); + break; + } + + if(unlikely(st->debug)) debug(D_PLUGINSD, "PLUGINSD: '%s' is setting dimension %s/%s to %s", cd->fullfilename, st->id, dimension, value?value:"<nothing>"); + + if(value) rrddim_set(st, dimension, atoll(value)); + + count++; + } + else if(likely(hash == BEGIN_HASH && !strcmp(s, "BEGIN"))) { + char *id = words[1]; + char *microseconds_txt = words[2]; + + if(unlikely(!id)) { + error("PLUGINSD: '%s' is requesting a BEGIN without a chart id. Disabling it.", cd->fullfilename); + cd->enabled = 0; + killpid(cd->pid, SIGTERM); + break; + } + + st = rrdset_find(id); + if(unlikely(!st)) { + error("PLUGINSD: '%s' is requesting a BEGIN on chart '%s', which does not exist. Disabling it.", cd->fullfilename, id); + cd->enabled = 0; + killpid(cd->pid, SIGTERM); + break; + } + + if(likely(st->counter_done)) { + unsigned long long microseconds = 0; + if(microseconds_txt && *microseconds_txt) microseconds = strtoull(microseconds_txt, NULL, 10); + if(microseconds) rrdset_next_usec(st, microseconds); + else rrdset_next_plugins(st); + } + } + else if(likely(hash == END_HASH && !strcmp(s, "END"))) { + if(unlikely(!st)) { + error("PLUGINSD: '%s' is requesting an END, without a BEGIN. Disabling it.", cd->fullfilename); + cd->enabled = 0; + killpid(cd->pid, SIGTERM); + break; + } + + if(unlikely(st->debug)) debug(D_PLUGINSD, "PLUGINSD: '%s' is requesting a END on chart %s", cd->fullfilename, st->id); + + rrdset_done(st); + st = NULL; + } + else if(likely(hash == FLUSH_HASH && !strcmp(s, "FLUSH"))) { + debug(D_PLUGINSD, "PLUGINSD: '%s' is requesting a FLUSH", cd->fullfilename); + st = NULL; + } + else if(likely(hash == CHART_HASH && !strcmp(s, "CHART"))) { + int noname = 0; + st = NULL; + + if((words[1]) != NULL && (words[2]) != NULL && strcmp(words[1], words[2]) == 0) + noname = 1; + + char *type = words[1]; + char *id = NULL; + if(likely(type)) { + id = strchr(type, '.'); + if(likely(id)) { *id = '\0'; id++; } + } + char *name = words[2]; + char *title = words[3]; + char *units = words[4]; + char *family = words[5]; + char *context = words[6]; + char *chart = words[7]; + char *priority_s = words[8]; + char *update_every_s = words[9]; + + if(unlikely(!type || !*type || !id || !*id)) { + error("PLUGINSD: '%s' is requesting a CHART, without a type.id. Disabling it.", cd->fullfilename); + cd->enabled = 0; + killpid(cd->pid, SIGTERM); + break; + } + + int priority = 1000; + if(likely(priority_s)) priority = atoi(priority_s); + + int update_every = cd->update_every; + if(likely(update_every_s)) update_every = atoi(update_every_s); + if(unlikely(!update_every)) update_every = cd->update_every; + + int chart_type = RRDSET_TYPE_LINE; + if(unlikely(chart)) chart_type = rrdset_type_id(chart); + + if(unlikely(noname || !name || !*name || strcasecmp(name, "NULL") == 0 || strcasecmp(name, "(NULL)") == 0)) name = NULL; + if(unlikely(!family || !*family)) family = NULL; + if(unlikely(!context || !*context)) context = NULL; + + st = rrdset_find_bytype(type, id); + if(unlikely(!st)) { + debug(D_PLUGINSD, "PLUGINSD: Creating chart type='%s', id='%s', name='%s', family='%s', context='%s', chart='%s', priority=%d, update_every=%d" + , type, id + , name?name:"" + , family?family:"" + , context?context:"" + , rrdset_type_name(chart_type) + , priority + , update_every + ); + + st = rrdset_create(type, id, name, family, context, title, units, priority, update_every, chart_type); + cd->update_every = update_every; + } + else debug(D_PLUGINSD, "PLUGINSD: Chart '%s' already exists. Not adding it again.", st->id); + } + else if(likely(hash == DIMENSION_HASH && !strcmp(s, "DIMENSION"))) { + char *id = words[1]; + char *name = words[2]; + char *algorithm = words[3]; + char *multiplier_s = words[4]; + char *divisor_s = words[5]; + char *options = words[6]; + + if(unlikely(!id || !*id)) { + error("PLUGINSD: '%s' is requesting a DIMENSION, without an id. Disabling it.", cd->fullfilename); + cd->enabled = 0; + killpid(cd->pid, SIGTERM); + break; + } + + if(unlikely(!st)) { + error("PLUGINSD: '%s' is requesting a DIMENSION, without a CHART. Disabling it.", cd->fullfilename); + cd->enabled = 0; + killpid(cd->pid, SIGTERM); + break; + } + + long multiplier = 1; + if(multiplier_s && *multiplier_s) multiplier = atol(multiplier_s); + if(unlikely(!multiplier)) multiplier = 1; + + long divisor = 1; + if(likely(divisor_s && *divisor_s)) divisor = atol(divisor_s); + if(unlikely(!divisor)) divisor = 1; + + if(unlikely(!algorithm || !*algorithm)) algorithm = "absolute"; + + if(unlikely(st->debug)) debug(D_PLUGINSD, "PLUGINSD: Creating dimension in chart %s, id='%s', name='%s', algorithm='%s', multiplier=%ld, divisor=%ld, hidden='%s'" + , st->id + , id + , name?name:"" + , rrddim_algorithm_name(rrddim_algorithm_id(algorithm)) + , multiplier + , divisor + , options?options:"" + ); + + RRDDIM *rd = rrddim_find(st, id); + if(unlikely(!rd)) { + rd = rrddim_add(st, id, name, multiplier, divisor, rrddim_algorithm_id(algorithm)); + rd->flags = 0x00000000; + if(options && *options) { + if(strstr(options, "hidden") != NULL) rd->flags |= RRDDIM_FLAG_HIDDEN; + if(strstr(options, "noreset") != NULL) rd->flags |= RRDDIM_FLAG_DONT_DETECT_RESETS_OR_OVERFLOWS; + if(strstr(options, "nooverflow") != NULL) rd->flags |= RRDDIM_FLAG_DONT_DETECT_RESETS_OR_OVERFLOWS; + } + } + else if(unlikely(st->debug)) debug(D_PLUGINSD, "PLUGINSD: dimension %s/%s already exists. Not adding it again.", st->id, id); + } + else if(unlikely(hash == DISABLE_HASH && !strcmp(s, "DISABLE"))) { + error("PLUGINSD: '%s' called DISABLE. Disabling it.", cd->fullfilename); + cd->enabled = 0; + killpid(cd->pid, SIGTERM); + break; + } +#ifdef DETACH_PLUGINS_FROM_NETDATA + else if(likely(hash == MYPID_HASH && !strcmp(s, "MYPID"))) { + char *pid_s = words[1]; + pid_t pid = atol(pid_s); + + if(likely(pid)) cd->pid = pid; + debug(D_PLUGINSD, "PLUGINSD: %s is on pid %d", cd->id, cd->pid); + } + else if(likely(hash == STOPPING_WAKE_ME_UP_PLEASE_HASH && !strcmp(s, "STOPPING_WAKE_ME_UP_PLEASE"))) { + error("PLUGINSD: '%s' (pid %d) called STOPPING_WAKE_ME_UP_PLEASE.", cd->fullfilename, cd->pid); + + gettimeofday(&now, NULL); + if(unlikely(!usec && !susec)) { + // our first run + susec = cd->rrd_update_every * 1000000ULL; + } + else { + // second+ run + usec = usecdiff(&now, &last) - susec; + error("PLUGINSD: %s last loop took %llu usec (worked for %llu, sleeped for %llu).\n", cd->fullfilename, usec + susec, usec, susec); + if(unlikely(usec < (rrd_update_every * 1000000ULL / 2ULL))) susec = (rrd_update_every * 1000000ULL) - usec; + else susec = rrd_update_every * 1000000ULL / 2ULL; + } + + error("PLUGINSD: %s sleeping for %llu. Will kill with SIGCONT pid %d to wake it up.\n", cd->fullfilename, susec, cd->pid); + usleep(susec); + killpid(cd->pid, SIGCONT); + bcopy(&now, &last, sizeof(struct timeval)); + break; + } +#endif + else { + error("PLUGINSD: '%s' is sending command '%s' which is not known by netdata. Disabling it.", cd->fullfilename, s); + cd->enabled = 0; + killpid(cd->pid, SIGTERM); + break; + } + } + + info("PLUGINSD: '%s' on pid %d stopped.", cd->fullfilename, cd->pid); + + // fgets() failed or loop broke + int code = mypclose(fp, cd->pid); + if(code == 1 || code == 127) { + // 1 = DISABLE + // 127 = cannot even run it + error("PLUGINSD: '%s' (pid %d) exited with code %d. Disabling it.", cd->fullfilename, cd->pid, code); + cd->enabled = 0; + } + + if(netdata_exit) { + cd->pid = 0; + cd->enabled = 0; + cd->obsolete = 1; + pthread_exit(NULL); + return NULL; + } + + if(unlikely(!count && cd->enabled)) { + error("PLUGINSD: '%s' (pid %d) does not generate usefull output. Waiting a bit before starting it again.", cd->fullfilename, cd->pid); + sleep((unsigned int) (cd->update_every * 10)); + } + + cd->pid = 0; + if(likely(cd->enabled)) sleep((unsigned int) cd->update_every); + else break; + } + + cd->obsolete = 1; + pthread_exit(NULL); + return NULL; +} + +void *pluginsd_main(void *ptr) +{ + if(ptr) { ; } + + info("PLUGINS.D thread created with task id %d", gettid()); + + if(pthread_setcanceltype(PTHREAD_CANCEL_DEFERRED, NULL) != 0) + error("Cannot set pthread cancel type to DEFERRED."); + + if(pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, NULL) != 0) + error("Cannot set pthread cancel state to ENABLE."); + + char *dir_name = config_get("plugins", "plugins directory", PLUGINS_DIR); + int automatic_run = config_get_boolean("plugins", "enable running new plugins", 1); + int scan_frequency = (int) config_get_number("plugins", "check for new plugins every", 60); + DIR *dir = NULL; + struct dirent *file = NULL; + struct plugind *cd; + + // enable the apps plugin by default + // config_get_boolean("plugins", "apps", 1); + + if(scan_frequency < 1) scan_frequency = 1; + + while(likely(1)) { + if(unlikely(netdata_exit)) break; + + dir = opendir(dir_name); + if(unlikely(!dir)) { + error("Cannot open directory '%s'.", dir_name); + pthread_exit(NULL); + return NULL; + } + + while(likely((file = readdir(dir)))) { + if(unlikely(netdata_exit)) break; + + debug(D_PLUGINSD, "PLUGINSD: Examining file '%s'", file->d_name); + + if(unlikely(strcmp(file->d_name, ".") == 0 || strcmp(file->d_name, "..") == 0)) continue; + + int len = (int) strlen(file->d_name); + if(unlikely(len <= (int)PLUGINSD_FILE_SUFFIX_LEN)) continue; + if(unlikely(strcmp(PLUGINSD_FILE_SUFFIX, &file->d_name[len - (int)PLUGINSD_FILE_SUFFIX_LEN]) != 0)) { + debug(D_PLUGINSD, "PLUGINSD: File '%s' does not end in '%s'.", file->d_name, PLUGINSD_FILE_SUFFIX); + continue; + } + + char pluginname[CONFIG_MAX_NAME + 1]; + snprintf(pluginname, CONFIG_MAX_NAME, "%.*s", (int)(len - PLUGINSD_FILE_SUFFIX_LEN), file->d_name); + int enabled = config_get_boolean("plugins", pluginname, automatic_run); + + if(unlikely(!enabled)) { + debug(D_PLUGINSD, "PLUGINSD: plugin '%s' is not enabled", file->d_name); + continue; + } + + // check if it runs already + for(cd = pluginsd_root ; likely(cd) ; cd = cd->next) { + if(unlikely(strcmp(cd->filename, file->d_name) == 0)) break; + } + if(likely(cd && !cd->obsolete)) { + debug(D_PLUGINSD, "PLUGINSD: plugin '%s' is already running", cd->filename); + continue; + } + + // it is not running + // allocate a new one, or use the obsolete one + if(unlikely(!cd)) { + cd = calloc(sizeof(struct plugind), 1); + if(unlikely(!cd)) fatal("Cannot allocate memory for plugin."); + + snprintf(cd->id, CONFIG_MAX_NAME, "plugin:%s", pluginname); + + strncpy(cd->filename, file->d_name, FILENAME_MAX); + snprintf(cd->fullfilename, FILENAME_MAX, "%s/%s", dir_name, cd->filename); + + cd->enabled = enabled; + cd->update_every = (int) config_get_number(cd->id, "update every", rrd_update_every); + cd->started_t = time(NULL); + + char *def = ""; + snprintf(cd->cmd, PLUGINSD_CMD_MAX, "exec %s %d %s", cd->fullfilename, cd->update_every, config_get(cd->id, "command options", def)); + + // link it + if(likely(pluginsd_root)) cd->next = pluginsd_root; + pluginsd_root = cd; + } + cd->obsolete = 0; + + if(unlikely(!cd->enabled)) continue; + + // spawn a new thread for it + if(unlikely(pthread_create(&cd->thread, NULL, pluginsd_worker_thread, cd) != 0)) { + error("PLUGINSD: failed to create new thread for plugin '%s'.", cd->filename); + cd->obsolete = 1; + } + else if(unlikely(pthread_detach(cd->thread) != 0)) + error("PLUGINSD: Cannot request detach of newly created thread for plugin '%s'.", cd->filename); + } + + closedir(dir); + sleep((unsigned int) scan_frequency); + } + + pthread_exit(NULL); + return NULL; +} + + diff --git a/src/plugins_d.h b/src/plugins_d.h new file mode 100755 index 000000000..11e89e0ac --- /dev/null +++ b/src/plugins_d.h @@ -0,0 +1,36 @@ +#include <sys/types.h> +#include <unistd.h> + + +#ifndef NETDATA_PLUGINS_D_H +#define NETDATA_PLUGINS_D_H 1 + +#define PLUGINSD_FILE_SUFFIX ".plugin" +#define PLUGINSD_FILE_SUFFIX_LEN strlen(PLUGINSD_FILE_SUFFIX) +#define PLUGINSD_CMD_MAX (FILENAME_MAX*2) +#define PLUGINSD_LINE_MAX 1024 + +struct plugind { + char id[CONFIG_MAX_NAME+1]; // config node id + + char filename[FILENAME_MAX+1]; // just the filename + char fullfilename[FILENAME_MAX+1]; // with path + char cmd[PLUGINSD_CMD_MAX+1]; // the command that is executes + + pid_t pid; + pthread_t thread; + + int update_every; + int obsolete; + int enabled; + + time_t started_t; + + struct plugind *next; +}; + +extern struct plugind *pluginsd_root; + +extern void *pluginsd_main(void *ptr); + +#endif /* NETDATA_PLUGINS_D_H */ diff --git a/src/popen.c b/src/popen.c new file mode 100755 index 000000000..882a4cc5a --- /dev/null +++ b/src/popen.c @@ -0,0 +1,171 @@ +#ifdef HAVE_CONFIG_H +#include <config.h> +#endif +#include <signal.h> +#include <unistd.h> +#include <stdlib.h> +#include <sys/types.h> +#include <sys/wait.h> + +#include "log.h" +#include "popen.h" +#include "common.h" + +/* +struct mypopen { + pid_t pid; + FILE *fp; + struct mypopen *next; + struct mypopen *prev; +}; + +static struct mypopen *mypopen_root = NULL; + +static void mypopen_add(FILE *fp, pid_t *pid) { + struct mypopen *mp = malloc(sizeof(struct mypopen)); + if(!mp) { + fatal("Cannot allocate %zu bytes", sizeof(struct mypopen)) + return; + } + + mp->fp = fp; + mp->pid = pid; + mp->next = popen_root; + mp->prev = NULL; + if(mypopen_root) mypopen_root->prev = mp; + mypopen_root = mp; +} + +static void mypopen_del(FILE *fp) { + struct mypopen *mp; + + for(mp = mypopen_root; mp; mp = mp->next) + if(mp->fd == fp) break; + + if(!mp) error("Cannot find mypopen() file pointer in open childs."); + else { + if(mp->next) mp->next->prev = mp->prev; + if(mp->prev) mp->prev->next = mp->next; + if(mypopen_root == mp) mypopen_root = mp->next; + free(mp); + } +} +*/ +#define PIPE_READ 0 +#define PIPE_WRITE 1 + +FILE *mypopen(const char *command, pid_t *pidptr) +{ + int pipefd[2]; + + if(pipe(pipefd) == -1) return NULL; + + int pid = fork(); + if(pid == -1) { + close(pipefd[PIPE_READ]); + close(pipefd[PIPE_WRITE]); + return NULL; + } + if(pid != 0) { + // the parent + *pidptr = pid; + close(pipefd[PIPE_WRITE]); + FILE *fp = fdopen(pipefd[PIPE_READ], "r"); + /*mypopen_add(fp, pid);*/ + return(fp); + } + // the child + + // close all files + int i; + for(i = (int) (sysconf(_SC_OPEN_MAX) - 1); i > 0; i--) + if(i != STDIN_FILENO && i != STDERR_FILENO && i != pipefd[PIPE_WRITE]) close(i); + + // move the pipe to stdout + if(pipefd[PIPE_WRITE] != STDOUT_FILENO) { + dup2(pipefd[PIPE_WRITE], STDOUT_FILENO); + close(pipefd[PIPE_WRITE]); + } + +#ifdef DETACH_PLUGINS_FROM_NETDATA + // this was an attempt to detach the child and use the suspend mode charts.d + // unfortunatelly it does not work as expected. + + // fork again to become session leader + pid = fork(); + if(pid == -1) fprintf(stderr, "Cannot fork again on pid %d\n", getpid()); + if(pid != 0) { + // the parent + exit(0); + } + + // set a new process group id for just this child + if( setpgid(0, 0) != 0 ) + error("Cannot set a new process group for pid %d (%s)", getpid(), strerror(errno)); + + if( getpgid(0) != getpid() ) + error("Process group set is incorrect. Expected %d, found %d", getpid(), getpgid(0)); + + if( setsid() != 0 ) + error("Cannot set session id for pid %d (%s)", getpid(), strerror(errno)); + + fprintf(stdout, "MYPID %d\n", getpid()); + fflush(NULL); +#endif + + // reset all signals + for (i = 1 ; i < 65 ;i++) if(i != SIGSEGV) signal(i, SIG_DFL); + + info("executing command: '%s' on pid %d.", command, getpid()); + execl("/bin/sh", "sh", "-c", command, NULL); + exit(1); +} + +int mypclose(FILE *fp, pid_t pid) { + debug(D_EXIT, "Request to mypclose() on pid %d", pid); + + /*mypopen_del(fp);*/ + fclose(fp); + + siginfo_t info; + if(waitid(P_PID, (id_t) pid, &info, WEXITED) != -1) { + switch(info.si_code) { + case CLD_EXITED: + error("pid %d exited with code %d.", info.si_pid, info.si_status); + return(info.si_status); + break; + + case CLD_KILLED: + error("pid %d killed by signal %d.", info.si_pid, info.si_status); + return(-1); + break; + + case CLD_DUMPED: + error("pid %d core dumped by signal %d.", info.si_pid, info.si_status); + return(-2); + break; + + case CLD_STOPPED: + error("pid %d stopped by signal %d.", info.si_pid, info.si_status); + return(0); + break; + + case CLD_TRAPPED: + error("pid %d trapped by signal %d.", info.si_pid, info.si_status); + return(-4); + break; + + case CLD_CONTINUED: + error("pid %d continued by signal %d.", info.si_pid, info.si_status); + return(0); + break; + + default: + error("pid %d gave us a SIGCHLD with code %d and status %d.", info.si_pid, info.si_code, info.si_status); + return(-5); + break; + } + } + else error("Cannot waitid() for pid %d", pid); + return 0; +} diff --git a/src/popen.h b/src/popen.h new file mode 100755 index 000000000..10680f0c8 --- /dev/null +++ b/src/popen.h @@ -0,0 +1,14 @@ +#include <stdio.h> +#include <unistd.h> +#include <sys/types.h> + +#ifndef NETDATA_POPEN_H +#define NETDATA_POPEN_H 1 + +#define PIPE_READ 0 +#define PIPE_WRITE 1 + +extern FILE *mypopen(const char *command, pid_t *pidptr); +extern int mypclose(FILE *fp, pid_t pid); + +#endif /* NETDATA_POPEN_H */ diff --git a/src/proc_diskstats.c b/src/proc_diskstats.c new file mode 100755 index 000000000..f404118fa --- /dev/null +++ b/src/proc_diskstats.c @@ -0,0 +1,470 @@ +#ifdef HAVE_CONFIG_H +#include <config.h> +#endif +#include <stdio.h> +#include <stdlib.h> +#include <string.h> + +#include "common.h" +#include "log.h" +#include "appconfig.h" +#include "procfile.h" +#include "rrd.h" +#include "plugin_proc.h" + +#define RRD_TYPE_DISK "disk" + +int do_proc_diskstats(int update_every, unsigned long long dt) { + static procfile *ff = NULL; + static char path_to_get_hw_sector_size[FILENAME_MAX + 1] = ""; + static int enable_new_disks = -1; + static int do_io = -1, do_ops = -1, do_mops = -1, do_iotime = -1, do_qops = -1, do_util = -1, do_backlog = -1; + + if(enable_new_disks == -1) enable_new_disks = config_get_boolean_ondemand("plugin:proc:/proc/diskstats", "enable new disks detected at runtime", CONFIG_ONDEMAND_ONDEMAND); + + if(do_io == -1) do_io = config_get_boolean_ondemand("plugin:proc:/proc/diskstats", "bandwidth for all disks", CONFIG_ONDEMAND_ONDEMAND); + if(do_ops == -1) do_ops = config_get_boolean_ondemand("plugin:proc:/proc/diskstats", "operations for all disks", CONFIG_ONDEMAND_ONDEMAND); + if(do_mops == -1) do_mops = config_get_boolean_ondemand("plugin:proc:/proc/diskstats", "merged operations for all disks", CONFIG_ONDEMAND_ONDEMAND); + if(do_iotime == -1) do_iotime = config_get_boolean_ondemand("plugin:proc:/proc/diskstats", "i/o time for all disks", CONFIG_ONDEMAND_ONDEMAND); + if(do_qops == -1) do_qops = config_get_boolean_ondemand("plugin:proc:/proc/diskstats", "queued operations for all disks", CONFIG_ONDEMAND_ONDEMAND); + if(do_util == -1) do_util = config_get_boolean_ondemand("plugin:proc:/proc/diskstats", "utilization percentage for all disks", CONFIG_ONDEMAND_ONDEMAND); + if(do_backlog == -1)do_backlog = config_get_boolean_ondemand("plugin:proc:/proc/diskstats", "backlog for all disks", CONFIG_ONDEMAND_ONDEMAND); + + if(!ff) { + char filename[FILENAME_MAX + 1]; + snprintf(filename, FILENAME_MAX, "%s%s", global_host_prefix, "/proc/diskstats"); + ff = procfile_open(config_get("plugin:proc:/proc/diskstats", "filename to monitor", filename), " \t", PROCFILE_FLAG_DEFAULT); + } + if(!ff) return 1; + + if(!path_to_get_hw_sector_size[0]) { + char filename[FILENAME_MAX + 1]; + snprintf(filename, FILENAME_MAX, "%s%s", global_host_prefix, "/sys/block/%s/queue/hw_sector_size"); + snprintf(path_to_get_hw_sector_size, FILENAME_MAX, "%s%s", global_host_prefix, config_get("plugin:proc:/proc/diskstats", "path to get h/w sector size", filename)); + } + + ff = procfile_readall(ff); + if(!ff) return 0; // we return 0, so that we will retry to open it next time + + uint32_t lines = procfile_lines(ff), l; + uint32_t words; + + for(l = 0; l < lines ;l++) { + char *disk; + unsigned long long major = 0, minor = 0, + reads = 0, mreads = 0, readsectors = 0, readms = 0, + writes = 0, mwrites = 0, writesectors = 0, writems = 0, + queued_ios = 0, busy_ms = 0, backlog_ms = 0; + + unsigned long long last_reads = 0, last_readsectors = 0, last_readms = 0, + last_writes = 0, last_writesectors = 0, last_writems = 0, + last_busy_ms = 0; + + words = procfile_linewords(ff, l); + if(words < 14) continue; + + major = strtoull(procfile_lineword(ff, l, 0), NULL, 10); + minor = strtoull(procfile_lineword(ff, l, 1), NULL, 10); + disk = procfile_lineword(ff, l, 2); + + // # of reads completed # of writes completed + // This is the total number of reads or writes completed successfully. + reads = strtoull(procfile_lineword(ff, l, 3), NULL, 10); // rd_ios + writes = strtoull(procfile_lineword(ff, l, 7), NULL, 10); // wr_ios + + // # of reads merged # of writes merged + // Reads and writes which are adjacent to each other may be merged for + // efficiency. Thus two 4K reads may become one 8K read before it is + // ultimately handed to the disk, and so it will be counted (and queued) + mreads = strtoull(procfile_lineword(ff, l, 4), NULL, 10); // rd_merges_or_rd_sec + mwrites = strtoull(procfile_lineword(ff, l, 8), NULL, 10); // wr_merges + + // # of sectors read # of sectors written + // This is the total number of sectors read or written successfully. + readsectors = strtoull(procfile_lineword(ff, l, 5), NULL, 10); // rd_sec_or_wr_ios + writesectors = strtoull(procfile_lineword(ff, l, 9), NULL, 10); // wr_sec + + // # of milliseconds spent reading # of milliseconds spent writing + // This is the total number of milliseconds spent by all reads or writes (as + // measured from __make_request() to end_that_request_last()). + readms = strtoull(procfile_lineword(ff, l, 6), NULL, 10); // rd_ticks_or_wr_sec + writems = strtoull(procfile_lineword(ff, l, 10), NULL, 10); // wr_ticks + + // # of I/Os currently in progress + // The only field that should go to zero. Incremented as requests are + // given to appropriate struct request_queue and decremented as they finish. + queued_ios = strtoull(procfile_lineword(ff, l, 11), NULL, 10); // ios_pgr + + // # of milliseconds spent doing I/Os + // This field increases so long as field queued_ios is nonzero. + busy_ms = strtoull(procfile_lineword(ff, l, 12), NULL, 10); // tot_ticks + + // weighted # of milliseconds spent doing I/Os + // This field is incremented at each I/O start, I/O completion, I/O + // merge, or read of these stats by the number of I/Os in progress + // (field queued_ios) times the number of milliseconds spent doing I/O since the + // last update of this field. This can provide an easy measure of both + // I/O completion time and the backlog that may be accumulating. + backlog_ms = strtoull(procfile_lineword(ff, l, 13), NULL, 10); // rq_ticks + + + int def_enabled = 0; + + // remove slashes from disk names + char *s; + for(s = disk; *s ;s++) if(*s == '/') *s = '_'; + + switch(major) { + case 9: // MDs + case 43: // network block + case 144: // nfs + case 145: // nfs + case 146: // nfs + case 199: // veritas + case 201: // veritas + case 251: // dm + def_enabled = enable_new_disks; + break; + + case 48: // RAID + case 49: // RAID + case 50: // RAID + case 51: // RAID + case 52: // RAID + case 53: // RAID + case 54: // RAID + case 55: // RAID + case 112: // RAID + case 136: // RAID + case 137: // RAID + case 138: // RAID + case 139: // RAID + case 140: // RAID + case 141: // RAID + case 142: // RAID + case 143: // RAID + case 179: // MMC + case 180: // USB + if(minor % 8) def_enabled = 0; // partitions + else def_enabled = enable_new_disks; + break; + + case 8: // scsi disks + case 65: // scsi disks + case 66: // scsi disks + case 67: // scsi disks + case 68: // scsi disks + case 69: // scsi disks + case 70: // scsi disks + case 71: // scsi disks + case 72: // scsi disks + case 73: // scsi disks + case 74: // scsi disks + case 75: // scsi disks + case 76: // scsi disks + case 77: // scsi disks + case 78: // scsi disks + case 79: // scsi disks + case 80: // i2o + case 81: // i2o + case 82: // i2o + case 83: // i2o + case 84: // i2o + case 85: // i2o + case 86: // i2o + case 87: // i2o + case 101: // hyperdisk + case 102: // compressed + case 104: // scsi + case 105: // scsi + case 106: // scsi + case 107: // scsi + case 108: // scsi + case 109: // scsi + case 110: // scsi + case 111: // scsi + case 114: // bios raid + case 116: // ram board + case 128: // scsi + case 129: // scsi + case 130: // scsi + case 131: // scsi + case 132: // scsi + case 133: // scsi + case 134: // scsi + case 135: // scsi + case 153: // raid + case 202: // xen + case 256: // flash + case 257: // flash + if(minor % 16) def_enabled = 0; // partitions + else def_enabled = enable_new_disks; + break; + + case 160: // raid + case 161: // raid + if(minor % 32) def_enabled = 0; // partitions + else def_enabled = enable_new_disks; + break; + + case 3: // ide + case 13: // 8bit ide + case 22: // ide + case 33: // ide + case 34: // ide + case 56: // ide + case 57: // ide + case 88: // ide + case 89: // ide + case 90: // ide + case 91: // ide + if(minor % 64) def_enabled = 0; // partitions + else def_enabled = enable_new_disks; + break; + + case 252: // zram + def_enabled = 0; + break; + + default: + def_enabled = 0; + break; + } + + int ddo_io = do_io, ddo_ops = do_ops, ddo_mops = do_mops, ddo_iotime = do_iotime, ddo_qops = do_qops, ddo_util = do_util, ddo_backlog = do_backlog; + + // check which charts are enabled for this disk + { + char var_name[4096 + 1]; + snprintf(var_name, 4096, "plugin:proc:/proc/diskstats:%s", disk); + def_enabled = config_get_boolean_ondemand(var_name, "enabled", def_enabled); + if(def_enabled == CONFIG_ONDEMAND_NO) continue; + if(def_enabled == CONFIG_ONDEMAND_ONDEMAND && !reads && !writes) continue; + + + ddo_io = config_get_boolean_ondemand(var_name, "bandwidth", ddo_io); + ddo_ops = config_get_boolean_ondemand(var_name, "operations", ddo_ops); + ddo_mops = config_get_boolean_ondemand(var_name, "merged operations", ddo_mops); + ddo_iotime = config_get_boolean_ondemand(var_name, "i/o time", ddo_iotime); + ddo_qops = config_get_boolean_ondemand(var_name, "queued operations", ddo_qops); + ddo_util = config_get_boolean_ondemand(var_name, "utilization percentage", ddo_util); + ddo_backlog = config_get_boolean_ondemand(var_name, "backlog", ddo_backlog); + + // by default, do not add charts that do not have values + if(ddo_io == CONFIG_ONDEMAND_ONDEMAND && !reads && !writes) ddo_io = 0; + if(ddo_mops == CONFIG_ONDEMAND_ONDEMAND && mreads == 0 && mwrites == 0) ddo_mops = 0; + if(ddo_iotime == CONFIG_ONDEMAND_ONDEMAND && readms == 0 && writems == 0) ddo_iotime = 0; + if(ddo_util == CONFIG_ONDEMAND_ONDEMAND && busy_ms == 0) ddo_util = 0; + if(ddo_backlog == CONFIG_ONDEMAND_ONDEMAND && backlog_ms == 0) ddo_backlog = 0; + if(ddo_qops == CONFIG_ONDEMAND_ONDEMAND && backlog_ms == 0) ddo_qops = 0; + + // for absolute values, we need to switch the setting to 'yes' + // to allow it refresh from now on + if(ddo_qops == CONFIG_ONDEMAND_ONDEMAND) config_set(var_name, "queued operations", "yes"); + } + + RRDSET *st; + + // -------------------------------------------------------------------- + + int sector_size = 512; + if(ddo_io) { + st = rrdset_find_bytype(RRD_TYPE_DISK, disk); + if(!st) { + char tf[FILENAME_MAX + 1], *t; + char ssfilename[FILENAME_MAX + 1]; + + strncpy(tf, disk, FILENAME_MAX); + tf[FILENAME_MAX] = '\0'; + + // replace all / with ! + while((t = strchr(tf, '/'))) *t = '!'; + + snprintf(ssfilename, FILENAME_MAX, path_to_get_hw_sector_size, tf); + FILE *fpss = fopen(ssfilename, "r"); + if(fpss) { + char ssbuffer[1025]; + char *tmp = fgets(ssbuffer, 1024, fpss); + + if(tmp) { + sector_size = atoi(tmp); + if(sector_size <= 0) { + error("Invalid sector size %d for device %s in %s. Assuming 512.", sector_size, disk, ssfilename); + sector_size = 512; + } + } + else error("Cannot read data for sector size for device %s from %s. Assuming 512.", disk, ssfilename); + + fclose(fpss); + } + else error("Cannot read sector size for device %s from %s. Assuming 512.", disk, ssfilename); + + st = rrdset_create(RRD_TYPE_DISK, disk, NULL, disk, "disk.io", "Disk I/O Bandwidth", "kilobytes/s", 2000, update_every, RRDSET_TYPE_AREA); + + rrddim_add(st, "reads", NULL, sector_size, 1024, RRDDIM_INCREMENTAL); + rrddim_add(st, "writes", NULL, sector_size * -1, 1024, RRDDIM_INCREMENTAL); + } + else rrdset_next_usec(st, dt); + + last_readsectors = rrddim_set(st, "reads", readsectors); + last_writesectors = rrddim_set(st, "writes", writesectors); + rrdset_done(st); + } + + // -------------------------------------------------------------------- + + if(ddo_ops) { + st = rrdset_find_bytype("disk_ops", disk); + if(!st) { + st = rrdset_create("disk_ops", disk, NULL, disk, "disk.ops", "Disk Completed I/O Operations", "operations/s", 2001, update_every, RRDSET_TYPE_LINE); + st->isdetail = 1; + + rrddim_add(st, "reads", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "writes", NULL, -1, 1, RRDDIM_INCREMENTAL); + } + else rrdset_next_usec(st, dt); + + last_reads = rrddim_set(st, "reads", reads); + last_writes = rrddim_set(st, "writes", writes); + rrdset_done(st); + } + + // -------------------------------------------------------------------- + + if(ddo_qops) { + st = rrdset_find_bytype("disk_qops", disk); + if(!st) { + st = rrdset_create("disk_qops", disk, NULL, disk, "disk.qops", "Disk Current I/O Operations", "operations", 2002, update_every, RRDSET_TYPE_LINE); + st->isdetail = 1; + + rrddim_add(st, "operations", NULL, 1, 1, RRDDIM_ABSOLUTE); + } + else rrdset_next_usec(st, dt); + + rrddim_set(st, "operations", queued_ios); + rrdset_done(st); + } + + // -------------------------------------------------------------------- + + if(ddo_backlog) { + st = rrdset_find_bytype("disk_backlog", disk); + if(!st) { + st = rrdset_create("disk_backlog", disk, NULL, disk, "disk.backlog", "Disk Backlog", "backlog (ms)", 2003, update_every, RRDSET_TYPE_AREA); + st->isdetail = 1; + + rrddim_add(st, "backlog", NULL, 1, 10, RRDDIM_INCREMENTAL); + } + else rrdset_next_usec(st, dt); + + rrddim_set(st, "backlog", backlog_ms); + rrdset_done(st); + } + + // -------------------------------------------------------------------- + + if(ddo_util) { + st = rrdset_find_bytype("disk_util", disk); + if(!st) { + st = rrdset_create("disk_util", disk, NULL, disk, "disk.util", "Disk Utilization Time", "% of time working", 2004, update_every, RRDSET_TYPE_AREA); + st->isdetail = 1; + + rrddim_add(st, "utilization", NULL, 1, 10, RRDDIM_INCREMENTAL); + } + else rrdset_next_usec(st, dt); + + last_busy_ms = rrddim_set(st, "utilization", busy_ms); + rrdset_done(st); + } + + // -------------------------------------------------------------------- + + if(ddo_mops) { + st = rrdset_find_bytype("disk_mops", disk); + if(!st) { + st = rrdset_create("disk_mops", disk, NULL, disk, "disk.mops", "Disk Merged Operations", "merged operations/s", 2021, update_every, RRDSET_TYPE_LINE); + st->isdetail = 1; + + rrddim_add(st, "reads", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "writes", NULL, -1, 1, RRDDIM_INCREMENTAL); + } + else rrdset_next_usec(st, dt); + + rrddim_set(st, "reads", mreads); + rrddim_set(st, "writes", mwrites); + rrdset_done(st); + } + + // -------------------------------------------------------------------- + + if(ddo_iotime) { + st = rrdset_find_bytype("disk_iotime", disk); + if(!st) { + st = rrdset_create("disk_iotime", disk, NULL, disk, "disk.iotime", "Disk Total I/O Time", "milliseconds/s", 2022, update_every, RRDSET_TYPE_LINE); + st->isdetail = 1; + + rrddim_add(st, "reads", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "writes", NULL, -1, 1, RRDDIM_INCREMENTAL); + } + else rrdset_next_usec(st, dt); + + last_readms = rrddim_set(st, "reads", readms); + last_writems = rrddim_set(st, "writes", writems); + rrdset_done(st); + } + + // -------------------------------------------------------------------- + // calculate differential charts + // only if this is not the first time we run + + if(dt) { + if(ddo_iotime && ddo_ops) { + st = rrdset_find_bytype("disk_await", disk); + if(!st) { + st = rrdset_create("disk_await", disk, NULL, disk, "disk.await", "Average Completed I/O Operation Time", "ms per operation", 2005, update_every, RRDSET_TYPE_LINE); + st->isdetail = 1; + + rrddim_add(st, "reads", NULL, 1, 1, RRDDIM_ABSOLUTE); + rrddim_add(st, "writes", NULL, -1, 1, RRDDIM_ABSOLUTE); + } + else rrdset_next_usec(st, dt); + + rrddim_set(st, "reads", (reads - last_reads) ? (readms - last_readms) / (reads - last_reads) : 0); + rrddim_set(st, "writes", (writes - last_writes) ? (writems - last_writems) / (writes - last_writes) : 0); + rrdset_done(st); + } + + if(ddo_io && ddo_ops) { + st = rrdset_find_bytype("disk_avgsz", disk); + if(!st) { + st = rrdset_create("disk_avgsz", disk, NULL, disk, "disk.avgsz", "Average Completed I/O Operation Bandwidth", "kilobytes per operation", 2006, update_every, RRDSET_TYPE_AREA); + st->isdetail = 1; + + rrddim_add(st, "reads", NULL, sector_size, 1024, RRDDIM_ABSOLUTE); + rrddim_add(st, "writes", NULL, -sector_size, 1024, RRDDIM_ABSOLUTE); + } + else rrdset_next_usec(st, dt); + + rrddim_set(st, "reads", (reads - last_reads) ? (readsectors - last_readsectors) / (reads - last_reads) : 0); + rrddim_set(st, "writes", (writes - last_writes) ? (writesectors - last_writesectors) / (writes - last_writes) : 0); + rrdset_done(st); + } + + if(ddo_util && ddo_ops) { + st = rrdset_find_bytype("disk_svctm", disk); + if(!st) { + st = rrdset_create("disk_svctm", disk, NULL, disk, "disk.svctm", "Average Service Time", "ms per operation", 2007, update_every, RRDSET_TYPE_LINE); + st->isdetail = 1; + + rrddim_add(st, "svctm", NULL, 1, 1, RRDDIM_ABSOLUTE); + } + else rrdset_next_usec(st, dt); + + rrddim_set(st, "svctm", ((reads - last_reads) + (writes - last_writes)) ? (busy_ms - last_busy_ms) / ((reads - last_reads) + (writes - last_writes)) : 0); + rrdset_done(st); + } + } + } + + return 0; +} diff --git a/src/proc_interrupts.c b/src/proc_interrupts.c new file mode 100755 index 000000000..25482dfa7 --- /dev/null +++ b/src/proc_interrupts.c @@ -0,0 +1,164 @@ +#ifdef HAVE_CONFIG_H +#include <config.h> +#endif +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <ctype.h> + +#include "common.h" +#include "appconfig.h" +#include "procfile.h" +#include "rrd.h" +#include "plugin_proc.h" +#include "log.h" + +#define MAX_INTERRUPTS 256 +#define MAX_INTERRUPT_CPUS 256 +#define MAX_INTERRUPT_NAME 50 + +struct interrupt { + int used; + char *id; + char name[MAX_INTERRUPT_NAME + 1]; + unsigned long long value[MAX_INTERRUPT_CPUS]; + unsigned long long total; +}; + +int do_proc_interrupts(int update_every, unsigned long long dt) { + static procfile *ff = NULL; + static int cpus = -1, do_per_core = -1; + + if(dt) {}; + + if(do_per_core == -1) do_per_core = config_get_boolean("plugin:proc:/proc/interrupts", "interrupts per core", 1); + + if(!ff) { + char filename[FILENAME_MAX + 1]; + snprintf(filename, FILENAME_MAX, "%s%s", global_host_prefix, "/proc/interrupts"); + ff = procfile_open(config_get("plugin:proc:/proc/interrupts", "filename to monitor", filename), " \t", PROCFILE_FLAG_DEFAULT); + } + if(!ff) return 1; + + ff = procfile_readall(ff); + if(!ff) return 0; // we return 0, so that we will retry to open it next time + + uint32_t lines = procfile_lines(ff), l; + uint32_t words = procfile_linewords(ff, 0), w; + + // find how many CPUs are there + if(cpus == -1) { + cpus = 0; + for(w = 0; w < words ; w++) { + if(strncmp(procfile_lineword(ff, 0, w), "CPU", 3) == 0) + cpus++; + } + + if(cpus > MAX_INTERRUPT_CPUS) cpus = MAX_INTERRUPT_CPUS; + } + + if(!cpus) { + error("PLUGIN: PROC_INTERRUPTS: Cannot find the number of CPUs in /proc/interrupts"); + return 1; + } + + // allocate the size we need; + struct interrupt irrs[lines]; + irrs[0].used = 0; + + // loop through all lines + for(l = 1; l < lines ;l++) { + struct interrupt *irr = &irrs[l]; + irr->used = 0; + irr->total = 0; + + words = procfile_linewords(ff, l); + if(!words) continue; + + irr->id = procfile_lineword(ff, l, 0); + if(!irr->id || !irr->id[0]) continue; + + int idlen = strlen(irr->id); + if(irr->id[idlen - 1] == ':') + irr->id[idlen - 1] = '\0'; + + int c; + for(c = 0; c < cpus ;c++) { + if((c + 1) < (int)words) + irr->value[c] = strtoull(procfile_lineword(ff, l, (uint32_t)(c + 1)), NULL, 10); + else + irr->value[c] = 0; + + irr->total += irr->value[c]; + } + + if(isdigit(irr->id[0]) && (uint32_t)(cpus + 2) < words) { + strncpy(irr->name, procfile_lineword(ff, l, words - 1), MAX_INTERRUPT_NAME); + irr->name[MAX_INTERRUPT_NAME] = '\0'; + int nlen = strlen(irr->name); + if(nlen < (MAX_INTERRUPT_NAME-1)) { + irr->name[nlen] = '_'; + strncpy(&irr->name[nlen + 1], irr->id, MAX_INTERRUPT_NAME - nlen); + irr->name[MAX_INTERRUPT_NAME] = '\0'; + } + } + else { + strncpy(irr->name, irr->id, MAX_INTERRUPT_NAME); + irr->name[MAX_INTERRUPT_NAME] = '\0'; + } + + irr->used = 1; + } + + RRDSET *st; + + // -------------------------------------------------------------------- + + st = rrdset_find_bytype("system", "interrupts"); + if(!st) { + st = rrdset_create("system", "interrupts", NULL, "interrupts", NULL, "System interrupts", "interrupts/s", 1000, update_every, RRDSET_TYPE_STACKED); + + for(l = 0; l < lines ;l++) { + if(!irrs[l].used) continue; + rrddim_add(st, irrs[l].id, irrs[l].name, 1, 1, RRDDIM_INCREMENTAL); + } + } + else rrdset_next(st); + + for(l = 0; l < lines ;l++) { + if(!irrs[l].used) continue; + rrddim_set(st, irrs[l].id, irrs[l].total); + } + rrdset_done(st); + + if(do_per_core) { + int c; + + for(c = 0; c < cpus ; c++) { + char id[256]; + snprintf(id, 256, "cpu%d_interrupts", c); + + st = rrdset_find_bytype("cpu", id); + if(!st) { + char name[256], title[256]; + snprintf(name, 256, "cpu%d_interrupts", c); + snprintf(title, 256, "CPU%d Interrupts", c); + st = rrdset_create("cpu", id, name, "interrupts", "cpu.interrupts", title, "interrupts/s", 2000 + c, update_every, RRDSET_TYPE_STACKED); + + for(l = 0; l < lines ;l++) { + if(!irrs[l].used) continue; + rrddim_add(st, irrs[l].id, irrs[l].name, 1, 1, RRDDIM_INCREMENTAL); + } + } + else rrdset_next(st); + + for(l = 0; l < lines ;l++) { + if(!irrs[l].used) continue; + rrddim_set(st, irrs[l].id, irrs[l].value[c]); + } + rrdset_done(st); + } + } + + return 0; +} diff --git a/src/proc_loadavg.c b/src/proc_loadavg.c new file mode 100755 index 000000000..cd7edc832 --- /dev/null +++ b/src/proc_loadavg.c @@ -0,0 +1,89 @@ +#ifdef HAVE_CONFIG_H +#include <config.h> +#endif +#include <stdio.h> +#include <stdlib.h> +#include <string.h> + +#include "log.h" +#include "common.h" +#include "appconfig.h" +#include "procfile.h" +#include "rrd.h" +#include "plugin_proc.h" + +int do_proc_loadavg(int update_every, unsigned long long dt) { + static procfile *ff = NULL; + static int do_loadavg = -1, do_all_processes = -1; + + if(dt) {}; + + if(!ff) { + char filename[FILENAME_MAX + 1]; + snprintf(filename, FILENAME_MAX, "%s%s", global_host_prefix, "/proc/loadavg"); + ff = procfile_open(config_get("plugin:proc:/proc/loadavg", "filename to monitor", filename), " \t,:|/", PROCFILE_FLAG_DEFAULT); + } + if(!ff) return 1; + + ff = procfile_readall(ff); + if(!ff) return 0; // we return 0, so that we will retry to open it next time + + if(do_loadavg == -1) do_loadavg = config_get_boolean("plugin:proc:/proc/loadavg", "enable load average", 1); + if(do_all_processes == -1) do_all_processes = config_get_boolean("plugin:proc:/proc/loadavg", "enable total processes", 1); + + if(procfile_lines(ff) < 1) { + error("/proc/loadavg has no lines."); + return 1; + } + if(procfile_linewords(ff, 0) < 6) { + error("/proc/loadavg has less than 6 words in it."); + return 1; + } + + double load1 = strtod(procfile_lineword(ff, 0, 0), NULL); + double load5 = strtod(procfile_lineword(ff, 0, 1), NULL); + double load15 = strtod(procfile_lineword(ff, 0, 2), NULL); + + //unsigned long long running_processes = strtoull(procfile_lineword(ff, 0, 3), NULL, 10); + unsigned long long active_processes = strtoull(procfile_lineword(ff, 0, 4), NULL, 10); + //unsigned long long next_pid = strtoull(procfile_lineword(ff, 0, 5), NULL, 10); + + + RRDSET *st; + + // -------------------------------------------------------------------- + + if(do_loadavg) { + st = rrdset_find_byname("system.load"); + if(!st) { + st = rrdset_create("system", "load", NULL, "load", NULL, "System Load Average", "load", 100, update_every, RRDSET_TYPE_LINE); + + rrddim_add(st, "load1", NULL, 1, 1000, RRDDIM_ABSOLUTE); + rrddim_add(st, "load5", NULL, 1, 1000, RRDDIM_ABSOLUTE); + rrddim_add(st, "load15", NULL, 1, 1000, RRDDIM_ABSOLUTE); + } + else rrdset_next(st); + + rrddim_set(st, "load1", load1 * 1000); + rrddim_set(st, "load5", load5 * 1000); + rrddim_set(st, "load15", load15 * 1000); + rrdset_done(st); + } + + // -------------------------------------------------------------------- + + if(do_all_processes) { + st = rrdset_find_byname("system.active_processes"); + if(!st) { + st = rrdset_create("system", "active_processes", NULL, "processes", NULL, "System Active Processes", "processes", 750, update_every, RRDSET_TYPE_LINE); + + rrddim_add(st, "active", NULL, 1, 1, RRDDIM_ABSOLUTE); + } + else rrdset_next(st); + + rrddim_set(st, "active", active_processes); + rrdset_done(st); + } + + return 0; +} diff --git a/src/proc_meminfo.c b/src/proc_meminfo.c new file mode 100755 index 000000000..dbd43369f --- /dev/null +++ b/src/proc_meminfo.c @@ -0,0 +1,254 @@ +#ifdef HAVE_CONFIG_H +#include <config.h> +#endif +#include <stdio.h> +#include <stdlib.h> +#include <string.h> + +#include "common.h" +#include "log.h" +#include "appconfig.h" +#include "procfile.h" +#include "rrd.h" +#include "plugin_proc.h" + +#define MAX_PROC_MEMINFO_LINE 4096 +#define MAX_PROC_MEMINFO_NAME 1024 + +int do_proc_meminfo(int update_every, unsigned long long dt) { + static procfile *ff = NULL; + + static int do_ram = -1, do_swap = -1, do_hwcorrupt = -1, do_committed = -1, do_writeback = -1, do_kernel = -1, do_slab = -1; + + if(do_ram == -1) do_ram = config_get_boolean("plugin:proc:/proc/meminfo", "system ram", 1); + if(do_swap == -1) do_swap = config_get_boolean("plugin:proc:/proc/meminfo", "system swap", 1); + if(do_hwcorrupt == -1) do_hwcorrupt = config_get_boolean_ondemand("plugin:proc:/proc/meminfo", "hardware corrupted ECC", CONFIG_ONDEMAND_ONDEMAND); + if(do_committed == -1) do_committed = config_get_boolean("plugin:proc:/proc/meminfo", "committed memory", 1); + if(do_writeback == -1) do_writeback = config_get_boolean("plugin:proc:/proc/meminfo", "writeback memory", 1); + if(do_kernel == -1) do_kernel = config_get_boolean("plugin:proc:/proc/meminfo", "kernel memory", 1); + if(do_slab == -1) do_slab = config_get_boolean("plugin:proc:/proc/meminfo", "slab memory", 1); + + if(dt) {}; + + if(!ff) { + char filename[FILENAME_MAX + 1]; + snprintf(filename, FILENAME_MAX, "%s%s", global_host_prefix, "/proc/meminfo"); + ff = procfile_open(config_get("plugin:proc:/proc/meminfo", "filename to monitor", filename), " \t:", PROCFILE_FLAG_DEFAULT); + } + if(!ff) return 1; + + ff = procfile_readall(ff); + if(!ff) return 0; // we return 0, so that we will retry to open it next time + + uint32_t lines = procfile_lines(ff), l; + uint32_t words; + + int hwcorrupted = 0; + + unsigned long long MemTotal = 0, MemFree = 0, Buffers = 0, Cached = 0, SwapCached = 0, + Active = 0, Inactive = 0, ActiveAnon = 0, InactiveAnon = 0, ActiveFile = 0, InactiveFile = 0, + Unevictable = 0, Mlocked = 0, SwapTotal = 0, SwapFree = 0, Dirty = 0, Writeback = 0, AnonPages = 0, + Mapped = 0, Shmem = 0, Slab = 0, SReclaimable = 0, SUnreclaim = 0, KernelStack = 0, PageTables = 0, + NFS_Unstable = 0, Bounce = 0, WritebackTmp = 0, CommitLimit = 0, Committed_AS = 0, + VmallocTotal = 0, VmallocUsed = 0, VmallocChunk = 0, + AnonHugePages = 0, HugePages_Total = 0, HugePages_Free = 0, HugePages_Rsvd = 0, HugePages_Surp = 0, Hugepagesize = 0, + DirectMap4k = 0, DirectMap2M = 0, HardwareCorrupted = 0; + + for(l = 0; l < lines ;l++) { + words = procfile_linewords(ff, l); + if(words < 2) continue; + + char *name = procfile_lineword(ff, l, 0); + unsigned long long value = strtoull(procfile_lineword(ff, l, 1), NULL, 10); + + if(!MemTotal && strcmp(name, "MemTotal") == 0) MemTotal = value; + else if(!MemFree && strcmp(name, "MemFree") == 0) MemFree = value; + else if(!Buffers && strcmp(name, "Buffers") == 0) Buffers = value; + else if(!Cached && strcmp(name, "Cached") == 0) Cached = value; + else if(!SwapCached && strcmp(name, "SwapCached") == 0) SwapCached = value; + else if(!Active && strcmp(name, "Active") == 0) Active = value; + else if(!Inactive && strcmp(name, "Inactive") == 0) Inactive = value; + else if(!ActiveAnon && strcmp(name, "ActiveAnon") == 0) ActiveAnon = value; + else if(!InactiveAnon && strcmp(name, "InactiveAnon") == 0) InactiveAnon = value; + else if(!ActiveFile && strcmp(name, "ActiveFile") == 0) ActiveFile = value; + else if(!InactiveFile && strcmp(name, "InactiveFile") == 0) InactiveFile = value; + else if(!Unevictable && strcmp(name, "Unevictable") == 0) Unevictable = value; + else if(!Mlocked && strcmp(name, "Mlocked") == 0) Mlocked = value; + else if(!SwapTotal && strcmp(name, "SwapTotal") == 0) SwapTotal = value; + else if(!SwapFree && strcmp(name, "SwapFree") == 0) SwapFree = value; + else if(!Dirty && strcmp(name, "Dirty") == 0) Dirty = value; + else if(!Writeback && strcmp(name, "Writeback") == 0) Writeback = value; + else if(!AnonPages && strcmp(name, "AnonPages") == 0) AnonPages = value; + else if(!Mapped && strcmp(name, "Mapped") == 0) Mapped = value; + else if(!Shmem && strcmp(name, "Shmem") == 0) Shmem = value; + else if(!Slab && strcmp(name, "Slab") == 0) Slab = value; + else if(!SReclaimable && strcmp(name, "SReclaimable") == 0) SReclaimable = value; + else if(!SUnreclaim && strcmp(name, "SUnreclaim") == 0) SUnreclaim = value; + else if(!KernelStack && strcmp(name, "KernelStack") == 0) KernelStack = value; + else if(!PageTables && strcmp(name, "PageTables") == 0) PageTables = value; + else if(!NFS_Unstable && strcmp(name, "NFS_Unstable") == 0) NFS_Unstable = value; + else if(!Bounce && strcmp(name, "Bounce") == 0) Bounce = value; + else if(!WritebackTmp && strcmp(name, "WritebackTmp") == 0) WritebackTmp = value; + else if(!CommitLimit && strcmp(name, "CommitLimit") == 0) CommitLimit = value; + else if(!Committed_AS && strcmp(name, "Committed_AS") == 0) Committed_AS = value; + else if(!VmallocTotal && strcmp(name, "VmallocTotal") == 0) VmallocTotal = value; + else if(!VmallocUsed && strcmp(name, "VmallocUsed") == 0) VmallocUsed = value; + else if(!VmallocChunk && strcmp(name, "VmallocChunk") == 0) VmallocChunk = value; + else if(!HardwareCorrupted && strcmp(name, "HardwareCorrupted") == 0) { HardwareCorrupted = value; hwcorrupted = 1; } + else if(!AnonHugePages && strcmp(name, "AnonHugePages") == 0) AnonHugePages = value; + else if(!HugePages_Total && strcmp(name, "HugePages_Total") == 0) HugePages_Total = value; + else if(!HugePages_Free && strcmp(name, "HugePages_Free") == 0) HugePages_Free = value; + else if(!HugePages_Rsvd && strcmp(name, "HugePages_Rsvd") == 0) HugePages_Rsvd = value; + else if(!HugePages_Surp && strcmp(name, "HugePages_Surp") == 0) HugePages_Surp = value; + else if(!Hugepagesize && strcmp(name, "Hugepagesize") == 0) Hugepagesize = value; + else if(!DirectMap4k && strcmp(name, "DirectMap4k") == 0) DirectMap4k = value; + else if(!DirectMap2M && strcmp(name, "DirectMap2M") == 0) DirectMap2M = value; + } + + RRDSET *st; + + // -------------------------------------------------------------------- + + // http://stackoverflow.com/questions/3019748/how-to-reliably-measure-available-memory-in-linux + unsigned long long MemUsed = MemTotal - MemFree - Cached - Buffers; + + if(do_ram) { + st = rrdset_find("system.ram"); + if(!st) { + st = rrdset_create("system", "ram", NULL, "ram", NULL, "System RAM", "MB", 200, update_every, RRDSET_TYPE_STACKED); + + rrddim_add(st, "buffers", NULL, 1, 1024, RRDDIM_ABSOLUTE); + rrddim_add(st, "used", NULL, 1, 1024, RRDDIM_ABSOLUTE); + rrddim_add(st, "cached", NULL, 1, 1024, RRDDIM_ABSOLUTE); + rrddim_add(st, "free", NULL, 1, 1024, RRDDIM_ABSOLUTE); + } + else rrdset_next(st); + + rrddim_set(st, "used", MemUsed); + rrddim_set(st, "free", MemFree); + rrddim_set(st, "cached", Cached); + rrddim_set(st, "buffers", Buffers); + rrdset_done(st); + } + + // -------------------------------------------------------------------- + + unsigned long long SwapUsed = SwapTotal - SwapFree; + + if(do_swap) { + st = rrdset_find("system.swap"); + if(!st) { + st = rrdset_create("system", "swap", NULL, "swap", NULL, "System Swap", "MB", 201, update_every, RRDSET_TYPE_STACKED); + st->isdetail = 1; + + rrddim_add(st, "free", NULL, 1, 1024, RRDDIM_ABSOLUTE); + rrddim_add(st, "used", NULL, 1, 1024, RRDDIM_ABSOLUTE); + } + else rrdset_next(st); + + rrddim_set(st, "used", SwapUsed); + rrddim_set(st, "free", SwapFree); + rrdset_done(st); + } + + // -------------------------------------------------------------------- + + if(hwcorrupted && do_hwcorrupt && HardwareCorrupted > 0) { + do_hwcorrupt = CONFIG_ONDEMAND_YES; + + st = rrdset_find("mem.hwcorrupt"); + if(!st) { + st = rrdset_create("mem", "hwcorrupt", NULL, "errors", NULL, "Hardware Corrupted ECC", "MB", 9000, update_every, RRDSET_TYPE_LINE); + st->isdetail = 1; + + rrddim_add(st, "HardwareCorrupted", NULL, 1, 1024, RRDDIM_ABSOLUTE); + } + else rrdset_next(st); + + rrddim_set(st, "HardwareCorrupted", HardwareCorrupted); + rrdset_done(st); + } + + // -------------------------------------------------------------------- + + if(do_committed) { + st = rrdset_find("mem.committed"); + if(!st) { + st = rrdset_create("mem", "committed", NULL, "system", NULL, "Committed (Allocated) Memory", "MB", 5000, update_every, RRDSET_TYPE_AREA); + st->isdetail = 1; + + rrddim_add(st, "Committed_AS", NULL, 1, 1024, RRDDIM_ABSOLUTE); + } + else rrdset_next(st); + + rrddim_set(st, "Committed_AS", Committed_AS); + rrdset_done(st); + } + + // -------------------------------------------------------------------- + + if(do_writeback) { + st = rrdset_find("mem.writeback"); + if(!st) { + st = rrdset_create("mem", "writeback", NULL, "kernel", NULL, "Writeback Memory", "MB", 4000, update_every, RRDSET_TYPE_LINE); + st->isdetail = 1; + + rrddim_add(st, "Dirty", NULL, 1, 1024, RRDDIM_ABSOLUTE); + rrddim_add(st, "Writeback", NULL, 1, 1024, RRDDIM_ABSOLUTE); + rrddim_add(st, "FuseWriteback", NULL, 1, 1024, RRDDIM_ABSOLUTE); + rrddim_add(st, "NfsWriteback", NULL, 1, 1024, RRDDIM_ABSOLUTE); + rrddim_add(st, "Bounce", NULL, 1, 1024, RRDDIM_ABSOLUTE); + } + else rrdset_next(st); + + rrddim_set(st, "Dirty", Dirty); + rrddim_set(st, "Writeback", Writeback); + rrddim_set(st, "FuseWriteback", WritebackTmp); + rrddim_set(st, "NfsWriteback", NFS_Unstable); + rrddim_set(st, "Bounce", Bounce); + rrdset_done(st); + } + + // -------------------------------------------------------------------- + + if(do_kernel) { + st = rrdset_find("mem.kernel"); + if(!st) { + st = rrdset_create("mem", "kernel", NULL, "kernel", NULL, "Memory Used by Kernel", "MB", 6000, update_every, RRDSET_TYPE_STACKED); + st->isdetail = 1; + + rrddim_add(st, "Slab", NULL, 1, 1024, RRDDIM_ABSOLUTE); + rrddim_add(st, "KernelStack", NULL, 1, 1024, RRDDIM_ABSOLUTE); + rrddim_add(st, "PageTables", NULL, 1, 1024, RRDDIM_ABSOLUTE); + rrddim_add(st, "VmallocUsed", NULL, 1, 1024, RRDDIM_ABSOLUTE); + } + else rrdset_next(st); + + rrddim_set(st, "KernelStack", KernelStack); + rrddim_set(st, "Slab", Slab); + rrddim_set(st, "PageTables", PageTables); + rrddim_set(st, "VmallocUsed", VmallocUsed); + rrdset_done(st); + } + + // -------------------------------------------------------------------- + + if(do_slab) { + st = rrdset_find("mem.slab"); + if(!st) { + st = rrdset_create("mem", "slab", NULL, "slab", NULL, "Reclaimable Kernel Memory", "MB", 6500, update_every, RRDSET_TYPE_STACKED); + st->isdetail = 1; + + rrddim_add(st, "reclaimable", NULL, 1, 1024, RRDDIM_ABSOLUTE); + rrddim_add(st, "unreclaimable", NULL, 1, 1024, RRDDIM_ABSOLUTE); + } + else rrdset_next(st); + + rrddim_set(st, "reclaimable", SReclaimable); + rrddim_set(st, "unreclaimable", SUnreclaim); + rrdset_done(st); + } + + return 0; +} + diff --git a/src/proc_net_dev.c b/src/proc_net_dev.c new file mode 100755 index 000000000..5070ab817 --- /dev/null +++ b/src/proc_net_dev.c @@ -0,0 +1,247 @@ +#ifdef HAVE_CONFIG_H +#include <config.h> +#endif +#include <stdio.h> +#include <stdlib.h> +#include <string.h> + +#include "common.h" +#include "appconfig.h" +#include "procfile.h" +#include "rrd.h" +#include "plugin_proc.h" + +int do_proc_net_dev(int update_every, unsigned long long dt) { + static procfile *ff = NULL; + static int enable_new_interfaces = -1, enable_ifb_interfaces = -1; + static int do_bandwidth = -1, do_packets = -1, do_errors = -1, do_drops = -1, do_fifo = -1, do_compressed = -1, do_events = -1; + + if(dt) {}; + + if(!ff) { + char filename[FILENAME_MAX + 1]; + snprintf(filename, FILENAME_MAX, "%s%s", global_host_prefix, "/proc/net/dev"); + ff = procfile_open(config_get("plugin:proc:/proc/net/dev", "filename to monitor", filename), " \t,:|", PROCFILE_FLAG_DEFAULT); + } + if(!ff) return 1; + + ff = procfile_readall(ff); + if(!ff) return 0; // we return 0, so that we will retry to open it next time + + if(enable_new_interfaces == -1) enable_new_interfaces = config_get_boolean_ondemand("plugin:proc:/proc/net/dev", "enable new interfaces detected at runtime", CONFIG_ONDEMAND_ONDEMAND); + if(enable_ifb_interfaces == -1) enable_ifb_interfaces = config_get_boolean_ondemand("plugin:proc:/proc/net/dev", "enable ifb interfaces", CONFIG_ONDEMAND_NO); + + if(do_bandwidth == -1) do_bandwidth = config_get_boolean_ondemand("plugin:proc:/proc/net/dev", "bandwidth for all interfaces", CONFIG_ONDEMAND_ONDEMAND); + if(do_packets == -1) do_packets = config_get_boolean_ondemand("plugin:proc:/proc/net/dev", "packets for all interfaces", CONFIG_ONDEMAND_ONDEMAND); + if(do_errors == -1) do_errors = config_get_boolean_ondemand("plugin:proc:/proc/net/dev", "errors for all interfaces", CONFIG_ONDEMAND_ONDEMAND); + if(do_drops == -1) do_drops = config_get_boolean_ondemand("plugin:proc:/proc/net/dev", "drops for all interfaces", CONFIG_ONDEMAND_ONDEMAND); + if(do_fifo == -1) do_fifo = config_get_boolean_ondemand("plugin:proc:/proc/net/dev", "fifo for all interfaces", CONFIG_ONDEMAND_ONDEMAND); + if(do_compressed == -1) do_compressed = config_get_boolean_ondemand("plugin:proc:/proc/net/dev", "compressed packets for all interfaces", CONFIG_ONDEMAND_ONDEMAND); + if(do_events == -1) do_events = config_get_boolean_ondemand("plugin:proc:/proc/net/dev", "frames, collisions, carrier counters for all interfaces", CONFIG_ONDEMAND_ONDEMAND); + + uint32_t lines = procfile_lines(ff), l; + uint32_t words; + + char *iface; + unsigned long long rbytes, rpackets, rerrors, rdrops, rfifo, rframe, rcompressed, rmulticast; + unsigned long long tbytes, tpackets, terrors, tdrops, tfifo, tcollisions, tcarrier, tcompressed; + + for(l = 2; l < lines ;l++) { + words = procfile_linewords(ff, l); + if(words < 17) continue; + + iface = procfile_lineword(ff, l, 0); + + rbytes = strtoull(procfile_lineword(ff, l, 1), NULL, 10); + rpackets = strtoull(procfile_lineword(ff, l, 2), NULL, 10); + rerrors = strtoull(procfile_lineword(ff, l, 3), NULL, 10); + rdrops = strtoull(procfile_lineword(ff, l, 4), NULL, 10); + rfifo = strtoull(procfile_lineword(ff, l, 5), NULL, 10); + rframe = strtoull(procfile_lineword(ff, l, 6), NULL, 10); + rcompressed = strtoull(procfile_lineword(ff, l, 7), NULL, 10); + rmulticast = strtoull(procfile_lineword(ff, l, 8), NULL, 10); + + tbytes = strtoull(procfile_lineword(ff, l, 9), NULL, 10); + tpackets = strtoull(procfile_lineword(ff, l, 10), NULL, 10); + terrors = strtoull(procfile_lineword(ff, l, 11), NULL, 10); + tdrops = strtoull(procfile_lineword(ff, l, 12), NULL, 10); + tfifo = strtoull(procfile_lineword(ff, l, 13), NULL, 10); + tcollisions = strtoull(procfile_lineword(ff, l, 14), NULL, 10); + tcarrier = strtoull(procfile_lineword(ff, l, 15), NULL, 10); + tcompressed = strtoull(procfile_lineword(ff, l, 16), NULL, 10); + + int ddo_bandwidth = do_bandwidth, ddo_packets = do_packets, ddo_errors = do_errors, ddo_drops = do_drops, ddo_fifo = do_fifo, ddo_compressed = do_compressed, ddo_events = do_events; + + int default_enable = enable_new_interfaces; + + // prevent unused interfaces from creating charts + if(strcmp(iface, "lo") == 0) + default_enable = 0; + else { + int len = strlen(iface); + if(len >= 4 && strcmp(&iface[len-4], "-ifb") == 0) + default_enable = enable_ifb_interfaces; + } + + // check if the user wants it + { + char var_name[512 + 1]; + snprintf(var_name, 512, "plugin:proc:/proc/net/dev:%s", iface); + default_enable = config_get_boolean_ondemand(var_name, "enabled", default_enable); + if(default_enable == CONFIG_ONDEMAND_NO) continue; + if(default_enable == CONFIG_ONDEMAND_ONDEMAND && !rbytes && !tbytes) continue; + + ddo_bandwidth = config_get_boolean_ondemand(var_name, "bandwidth", ddo_bandwidth); + ddo_packets = config_get_boolean_ondemand(var_name, "packets", ddo_packets); + ddo_errors = config_get_boolean_ondemand(var_name, "errors", ddo_errors); + ddo_drops = config_get_boolean_ondemand(var_name, "drops", ddo_drops); + ddo_fifo = config_get_boolean_ondemand(var_name, "fifo", ddo_fifo); + ddo_compressed = config_get_boolean_ondemand(var_name, "compressed", ddo_compressed); + ddo_events = config_get_boolean_ondemand(var_name, "events", ddo_events); + + if(ddo_bandwidth == CONFIG_ONDEMAND_ONDEMAND && rbytes == 0 && tbytes == 0) ddo_bandwidth = 0; + if(ddo_errors == CONFIG_ONDEMAND_ONDEMAND && rerrors == 0 && terrors == 0) ddo_errors = 0; + if(ddo_drops == CONFIG_ONDEMAND_ONDEMAND && rdrops == 0 && tdrops == 0) ddo_drops = 0; + if(ddo_fifo == CONFIG_ONDEMAND_ONDEMAND && rfifo == 0 && tfifo == 0) ddo_fifo = 0; + if(ddo_compressed == CONFIG_ONDEMAND_ONDEMAND && rcompressed == 0 && tcompressed == 0) ddo_compressed = 0; + if(ddo_events == CONFIG_ONDEMAND_ONDEMAND && rframe == 0 && tcollisions == 0 && tcarrier == 0) ddo_events = 0; + + // for absolute values, we need to switch the setting to 'yes' + // to allow it refresh from now on + // if(ddo_fifo == CONFIG_ONDEMAND_ONDEMAND) config_set(var_name, "fifo", "yes"); + } + + RRDSET *st; + + // -------------------------------------------------------------------- + + if(ddo_bandwidth) { + st = rrdset_find_bytype("net", iface); + if(!st) { + st = rrdset_create("net", iface, NULL, iface, "net.net", "Bandwidth", "kilobits/s", 7000, update_every, RRDSET_TYPE_AREA); + + rrddim_add(st, "received", NULL, 8, 1024, RRDDIM_INCREMENTAL); + rrddim_add(st, "sent", NULL, -8, 1024, RRDDIM_INCREMENTAL); + } + else rrdset_next(st); + + rrddim_set(st, "received", rbytes); + rrddim_set(st, "sent", tbytes); + rrdset_done(st); + } + + // -------------------------------------------------------------------- + + if(ddo_packets) { + st = rrdset_find_bytype("net_packets", iface); + if(!st) { + st = rrdset_create("net_packets", iface, NULL, iface, "net.packets", "Packets", "packets/s", 7001, update_every, RRDSET_TYPE_LINE); + st->isdetail = 1; + + rrddim_add(st, "received", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "sent", NULL, -1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "multicast", NULL, 1, 1, RRDDIM_INCREMENTAL); + } + else rrdset_next(st); + + rrddim_set(st, "received", rpackets); + rrddim_set(st, "sent", tpackets); + rrddim_set(st, "multicast", rmulticast); + rrdset_done(st); + } + + // -------------------------------------------------------------------- + + if(ddo_errors) { + st = rrdset_find_bytype("net_errors", iface); + if(!st) { + st = rrdset_create("net_errors", iface, NULL, iface, "net.errors", "Interface Errors", "errors/s", 7002, update_every, RRDSET_TYPE_LINE); + st->isdetail = 1; + + rrddim_add(st, "inbound", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "outbound", NULL, -1, 1, RRDDIM_INCREMENTAL); + } + else rrdset_next(st); + + rrddim_set(st, "inbound", rerrors); + rrddim_set(st, "outbound", terrors); + rrdset_done(st); + } + + // -------------------------------------------------------------------- + + if(ddo_drops) { + st = rrdset_find_bytype("net_drops", iface); + if(!st) { + st = rrdset_create("net_drops", iface, NULL, iface, "net.drops", "Interface Drops", "drops/s", 7003, update_every, RRDSET_TYPE_LINE); + st->isdetail = 1; + + rrddim_add(st, "inbound", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "outbound", NULL, -1, 1, RRDDIM_INCREMENTAL); + } + else rrdset_next(st); + + rrddim_set(st, "inbound", rdrops); + rrddim_set(st, "outbound", tdrops); + rrdset_done(st); + } + + // -------------------------------------------------------------------- + + if(ddo_fifo) { + st = rrdset_find_bytype("net_fifo", iface); + if(!st) { + st = rrdset_create("net_fifo", iface, NULL, iface, "net.fifo", "Interface FIFO Buffer Errors", "errors", 7004, update_every, RRDSET_TYPE_LINE); + st->isdetail = 1; + + rrddim_add(st, "receive", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "transmit", NULL, -1, 1, RRDDIM_INCREMENTAL); + } + else rrdset_next(st); + + rrddim_set(st, "receive", rfifo); + rrddim_set(st, "transmit", tfifo); + rrdset_done(st); + } + + // -------------------------------------------------------------------- + + if(ddo_compressed) { + st = rrdset_find_bytype("net_compressed", iface); + if(!st) { + st = rrdset_create("net_compressed", iface, NULL, iface, "net.compressed", "Compressed Packets", "packets/s", 7005, update_every, RRDSET_TYPE_LINE); + st->isdetail = 1; + + rrddim_add(st, "received", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "sent", NULL, -1, 1, RRDDIM_INCREMENTAL); + } + else rrdset_next(st); + + rrddim_set(st, "received", rcompressed); + rrddim_set(st, "sent", tcompressed); + rrdset_done(st); + } + + // -------------------------------------------------------------------- + + if(ddo_events) { + st = rrdset_find_bytype("net_events", iface); + if(!st) { + st = rrdset_create("net_events", iface, NULL, iface, "net.events", "Network Interface Events", "events/s", 7006, update_every, RRDSET_TYPE_LINE); + st->isdetail = 1; + + rrddim_add(st, "frames", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "collisions", NULL, -1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "carrier", NULL, -1, 1, RRDDIM_INCREMENTAL); + } + else rrdset_next(st); + + rrddim_set(st, "frames", rframe); + rrddim_set(st, "collisions", tcollisions); + rrddim_set(st, "carrier", tcarrier); + rrdset_done(st); + } + } + + return 0; +} diff --git a/src/proc_net_ip_vs_stats.c b/src/proc_net_ip_vs_stats.c new file mode 100755 index 000000000..8c2ece7d3 --- /dev/null +++ b/src/proc_net_ip_vs_stats.c @@ -0,0 +1,102 @@ +#ifdef HAVE_CONFIG_H +#include <config.h> +#endif +#include <stdio.h> +#include <stdlib.h> + +#include "common.h" +#include "appconfig.h" +#include "procfile.h" +#include "rrd.h" +#include "plugin_proc.h" + +#define RRD_TYPE_NET_IPVS "ipvs" +#define RRD_TYPE_NET_IPVS_LEN strlen(RRD_TYPE_NET_IPVS) + +int do_proc_net_ip_vs_stats(int update_every, unsigned long long dt) { + static int do_bandwidth = -1, do_sockets = -1, do_packets = -1; + static procfile *ff = NULL; + + if(do_bandwidth == -1) do_bandwidth = config_get_boolean("plugin:proc:/proc/net/ip_vs_stats", "IPVS bandwidth", 1); + if(do_sockets == -1) do_sockets = config_get_boolean("plugin:proc:/proc/net/ip_vs_stats", "IPVS connections", 1); + if(do_packets == -1) do_packets = config_get_boolean("plugin:proc:/proc/net/ip_vs_stats", "IPVS packets", 1); + + if(dt) {}; + + if(!ff) { + char filename[FILENAME_MAX + 1]; + snprintf(filename, FILENAME_MAX, "%s%s", global_host_prefix, "/proc/net/ip_vs_stats"); + ff = procfile_open(config_get("plugin:proc:/proc/net/ip_vs_stats", "filename to monitor", filename), " \t,:|", PROCFILE_FLAG_DEFAULT); + } + if(!ff) return 1; + + ff = procfile_readall(ff); + if(!ff) return 0; // we return 0, so that we will retry to open it next time + + // make sure we have 3 lines + if(procfile_lines(ff) < 3) return 1; + + // make sure we have 5 words on the 3rd line + if(procfile_linewords(ff, 2) < 5) return 1; + + unsigned long long entries, InPackets, OutPackets, InBytes, OutBytes; + + entries = strtoull(procfile_lineword(ff, 2, 0), NULL, 16); + InPackets = strtoull(procfile_lineword(ff, 2, 1), NULL, 16); + OutPackets = strtoull(procfile_lineword(ff, 2, 2), NULL, 16); + InBytes = strtoull(procfile_lineword(ff, 2, 3), NULL, 16); + OutBytes = strtoull(procfile_lineword(ff, 2, 4), NULL, 16); + + RRDSET *st; + + // -------------------------------------------------------------------- + + if(do_sockets) { + st = rrdset_find(RRD_TYPE_NET_IPVS ".sockets"); + if(!st) { + st = rrdset_create(RRD_TYPE_NET_IPVS, "sockets", NULL, RRD_TYPE_NET_IPVS, NULL, "IPVS New Connections", "connections/s", 1001, update_every, RRDSET_TYPE_LINE); + + rrddim_add(st, "connections", NULL, 1, 1, RRDDIM_INCREMENTAL); + } + else rrdset_next(st); + + rrddim_set(st, "connections", entries); + rrdset_done(st); + } + + // -------------------------------------------------------------------- + + if(do_packets) { + st = rrdset_find(RRD_TYPE_NET_IPVS ".packets"); + if(!st) { + st = rrdset_create(RRD_TYPE_NET_IPVS, "packets", NULL, RRD_TYPE_NET_IPVS, NULL, "IPVS Packets", "packets/s", 1002, update_every, RRDSET_TYPE_LINE); + + rrddim_add(st, "received", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "sent", NULL, -1, 1, RRDDIM_INCREMENTAL); + } + else rrdset_next(st); + + rrddim_set(st, "received", InPackets); + rrddim_set(st, "sent", OutPackets); + rrdset_done(st); + } + + // -------------------------------------------------------------------- + + if(do_bandwidth) { + st = rrdset_find(RRD_TYPE_NET_IPVS ".net"); + if(!st) { + st = rrdset_create(RRD_TYPE_NET_IPVS, "net", NULL, RRD_TYPE_NET_IPVS, NULL, "IPVS Bandwidth", "kilobits/s", 1000, update_every, RRDSET_TYPE_AREA); + + rrddim_add(st, "received", NULL, 8, 1024, RRDDIM_INCREMENTAL); + rrddim_add(st, "sent", NULL, -8, 1024, RRDDIM_INCREMENTAL); + } + else rrdset_next(st); + + rrddim_set(st, "received", InBytes); + rrddim_set(st, "sent", OutBytes); + rrdset_done(st); + } + + return 0; +} diff --git a/src/proc_net_netstat.c b/src/proc_net_netstat.c new file mode 100755 index 000000000..3f178a4a3 --- /dev/null +++ b/src/proc_net_netstat.c @@ -0,0 +1,185 @@ +#ifdef HAVE_CONFIG_H +#include <config.h> +#endif +#include <stdio.h> +#include <stdlib.h> +#include <string.h> + +#include "common.h" +#include "log.h" +#include "appconfig.h" +#include "procfile.h" +#include "rrd.h" +#include "plugin_proc.h" + +int do_proc_net_netstat(int update_every, unsigned long long dt) { + static int do_bandwidth = -1, do_inerrors = -1, do_mcast = -1, do_bcast = -1, do_mcast_p = -1, do_bcast_p = -1; + static procfile *ff = NULL; + + if(do_bandwidth == -1) do_bandwidth = config_get_boolean("plugin:proc:/proc/net/netstat", "bandwidth", 1); + if(do_inerrors == -1) do_inerrors = config_get_boolean("plugin:proc:/proc/net/netstat", "input errors", 1); + if(do_mcast == -1) do_mcast = config_get_boolean("plugin:proc:/proc/net/netstat", "multicast bandwidth", 1); + if(do_bcast == -1) do_bcast = config_get_boolean("plugin:proc:/proc/net/netstat", "broadcast bandwidth", 1); + if(do_mcast_p == -1) do_mcast_p = config_get_boolean("plugin:proc:/proc/net/netstat", "multicast packets", 1); + if(do_bcast_p == -1) do_bcast_p = config_get_boolean("plugin:proc:/proc/net/netstat", "broadcast packets", 1); + + if(dt) {}; + + if(!ff) { + char filename[FILENAME_MAX + 1]; + snprintf(filename, FILENAME_MAX, "%s%s", global_host_prefix, "/proc/net/netstat"); + ff = procfile_open(config_get("plugin:proc:/proc/net/netstat", "filename to monitor", filename), " \t:", PROCFILE_FLAG_DEFAULT); + } + if(!ff) return 1; + + ff = procfile_readall(ff); + if(!ff) return 0; // we return 0, so that we will retry to open it next time + + uint32_t lines = procfile_lines(ff), l; + uint32_t words; + + for(l = 0; l < lines ;l++) { + if(strcmp(procfile_lineword(ff, l, 0), "IpExt") == 0) { + l++; // we need the next line + + if(strcmp(procfile_lineword(ff, l, 0), "IpExt") != 0) { + error("Cannot read IpExt line from /proc/net/netstat."); + break; + } + words = procfile_linewords(ff, l); + if(words < 12) { + error("Cannot read /proc/net/netstat IpExt line. Expected 12 params, read %d.", words); + continue; + } + + unsigned long long + InNoRoutes = 0, InTruncatedPkts = 0, + InOctets = 0, InMcastPkts = 0, InBcastPkts = 0, InMcastOctets = 0, InBcastOctets = 0, + OutOctets = 0, OutMcastPkts = 0, OutBcastPkts = 0, OutMcastOctets = 0, OutBcastOctets = 0; + + InNoRoutes = strtoull(procfile_lineword(ff, l, 1), NULL, 10); + InTruncatedPkts = strtoull(procfile_lineword(ff, l, 2), NULL, 10); + InMcastPkts = strtoull(procfile_lineword(ff, l, 3), NULL, 10); + OutMcastPkts = strtoull(procfile_lineword(ff, l, 4), NULL, 10); + InBcastPkts = strtoull(procfile_lineword(ff, l, 5), NULL, 10); + OutBcastPkts = strtoull(procfile_lineword(ff, l, 6), NULL, 10); + InOctets = strtoull(procfile_lineword(ff, l, 7), NULL, 10); + OutOctets = strtoull(procfile_lineword(ff, l, 8), NULL, 10); + InMcastOctets = strtoull(procfile_lineword(ff, l, 9), NULL, 10); + OutMcastOctets = strtoull(procfile_lineword(ff, l, 10), NULL, 10); + InBcastOctets = strtoull(procfile_lineword(ff, l, 11), NULL, 10); + OutBcastOctets = strtoull(procfile_lineword(ff, l, 12), NULL, 10); + + RRDSET *st; + + // -------------------------------------------------------------------- + + if(do_bandwidth) { + st = rrdset_find("system.ipv4"); + if(!st) { + st = rrdset_create("system", "ipv4", NULL, "network", NULL, "IPv4 Bandwidth", "kilobits/s", 500, update_every, RRDSET_TYPE_AREA); + + rrddim_add(st, "received", NULL, 8, 1024, RRDDIM_INCREMENTAL); + rrddim_add(st, "sent", NULL, -8, 1024, RRDDIM_INCREMENTAL); + } + else rrdset_next(st); + + rrddim_set(st, "sent", OutOctets); + rrddim_set(st, "received", InOctets); + rrdset_done(st); + } + + // -------------------------------------------------------------------- + + if(do_inerrors) { + st = rrdset_find("ipv4.inerrors"); + if(!st) { + st = rrdset_create("ipv4", "inerrors", NULL, "errors", NULL, "IPv4 Input Errors", "packets/s", 4000, update_every, RRDSET_TYPE_LINE); + st->isdetail = 1; + + rrddim_add(st, "noroutes", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "trunkated", NULL, 1, 1, RRDDIM_INCREMENTAL); + } + else rrdset_next(st); + + rrddim_set(st, "noroutes", InNoRoutes); + rrddim_set(st, "trunkated", InTruncatedPkts); + rrdset_done(st); + } + + // -------------------------------------------------------------------- + + if(do_mcast) { + st = rrdset_find("ipv4.mcast"); + if(!st) { + st = rrdset_create("ipv4", "mcast", NULL, "multicast", NULL, "IPv4 Multicast Bandwidth", "kilobits/s", 9000, update_every, RRDSET_TYPE_AREA); + st->isdetail = 1; + + rrddim_add(st, "received", NULL, 8, 1024, RRDDIM_INCREMENTAL); + rrddim_add(st, "sent", NULL, -8, 1024, RRDDIM_INCREMENTAL); + } + else rrdset_next(st); + + rrddim_set(st, "sent", OutMcastOctets); + rrddim_set(st, "received", InMcastOctets); + rrdset_done(st); + } + + // -------------------------------------------------------------------- + + if(do_bcast) { + st = rrdset_find("ipv4.bcast"); + if(!st) { + st = rrdset_create("ipv4", "bcast", NULL, "broadcast", NULL, "IPv4 Broadcast Bandwidth", "kilobits/s", 8000, update_every, RRDSET_TYPE_AREA); + st->isdetail = 1; + + rrddim_add(st, "received", NULL, 8, 1024, RRDDIM_INCREMENTAL); + rrddim_add(st, "sent", NULL, -8, 1024, RRDDIM_INCREMENTAL); + } + else rrdset_next(st); + + rrddim_set(st, "sent", OutBcastOctets); + rrddim_set(st, "received", InBcastOctets); + rrdset_done(st); + } + + // -------------------------------------------------------------------- + + if(do_mcast_p) { + st = rrdset_find("ipv4.mcastpkts"); + if(!st) { + st = rrdset_create("ipv4", "mcastpkts", NULL, "multicast", NULL, "IPv4 Multicast Packets", "packets/s", 9500, update_every, RRDSET_TYPE_LINE); + st->isdetail = 1; + + rrddim_add(st, "received", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "sent", NULL, -1, 1, RRDDIM_INCREMENTAL); + } + else rrdset_next(st); + + rrddim_set(st, "sent", OutMcastPkts); + rrddim_set(st, "received", InMcastPkts); + rrdset_done(st); + } + + // -------------------------------------------------------------------- + + if(do_bcast_p) { + st = rrdset_find("ipv4.bcastpkts"); + if(!st) { + st = rrdset_create("ipv4", "bcastpkts", NULL, "broadcast", NULL, "IPv4 Broadcast Packets", "packets/s", 8500, update_every, RRDSET_TYPE_LINE); + st->isdetail = 1; + + rrddim_add(st, "received", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "sent", NULL, -1, 1, RRDDIM_INCREMENTAL); + } + else rrdset_next(st); + + rrddim_set(st, "sent", OutBcastPkts); + rrddim_set(st, "received", InBcastPkts); + rrdset_done(st); + } + } + } + + return 0; +} diff --git a/src/proc_net_rpc_nfsd.c b/src/proc_net_rpc_nfsd.c new file mode 100644 index 000000000..12949f5d2 --- /dev/null +++ b/src/proc_net_rpc_nfsd.c @@ -0,0 +1,693 @@ +#ifdef HAVE_CONFIG_H +#include <config.h> +#endif +#include <stdio.h> +#include <stdlib.h> +#include <string.h> + +#include "common.h" +#include "log.h" +#include "appconfig.h" +#include "procfile.h" +#include "rrd.h" +#include "plugin_proc.h" + +struct nfsd_procs { + char name[30]; + unsigned long long proc2; + unsigned long long proc3; + unsigned long long proc4; + int present2; + int present3; + int present4; +}; + +struct nfsd_procs nfsd_proc_values[] = { + { "null", 0ULL, 0ULL, 0ULL, 0, 0, 0 }, + { "getattr", 0ULL, 0ULL, 0ULL, 0, 0, 0 }, + { "setattr", 0ULL, 0ULL, 0ULL, 0, 0, 0 }, + { "lookup", 0ULL, 0ULL, 0ULL, 0, 0, 0 }, + { "access", 0ULL, 0ULL, 0ULL, 0, 0, 0 }, + { "readlink", 0ULL, 0ULL, 0ULL, 0, 0, 0 }, + { "read", 0ULL, 0ULL, 0ULL, 0, 0, 0 }, + { "write", 0ULL, 0ULL, 0ULL, 0, 0, 0 }, + { "create", 0ULL, 0ULL, 0ULL, 0, 0, 0 }, + { "mkdir", 0ULL, 0ULL, 0ULL, 0, 0, 0 }, + { "symlink", 0ULL, 0ULL, 0ULL, 0, 0, 0 }, + { "mknod", 0ULL, 0ULL, 0ULL, 0, 0, 0 }, + { "remove", 0ULL, 0ULL, 0ULL, 0, 0, 0 }, + { "rmdir", 0ULL, 0ULL, 0ULL, 0, 0, 0 }, + { "rename", 0ULL, 0ULL, 0ULL, 0, 0, 0 }, + { "link", 0ULL, 0ULL, 0ULL, 0, 0, 0 }, + { "readdir", 0ULL, 0ULL, 0ULL, 0, 0, 0 }, + { "readdirplus", 0ULL, 0ULL, 0ULL, 0, 0, 0 }, + { "fsstat", 0ULL, 0ULL, 0ULL, 0, 0, 0 }, + { "fsinfo", 0ULL, 0ULL, 0ULL, 0, 0, 0 }, + { "pathconf", 0ULL, 0ULL, 0ULL, 0, 0, 0 }, + { "commit", 0ULL, 0ULL, 0ULL, 0, 0, 0 }, + { "", 0ULL, 0ULL, 0ULL, 0, 0, 0 }, +}; + +struct nfsd4_ops { + char name[30]; + unsigned long long value; + int present; +}; + +struct nfsd4_ops nfsd4_ops_values[] = { + { "access", 0ULL, 0}, + { "close", 0ULL, 0}, + { "commit", 0ULL, 0}, + { "create", 0ULL, 0}, + { "delegpurge", 0ULL, 0}, + { "delegreturn", 0ULL, 0}, + { "getattr", 0ULL, 0}, + { "getfh", 0ULL, 0}, + { "link", 0ULL, 0}, + { "lock", 0ULL, 0}, + { "lockt", 0ULL, 0}, + { "locku", 0ULL, 0}, + { "lookup", 0ULL, 0}, + { "lookupp", 0ULL, 0}, + { "nverify", 0ULL, 0}, + { "open", 0ULL, 0}, + { "openattr", 0ULL, 0}, + { "open_confirm", 0ULL, 0}, + { "open_downgrade", 0ULL, 0}, + { "putfh", 0ULL, 0}, + { "putpubfh", 0ULL, 0}, + { "putrootfh", 0ULL, 0}, + { "read", 0ULL, 0}, + { "readdir", 0ULL, 0}, + { "readlink", 0ULL, 0}, + { "remove", 0ULL, 0}, + { "rename", 0ULL, 0}, + { "renew", 0ULL, 0}, + { "restorefh", 0ULL, 0}, + { "savefh", 0ULL, 0}, + { "secinfo", 0ULL, 0}, + { "setattr", 0ULL, 0}, + { "setclientid", 0ULL, 0}, + { "setclientid_confirm", 0ULL, 0}, + { "verify", 0ULL, 0}, + { "write", 0ULL, 0}, + { "release_lockowner", 0ULL, 0}, + + /* nfs41 */ + { "backchannel_ctl", 0ULL, 0}, + { "bind_conn_to_session", 0ULL, 0}, + { "exchange_id", 0ULL, 0}, + { "create_session", 0ULL, 0}, + { "destroy_session", 0ULL, 0}, + { "free_stateid", 0ULL, 0}, + { "get_dir_delegation", 0ULL, 0}, + { "getdeviceinfo", 0ULL, 0}, + { "getdevicelist", 0ULL, 0}, + { "layoutcommit", 0ULL, 0}, + { "layoutget", 0ULL, 0}, + { "layoutreturn", 0ULL, 0}, + { "secinfo_no_name", 0ULL, 0}, + { "sequence", 0ULL, 0}, + { "set_ssv", 0ULL, 0}, + { "test_stateid", 0ULL, 0}, + { "want_delegation", 0ULL, 0}, + { "destroy_clientid", 0ULL, 0}, + { "reclaim_complete", 0ULL, 0}, + + /* nfs42 */ + { "allocate", 0ULL, 0}, + { "copy", 0ULL, 0}, + { "copy_notify", 0ULL, 0}, + { "deallocate", 0ULL, 0}, + { "io_advise", 0ULL, 0}, + { "layouterror", 0ULL, 0}, + { "layoutstats", 0ULL, 0}, + { "offload_cancel", 0ULL, 0}, + { "offload_status", 0ULL, 0}, + { "read_plus", 0ULL, 0}, + { "seek", 0ULL, 0}, + { "write_same", 0ULL, 0}, + + /* termination */ + { "", 0ULL, 0 } +}; + + +int do_proc_net_rpc_nfsd(int update_every, unsigned long long dt) { + static procfile *ff = NULL; + static int do_rc = -1, do_fh = -1, do_io = -1, do_th = -1, do_ra = -1, do_net = -1, do_rpc = -1, do_proc2 = -1, do_proc3 = -1, do_proc4 = -1, do_proc4ops = -1; + static int ra_warning = 0, th_warning = 0, proc2_warning = 0, proc3_warning = 0, proc4_warning = 0, proc4ops_warning = 0; + + if(dt) {}; + + if(!ff) { + char filename[FILENAME_MAX + 1]; + snprintf(filename, FILENAME_MAX, "%s%s", global_host_prefix, "/proc/net/rpc/nfsd"); + ff = procfile_open(config_get("plugin:proc:/proc/net/rpc/nfsd", "filename to monitor", filename), " \t", PROCFILE_FLAG_DEFAULT); + } + if(!ff) return 1; + + ff = procfile_readall(ff); + if(!ff) return 0; // we return 0, so that we will retry to open it next time + + if(do_rc == -1) do_rc = config_get_boolean("plugin:proc:/proc/net/rpc/nfsd", "read cache", 1); + if(do_fh == -1) do_fh = config_get_boolean("plugin:proc:/proc/net/rpc/nfsd", "file handles", 1); + if(do_io == -1) do_io = config_get_boolean("plugin:proc:/proc/net/rpc/nfsd", "I/O", 1); + if(do_th == -1) do_th = config_get_boolean("plugin:proc:/proc/net/rpc/nfsd", "threads", 1); + if(do_ra == -1) do_ra = config_get_boolean("plugin:proc:/proc/net/rpc/nfsd", "read ahead", 1); + if(do_net == -1) do_net = config_get_boolean("plugin:proc:/proc/net/rpc/nfsd", "network", 1); + if(do_rpc == -1) do_rpc = config_get_boolean("plugin:proc:/proc/net/rpc/nfsd", "rpc", 1); + if(do_proc2 == -1) do_proc2 = config_get_boolean("plugin:proc:/proc/net/rpc/nfsd", "NFS v2 procedures", 1); + if(do_proc3 == -1) do_proc3 = config_get_boolean("plugin:proc:/proc/net/rpc/nfsd", "NFS v3 procedures", 1); + if(do_proc4 == -1) do_proc4 = config_get_boolean("plugin:proc:/proc/net/rpc/nfsd", "NFS v4 procedures", 1); + if(do_proc4ops == -1) do_proc4ops = config_get_boolean("plugin:proc:/proc/net/rpc/nfsd", "NFS v4 operations", 1); + + // if they are enabled, reset them to 1 + // later we do them =2 to avoid doing strcmp for all lines + if(do_rc) do_rc = 1; + if(do_fh) do_fh = 1; + if(do_io) do_io = 1; + if(do_th) do_th = 1; + if(do_ra) do_ra = 1; + if(do_net) do_net = 1; + if(do_rpc) do_rpc = 1; + if(do_proc2) do_proc2 = 1; + if(do_proc3) do_proc3 = 1; + if(do_proc4) do_proc4 = 1; + if(do_proc4ops) do_proc4ops = 1; + + uint32_t lines = procfile_lines(ff), l; + uint32_t words; + + char *type; + unsigned long long rc_hits = 0, rc_misses = 0, rc_nocache = 0; + unsigned long long fh_stale = 0, fh_total_lookups = 0, fh_anonymous_lookups = 0, fh_dir_not_in_dcache = 0, fh_non_dir_not_in_dcache = 0; + unsigned long long io_read = 0, io_write = 0; + unsigned long long th_threads = 0, th_fullcnt = 0, th_hist10 = 0, th_hist20 = 0, th_hist30 = 0, th_hist40 = 0, th_hist50 = 0, th_hist60 = 0, th_hist70 = 0, th_hist80 = 0, th_hist90 = 0, th_hist100 = 0; + unsigned long long ra_size = 0, ra_hist10 = 0, ra_hist20 = 0, ra_hist30 = 0, ra_hist40 = 0, ra_hist50 = 0, ra_hist60 = 0, ra_hist70 = 0, ra_hist80 = 0, ra_hist90 = 0, ra_hist100 = 0, ra_none = 0; + unsigned long long net_count = 0, net_udp_count = 0, net_tcp_count = 0, net_tcp_connections = 0; + unsigned long long rpc_count = 0, rpc_bad_format = 0, rpc_bad_auth = 0, rpc_bad_client = 0; + + for(l = 0; l < lines ;l++) { + words = procfile_linewords(ff, l); + if(!words) continue; + + type = procfile_lineword(ff, l, 0); + + if(do_rc == 1 && strcmp(type, "rc") == 0) { + if(words < 4) { + error("%s line of /proc/net/rpc/nfsd has %d words, expected %d", type, words, 4); + continue; + } + + rc_hits = strtoull(procfile_lineword(ff, l, 1), NULL, 10); + rc_misses = strtoull(procfile_lineword(ff, l, 2), NULL, 10); + rc_nocache = strtoull(procfile_lineword(ff, l, 3), NULL, 10); + + unsigned long long sum = rc_hits + rc_misses + rc_nocache; + if(sum == 0ULL) do_rc = -1; + else do_rc = 2; + } + else if(do_fh == 1 && strcmp(type, "fh") == 0) { + if(words < 6) { + error("%s line of /proc/net/rpc/nfsd has %d words, expected %d", type, words, 6); + continue; + } + + fh_stale = strtoull(procfile_lineword(ff, l, 1), NULL, 10); + fh_total_lookups = strtoull(procfile_lineword(ff, l, 2), NULL, 10); + fh_anonymous_lookups = strtoull(procfile_lineword(ff, l, 3), NULL, 10); + fh_dir_not_in_dcache = strtoull(procfile_lineword(ff, l, 4), NULL, 10); + fh_non_dir_not_in_dcache = strtoull(procfile_lineword(ff, l, 5), NULL, 10); + + unsigned long long sum = fh_stale + fh_total_lookups + fh_anonymous_lookups + fh_dir_not_in_dcache + fh_non_dir_not_in_dcache; + if(sum == 0ULL) do_fh = -1; + else do_fh = 2; + } + else if(do_io == 1 && strcmp(type, "io") == 0) { + if(words < 3) { + error("%s line of /proc/net/rpc/nfsd has %d words, expected %d", type, words, 3); + continue; + } + + io_read = strtoull(procfile_lineword(ff, l, 1), NULL, 10); + io_write = strtoull(procfile_lineword(ff, l, 2), NULL, 10); + + unsigned long long sum = io_read + io_write; + if(sum == 0ULL) do_io = -1; + else do_io = 2; + } + else if(do_th == 1 && strcmp(type, "th") == 0) { + if(words < 13) { + error("%s line of /proc/net/rpc/nfsd has %d words, expected %d", type, words, 13); + continue; + } + + th_threads = strtoull(procfile_lineword(ff, l, 1), NULL, 10); + th_fullcnt = strtoull(procfile_lineword(ff, l, 2), NULL, 10); + th_hist10 = (unsigned long long)(atof(procfile_lineword(ff, l, 3)) * 1000.0); + th_hist20 = (unsigned long long)(atof(procfile_lineword(ff, l, 4)) * 1000.0); + th_hist30 = (unsigned long long)(atof(procfile_lineword(ff, l, 5)) * 1000.0); + th_hist40 = (unsigned long long)(atof(procfile_lineword(ff, l, 6)) * 1000.0); + th_hist50 = (unsigned long long)(atof(procfile_lineword(ff, l, 7)) * 1000.0); + th_hist60 = (unsigned long long)(atof(procfile_lineword(ff, l, 8)) * 1000.0); + th_hist70 = (unsigned long long)(atof(procfile_lineword(ff, l, 9)) * 1000.0); + th_hist80 = (unsigned long long)(atof(procfile_lineword(ff, l, 10)) * 1000.0); + th_hist90 = (unsigned long long)(atof(procfile_lineword(ff, l, 11)) * 1000.0); + th_hist100 = (unsigned long long)(atof(procfile_lineword(ff, l, 12)) * 1000.0); + + // threads histogram has been disabled on recent kernels + // http://permalink.gmane.org/gmane.linux.nfs/24528 + unsigned long long sum = th_hist10 + th_hist20 + th_hist30 + th_hist40 + th_hist50 + th_hist60 + th_hist70 + th_hist80 + th_hist90 + th_hist100; + if(sum == 0ULL) { + if(!th_warning) { + info("Disabling /proc/net/rpc/nfsd threads histogram. It seems unused on this machine. It will be enabled automatically when found with data in it."); + th_warning = 1; + } + do_th = -1; + } + else do_th = 2; + } + else if(do_ra == 1 && strcmp(type, "ra") == 0) { + if(words < 13) { + error("%s line of /proc/net/rpc/nfsd has %d words, expected %d", type, words, 13); + continue; + } + + ra_size = strtoull(procfile_lineword(ff, l, 1), NULL, 10); + ra_hist10 = strtoull(procfile_lineword(ff, l, 2), NULL, 10); + ra_hist20 = strtoull(procfile_lineword(ff, l, 3), NULL, 10); + ra_hist30 = strtoull(procfile_lineword(ff, l, 4), NULL, 10); + ra_hist40 = strtoull(procfile_lineword(ff, l, 5), NULL, 10); + ra_hist50 = strtoull(procfile_lineword(ff, l, 6), NULL, 10); + ra_hist60 = strtoull(procfile_lineword(ff, l, 7), NULL, 10); + ra_hist70 = strtoull(procfile_lineword(ff, l, 8), NULL, 10); + ra_hist80 = strtoull(procfile_lineword(ff, l, 9), NULL, 10); + ra_hist90 = strtoull(procfile_lineword(ff, l, 10), NULL, 10); + ra_hist100 = strtoull(procfile_lineword(ff, l, 11), NULL, 10); + ra_none = strtoull(procfile_lineword(ff, l, 12), NULL, 10); + + unsigned long long sum = ra_hist10 + ra_hist20 + ra_hist30 + ra_hist40 + ra_hist50 + ra_hist60 + ra_hist70 + ra_hist80 + ra_hist90 + ra_hist100 + ra_none; + if(sum == 0ULL) { + if(!ra_warning) { + info("Disabling /proc/net/rpc/nfsd read ahead histogram. It seems unused on this machine. It will be enabled automatically when found with data in it."); + ra_warning = 1; + } + do_ra = -1; + } + else do_ra = 2; + } + else if(do_net == 1 && strcmp(type, "net") == 0) { + if(words < 5) { + error("%s line of /proc/net/rpc/nfsd has %d words, expected %d", type, words, 5); + continue; + } + + net_count = strtoull(procfile_lineword(ff, l, 1), NULL, 10); + net_udp_count = strtoull(procfile_lineword(ff, l, 2), NULL, 10); + net_tcp_count = strtoull(procfile_lineword(ff, l, 3), NULL, 10); + net_tcp_connections = strtoull(procfile_lineword(ff, l, 4), NULL, 10); + + unsigned long long sum = net_count + net_udp_count + net_tcp_count + net_tcp_connections; + if(sum == 0ULL) do_net = -1; + else do_net = 2; + } + else if(do_rpc == 1 && strcmp(type, "rpc") == 0) { + if(words < 6) { + error("%s line of /proc/net/rpc/nfsd has %d words, expected %d", type, words, 6); + continue; + } + + rpc_count = strtoull(procfile_lineword(ff, l, 1), NULL, 10); + rpc_bad_format = strtoull(procfile_lineword(ff, l, 2), NULL, 10); + rpc_bad_auth = strtoull(procfile_lineword(ff, l, 3), NULL, 10); + rpc_bad_client = strtoull(procfile_lineword(ff, l, 4), NULL, 10); + + unsigned long long sum = rpc_count + rpc_bad_format + rpc_bad_auth + rpc_bad_client; + if(sum == 0ULL) do_rpc = -1; + else do_rpc = 2; + } + else if(do_proc2 == 1 && strcmp(type, "proc2") == 0) { + // the first number is the count of numbers present + // so we start for word 2 + + unsigned long long sum = 0; + unsigned int i, j; + for(i = 0, j = 2; j < words && nfsd_proc_values[i].name[0] ; i++, j++) { + nfsd_proc_values[i].proc2 = strtoull(procfile_lineword(ff, l, j), NULL, 10); + nfsd_proc_values[i].present2 = 1; + sum += nfsd_proc_values[i].proc2; + } + + if(sum == 0ULL) { + if(!proc2_warning) { + error("Disabling /proc/net/rpc/nfsd v2 procedure calls chart. It seems unused on this machine. It will be enabled automatically when found with data in it."); + proc2_warning = 1; + } + do_proc2 = 0; + } + else do_proc2 = 2; + } + else if(do_proc3 == 1 && strcmp(type, "proc3") == 0) { + // the first number is the count of numbers present + // so we start for word 2 + + unsigned long long sum = 0; + unsigned int i, j; + for(i = 0, j = 2; j < words && nfsd_proc_values[i].name[0] ; i++, j++) { + nfsd_proc_values[i].proc3 = strtoull(procfile_lineword(ff, l, j), NULL, 10); + nfsd_proc_values[i].present3 = 1; + sum += nfsd_proc_values[i].proc3; + } + + if(sum == 0ULL) { + if(!proc3_warning) { + info("Disabling /proc/net/rpc/nfsd v3 procedure calls chart. It seems unused on this machine. It will be enabled automatically when found with data in it."); + proc3_warning = 1; + } + do_proc3 = 0; + } + else do_proc3 = 2; + } + else if(do_proc4 == 1 && strcmp(type, "proc4") == 0) { + // the first number is the count of numbers present + // so we start for word 2 + + unsigned long long sum = 0; + unsigned int i, j; + for(i = 0, j = 2; j < words && nfsd_proc_values[i].name[0] ; i++, j++) { + nfsd_proc_values[i].proc4 = strtoull(procfile_lineword(ff, l, j), NULL, 10); + nfsd_proc_values[i].present4 = 1; + sum += nfsd_proc_values[i].proc4; + } + + if(sum == 0ULL) { + if(!proc4_warning) { + info("Disabling /proc/net/rpc/nfsd v4 procedure calls chart. It seems unused on this machine. It will be enabled automatically when found with data in it."); + proc4_warning = 1; + } + do_proc4 = 0; + } + else do_proc4 = 2; + } + else if(do_proc4ops == 1 && strcmp(type, "proc4ops") == 0) { + // the first number is the count of numbers present + // so we start for word 2 + + unsigned long long sum = 0; + unsigned int i, j; + for(i = 0, j = 2; j < words && nfsd4_ops_values[i].name[0] ; i++, j++) { + nfsd4_ops_values[i].value = strtoull(procfile_lineword(ff, l, j), NULL, 10); + nfsd4_ops_values[i].present = 1; + sum += nfsd4_ops_values[i].value; + } + + if(sum == 0ULL) { + if(!proc4ops_warning) { + info("Disabling /proc/net/rpc/nfsd v4 operations chart. It seems unused on this machine. It will be enabled automatically when found with data in it."); + proc4ops_warning = 1; + } + do_proc4ops = 0; + } + else do_proc4ops = 2; + } + } + + RRDSET *st; + + // -------------------------------------------------------------------- + + if(do_rc == 2) { + st = rrdset_find_bytype("nfsd", "readcache"); + if(!st) { + st = rrdset_create("nfsd", "readcache", NULL, "nfsd", NULL, "Read Cache", "reads/s", 5000, update_every, RRDSET_TYPE_STACKED); + + rrddim_add(st, "hits", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "misses", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "nocache", NULL, 1, 1, RRDDIM_INCREMENTAL); + } + else rrdset_next(st); + + rrddim_set(st, "hits", rc_hits); + rrddim_set(st, "misses", rc_misses); + rrddim_set(st, "nocache", rc_nocache); + rrdset_done(st); + } + + // -------------------------------------------------------------------- + + if(do_fh == 2) { + st = rrdset_find_bytype("nfsd", "filehandles"); + if(!st) { + st = rrdset_create("nfsd", "filehandles", NULL, "nfsd", NULL, "File Handles", "handles/s", 5001, update_every, RRDSET_TYPE_LINE); + st->isdetail = 1; + + rrddim_add(st, "stale", NULL, 1, 1, RRDDIM_ABSOLUTE); + rrddim_add(st, "total_lookups", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "anonymous_lookups", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "dir_not_in_dcache", NULL, -1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "non_dir_not_in_dcache", NULL, -1, 1, RRDDIM_INCREMENTAL); + } + else rrdset_next(st); + + rrddim_set(st, "stale", fh_stale); + rrddim_set(st, "total_lookups", fh_total_lookups); + rrddim_set(st, "anonymous_lookups", fh_anonymous_lookups); + rrddim_set(st, "dir_not_in_dcache", fh_dir_not_in_dcache); + rrddim_set(st, "non_dir_not_in_dcache", fh_non_dir_not_in_dcache); + rrdset_done(st); + } + + // -------------------------------------------------------------------- + + if(do_io == 2) { + st = rrdset_find_bytype("nfsd", "io"); + if(!st) { + st = rrdset_create("nfsd", "io", NULL, "nfsd", NULL, "I/O", "kilobytes/s", 5002, update_every, RRDSET_TYPE_AREA); + + rrddim_add(st, "read", NULL, 1, 1000, RRDDIM_INCREMENTAL); + rrddim_add(st, "write", NULL, -1, 1000, RRDDIM_INCREMENTAL); + } + else rrdset_next(st); + + rrddim_set(st, "read", io_read); + rrddim_set(st, "write", io_write); + rrdset_done(st); + } + + // -------------------------------------------------------------------- + + if(do_th == 2) { + st = rrdset_find_bytype("nfsd", "threads"); + if(!st) { + st = rrdset_create("nfsd", "threads", NULL, "nfsd", NULL, "Threads", "threads", 5003, update_every, RRDSET_TYPE_LINE); + + rrddim_add(st, "threads", NULL, 1, 1, RRDDIM_ABSOLUTE); + } + else rrdset_next(st); + + rrddim_set(st, "threads", th_threads); + rrdset_done(st); + + st = rrdset_find_bytype("nfsd", "threads_fullcnt"); + if(!st) { + st = rrdset_create("nfsd", "threads_fullcnt", NULL, "nfsd", NULL, "Threads Full Count", "ops/s", 5004, update_every, RRDSET_TYPE_LINE); + + rrddim_add(st, "full_count", NULL, 1, 1, RRDDIM_INCREMENTAL); + } + else rrdset_next(st); + + rrddim_set(st, "full_count", th_fullcnt); + rrdset_done(st); + + st = rrdset_find_bytype("nfsd", "threads_histogram"); + if(!st) { + st = rrdset_create("nfsd", "threads_histogram", NULL, "nfsd", NULL, "Threads Usage Histogram", "percentage", 5005, update_every, RRDSET_TYPE_LINE); + + rrddim_add(st, "0%-10%", NULL, 1, 1000, RRDDIM_ABSOLUTE); + rrddim_add(st, "10%-20%", NULL, 1, 1000, RRDDIM_ABSOLUTE); + rrddim_add(st, "20%-30%", NULL, 1, 1000, RRDDIM_ABSOLUTE); + rrddim_add(st, "30%-40%", NULL, 1, 1000, RRDDIM_ABSOLUTE); + rrddim_add(st, "40%-50%", NULL, 1, 1000, RRDDIM_ABSOLUTE); + rrddim_add(st, "50%-60%", NULL, 1, 1000, RRDDIM_ABSOLUTE); + rrddim_add(st, "60%-70%", NULL, 1, 1000, RRDDIM_ABSOLUTE); + rrddim_add(st, "70%-80%", NULL, 1, 1000, RRDDIM_ABSOLUTE); + rrddim_add(st, "80%-90%", NULL, 1, 1000, RRDDIM_ABSOLUTE); + rrddim_add(st, "90%-100%", NULL, 1, 1000, RRDDIM_ABSOLUTE); + } + else rrdset_next(st); + + rrddim_set(st, "0%-10%", th_hist10); + rrddim_set(st, "10%-20%", th_hist20); + rrddim_set(st, "20%-30%", th_hist30); + rrddim_set(st, "30%-40%", th_hist40); + rrddim_set(st, "40%-50%", th_hist50); + rrddim_set(st, "50%-60%", th_hist60); + rrddim_set(st, "60%-70%", th_hist70); + rrddim_set(st, "70%-80%", th_hist80); + rrddim_set(st, "80%-90%", th_hist90); + rrddim_set(st, "90%-100%", th_hist100); + rrdset_done(st); + } + + // -------------------------------------------------------------------- + + if(do_ra == 2) { + st = rrdset_find_bytype("nfsd", "readahead"); + if(!st) { + st = rrdset_create("nfsd", "readahead", NULL, "nfsd", NULL, "Read Ahead Depth", "percentage", 5005, update_every, RRDSET_TYPE_STACKED); + + rrddim_add(st, "10%", NULL, 1, 1, RRDDIM_PCENT_OVER_DIFF_TOTAL); + rrddim_add(st, "20%", NULL, 1, 1, RRDDIM_PCENT_OVER_DIFF_TOTAL); + rrddim_add(st, "30%", NULL, 1, 1, RRDDIM_PCENT_OVER_DIFF_TOTAL); + rrddim_add(st, "40%", NULL, 1, 1, RRDDIM_PCENT_OVER_DIFF_TOTAL); + rrddim_add(st, "50%", NULL, 1, 1, RRDDIM_PCENT_OVER_DIFF_TOTAL); + rrddim_add(st, "60%", NULL, 1, 1, RRDDIM_PCENT_OVER_DIFF_TOTAL); + rrddim_add(st, "70%", NULL, 1, 1, RRDDIM_PCENT_OVER_DIFF_TOTAL); + rrddim_add(st, "80%", NULL, 1, 1, RRDDIM_PCENT_OVER_DIFF_TOTAL); + rrddim_add(st, "90%", NULL, 1, 1, RRDDIM_PCENT_OVER_DIFF_TOTAL); + rrddim_add(st, "100%", NULL, 1, 1, RRDDIM_PCENT_OVER_DIFF_TOTAL); + rrddim_add(st, "misses", NULL, 1, 1, RRDDIM_PCENT_OVER_DIFF_TOTAL); + } + else rrdset_next(st); + + // ignore ra_size + if(ra_size) {}; + + rrddim_set(st, "10%", ra_hist10); + rrddim_set(st, "20%", ra_hist20); + rrddim_set(st, "30%", ra_hist30); + rrddim_set(st, "40%", ra_hist40); + rrddim_set(st, "50%", ra_hist50); + rrddim_set(st, "60%", ra_hist60); + rrddim_set(st, "70%", ra_hist70); + rrddim_set(st, "80%", ra_hist80); + rrddim_set(st, "90%", ra_hist90); + rrddim_set(st, "100%", ra_hist100); + rrddim_set(st, "misses", ra_none); + rrdset_done(st); + } + + // -------------------------------------------------------------------- + + if(do_net == 2) { + st = rrdset_find_bytype("nfsd", "net"); + if(!st) { + st = rrdset_create("nfsd", "net", NULL, "nfsd", NULL, "Network Reads", "reads/s", 5007, update_every, RRDSET_TYPE_STACKED); + st->isdetail = 1; + + rrddim_add(st, "udp", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "tcp", NULL, 1, 1, RRDDIM_INCREMENTAL); + } + else rrdset_next(st); + + // ignore net_count, net_tcp_connections + if(net_count) {}; + if(net_tcp_connections) {}; + + rrddim_set(st, "udp", net_udp_count); + rrddim_set(st, "tcp", net_tcp_count); + rrdset_done(st); + } + + // -------------------------------------------------------------------- + + if(do_rpc == 2) { + st = rrdset_find_bytype("nfsd", "rpc"); + if(!st) { + st = rrdset_create("nfsd", "rpc", NULL, "nfsd", NULL, "Remote Procedure Calls", "calls/s", 5008, update_every, RRDSET_TYPE_LINE); + st->isdetail = 1; + + rrddim_add(st, "all", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "bad_format", NULL, -1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "bad_auth", NULL, -1, 1, RRDDIM_INCREMENTAL); + } + else rrdset_next(st); + + // ignore rpc_bad_client + if(rpc_bad_client) {}; + + rrddim_set(st, "all", rpc_count); + rrddim_set(st, "bad_format", rpc_bad_format); + rrddim_set(st, "bad_auth", rpc_bad_auth); + rrdset_done(st); + } + + // -------------------------------------------------------------------- + + if(do_proc2 == 2) { + unsigned int i; + st = rrdset_find_bytype("nfsd", "proc2"); + if(!st) { + st = rrdset_create("nfsd", "proc2", NULL, "nfsd", NULL, "NFS v2 Calls", "calls/s", 5009, update_every, RRDSET_TYPE_STACKED); + + for(i = 0; nfsd_proc_values[i].present2 ; i++) + rrddim_add(st, nfsd_proc_values[i].name, NULL, 1, 1, RRDDIM_INCREMENTAL); + } + else rrdset_next(st); + + for(i = 0; nfsd_proc_values[i].present2 ; i++) + rrddim_set(st, nfsd_proc_values[i].name, nfsd_proc_values[i].proc2); + + rrdset_done(st); + } + + // -------------------------------------------------------------------- + + if(do_proc3 == 2) { + unsigned int i; + st = rrdset_find_bytype("nfsd", "proc3"); + if(!st) { + st = rrdset_create("nfsd", "proc3", NULL, "nfsd", NULL, "NFS v3 Calls", "calls/s", 5010, update_every, RRDSET_TYPE_STACKED); + + for(i = 0; nfsd_proc_values[i].present3 ; i++) + rrddim_add(st, nfsd_proc_values[i].name, NULL, 1, 1, RRDDIM_INCREMENTAL); + } + else rrdset_next(st); + + for(i = 0; nfsd_proc_values[i].present3 ; i++) + rrddim_set(st, nfsd_proc_values[i].name, nfsd_proc_values[i].proc3); + + rrdset_done(st); + } + + // -------------------------------------------------------------------- + + if(do_proc4 == 2) { + unsigned int i; + st = rrdset_find_bytype("nfsd", "proc4"); + if(!st) { + st = rrdset_create("nfsd", "proc4", NULL, "nfsd", NULL, "NFS v4 Calls", "calls/s", 5011, update_every, RRDSET_TYPE_STACKED); + + for(i = 0; nfsd_proc_values[i].present4 ; i++) + rrddim_add(st, nfsd_proc_values[i].name, NULL, 1, 1, RRDDIM_INCREMENTAL); + } + else rrdset_next(st); + + for(i = 0; nfsd_proc_values[i].present4 ; i++) + rrddim_set(st, nfsd_proc_values[i].name, nfsd_proc_values[i].proc4); + + rrdset_done(st); + } + + // -------------------------------------------------------------------- + + if(do_proc4ops == 2) { + unsigned int i; + st = rrdset_find_bytype("nfsd", "proc4ops"); + if(!st) { + st = rrdset_create("nfsd", "proc4ops", NULL, "nfsd", NULL, "NFS v4 Operations", "operations/s", 5012, update_every, RRDSET_TYPE_STACKED); + + for(i = 0; nfsd4_ops_values[i].present ; i++) + rrddim_add(st, nfsd4_ops_values[i].name, NULL, 1, 1, RRDDIM_INCREMENTAL); + } + else rrdset_next(st); + + for(i = 0; nfsd4_ops_values[i].present ; i++) + rrddim_set(st, nfsd4_ops_values[i].name, nfsd4_ops_values[i].value); + + rrdset_done(st); + } + + return 0; +} diff --git a/src/proc_net_snmp.c b/src/proc_net_snmp.c new file mode 100755 index 000000000..efefbadce --- /dev/null +++ b/src/proc_net_snmp.c @@ -0,0 +1,366 @@ +#ifdef HAVE_CONFIG_H +#include <config.h> +#endif +#include <stdio.h> +#include <stdlib.h> +#include <string.h> + +#include "common.h" +#include "log.h" +#include "appconfig.h" +#include "procfile.h" +#include "rrd.h" +#include "plugin_proc.h" + +#define RRD_TYPE_NET_SNMP "ipv4" +#define RRD_TYPE_NET_SNMP_LEN strlen(RRD_TYPE_NET_SNMP) + +int do_proc_net_snmp(int update_every, unsigned long long dt) { + static procfile *ff = NULL; + static int do_ip_packets = -1, do_ip_fragsout = -1, do_ip_fragsin = -1, do_ip_errors = -1, + do_tcp_sockets = -1, do_tcp_packets = -1, do_tcp_errors = -1, do_tcp_handshake = -1, + do_udp_packets = -1, do_udp_errors = -1; + + if(do_ip_packets == -1) do_ip_packets = config_get_boolean("plugin:proc:/proc/net/snmp", "ipv4 packets", 1); + if(do_ip_fragsout == -1) do_ip_fragsout = config_get_boolean("plugin:proc:/proc/net/snmp", "ipv4 fragrments sent", 1); + if(do_ip_fragsin == -1) do_ip_fragsin = config_get_boolean("plugin:proc:/proc/net/snmp", "ipv4 fragments assembly", 1); + if(do_ip_errors == -1) do_ip_errors = config_get_boolean("plugin:proc:/proc/net/snmp", "ipv4 errors", 1); + if(do_tcp_sockets == -1) do_tcp_sockets = config_get_boolean("plugin:proc:/proc/net/snmp", "ipv4 TCP connections", 1); + if(do_tcp_packets == -1) do_tcp_packets = config_get_boolean("plugin:proc:/proc/net/snmp", "ipv4 TCP packets", 1); + if(do_tcp_errors == -1) do_tcp_errors = config_get_boolean("plugin:proc:/proc/net/snmp", "ipv4 TCP errors", 1); + if(do_tcp_handshake == -1) do_tcp_handshake = config_get_boolean("plugin:proc:/proc/net/snmp", "ipv4 TCP handshake issues", 1); + if(do_udp_packets == -1) do_udp_packets = config_get_boolean("plugin:proc:/proc/net/snmp", "ipv4 UDP packets", 1); + if(do_udp_errors == -1) do_udp_errors = config_get_boolean("plugin:proc:/proc/net/snmp", "ipv4 UDP errors", 1); + + if(dt) {}; + + if(!ff) { + char filename[FILENAME_MAX + 1]; + snprintf(filename, FILENAME_MAX, "%s%s", global_host_prefix, "/proc/net/snmp"); + ff = procfile_open(config_get("plugin:proc:/proc/net/snmp", "filename to monitor", filename), " \t:", PROCFILE_FLAG_DEFAULT); + } + if(!ff) return 1; + + ff = procfile_readall(ff); + if(!ff) return 0; // we return 0, so that we will retry to open it next time + + uint32_t lines = procfile_lines(ff), l; + uint32_t words; + + RRDSET *st; + + for(l = 0; l < lines ;l++) { + if(strcmp(procfile_lineword(ff, l, 0), "Ip") == 0) { + l++; + + if(strcmp(procfile_lineword(ff, l, 0), "Ip") != 0) { + error("Cannot read Ip line from /proc/net/snmp."); + break; + } + + words = procfile_linewords(ff, l); + if(words < 20) { + error("Cannot read /proc/net/snmp Ip line. Expected 20 params, read %d.", words); + continue; + } + + // see also http://net-snmp.sourceforge.net/docs/mibs/ip.html + unsigned long long Forwarding, DefaultTTL, InReceives, InHdrErrors, InAddrErrors, ForwDatagrams, InUnknownProtos, InDiscards, InDelivers, + OutRequests, OutDiscards, OutNoRoutes, ReasmTimeout, ReasmReqds, ReasmOKs, ReasmFails, FragOKs, FragFails, FragCreates; + + Forwarding = strtoull(procfile_lineword(ff, l, 1), NULL, 10); + DefaultTTL = strtoull(procfile_lineword(ff, l, 2), NULL, 10); + InReceives = strtoull(procfile_lineword(ff, l, 3), NULL, 10); + InHdrErrors = strtoull(procfile_lineword(ff, l, 4), NULL, 10); + InAddrErrors = strtoull(procfile_lineword(ff, l, 5), NULL, 10); + ForwDatagrams = strtoull(procfile_lineword(ff, l, 6), NULL, 10); + InUnknownProtos = strtoull(procfile_lineword(ff, l, 7), NULL, 10); + InDiscards = strtoull(procfile_lineword(ff, l, 8), NULL, 10); + InDelivers = strtoull(procfile_lineword(ff, l, 9), NULL, 10); + OutRequests = strtoull(procfile_lineword(ff, l, 10), NULL, 10); + OutDiscards = strtoull(procfile_lineword(ff, l, 11), NULL, 10); + OutNoRoutes = strtoull(procfile_lineword(ff, l, 12), NULL, 10); + ReasmTimeout = strtoull(procfile_lineword(ff, l, 13), NULL, 10); + ReasmReqds = strtoull(procfile_lineword(ff, l, 14), NULL, 10); + ReasmOKs = strtoull(procfile_lineword(ff, l, 15), NULL, 10); + ReasmFails = strtoull(procfile_lineword(ff, l, 16), NULL, 10); + FragOKs = strtoull(procfile_lineword(ff, l, 17), NULL, 10); + FragFails = strtoull(procfile_lineword(ff, l, 18), NULL, 10); + FragCreates = strtoull(procfile_lineword(ff, l, 19), NULL, 10); + + // these are not counters + if(Forwarding) {}; // is forwarding enabled? + if(DefaultTTL) {}; // the default ttl on packets + if(ReasmTimeout) {}; // Reassembly timeout + + // this counter is not used + if(InDelivers) {}; // total number of packets delivered to IP user-protocols + + // -------------------------------------------------------------------- + + if(do_ip_packets) { + st = rrdset_find(RRD_TYPE_NET_SNMP ".packets"); + if(!st) { + st = rrdset_create(RRD_TYPE_NET_SNMP, "packets", NULL, "packets", NULL, "IPv4 Packets", "packets/s", 3000, update_every, RRDSET_TYPE_LINE); + + rrddim_add(st, "received", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "sent", NULL, -1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "forwarded", NULL, 1, 1, RRDDIM_INCREMENTAL); + } + else rrdset_next(st); + + rrddim_set(st, "sent", OutRequests); + rrddim_set(st, "received", InReceives); + rrddim_set(st, "forwarded", ForwDatagrams); + rrdset_done(st); + } + + // -------------------------------------------------------------------- + + if(do_ip_fragsout) { + st = rrdset_find(RRD_TYPE_NET_SNMP ".fragsout"); + if(!st) { + st = rrdset_create(RRD_TYPE_NET_SNMP, "fragsout", NULL, "fragments", NULL, "IPv4 Fragments Sent", "packets/s", 3010, update_every, RRDSET_TYPE_LINE); + st->isdetail = 1; + + rrddim_add(st, "ok", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "failed", NULL, -1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "all", NULL, 1, 1, RRDDIM_INCREMENTAL); + } + else rrdset_next(st); + + rrddim_set(st, "ok", FragOKs); + rrddim_set(st, "failed", FragFails); + rrddim_set(st, "all", FragCreates); + rrdset_done(st); + } + + // -------------------------------------------------------------------- + + if(do_ip_fragsin) { + st = rrdset_find(RRD_TYPE_NET_SNMP ".fragsin"); + if(!st) { + st = rrdset_create(RRD_TYPE_NET_SNMP, "fragsin", NULL, "fragments", NULL, "IPv4 Fragments Reassembly", "packets/s", 3011, update_every, RRDSET_TYPE_LINE); + st->isdetail = 1; + + rrddim_add(st, "ok", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "failed", NULL, -1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "all", NULL, 1, 1, RRDDIM_INCREMENTAL); + } + else rrdset_next(st); + + rrddim_set(st, "ok", ReasmOKs); + rrddim_set(st, "failed", ReasmFails); + rrddim_set(st, "all", ReasmReqds); + rrdset_done(st); + } + + // -------------------------------------------------------------------- + + if(do_ip_errors) { + st = rrdset_find(RRD_TYPE_NET_SNMP ".errors"); + if(!st) { + st = rrdset_create(RRD_TYPE_NET_SNMP, "errors", NULL, "errors", NULL, "IPv4 Errors", "packets/s", 3002, update_every, RRDSET_TYPE_LINE); + st->isdetail = 1; + + rrddim_add(st, "InDiscards", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "OutDiscards", NULL, -1, 1, RRDDIM_INCREMENTAL); + + rrddim_add(st, "InHdrErrors", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "InAddrErrors", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "InUnknownProtos", NULL, 1, 1, RRDDIM_INCREMENTAL); + + rrddim_add(st, "OutNoRoutes", NULL, -1, 1, RRDDIM_INCREMENTAL); + } + else rrdset_next(st); + + rrddim_set(st, "InDiscards", InDiscards); + rrddim_set(st, "OutDiscards", OutDiscards); + rrddim_set(st, "InHdrErrors", InHdrErrors); + rrddim_set(st, "InAddrErrors", InAddrErrors); + rrddim_set(st, "InUnknownProtos", InUnknownProtos); + rrddim_set(st, "OutNoRoutes", OutNoRoutes); + rrdset_done(st); + } + } + else if(strcmp(procfile_lineword(ff, l, 0), "Tcp") == 0) { + l++; + + if(strcmp(procfile_lineword(ff, l, 0), "Tcp") != 0) { + error("Cannot read Tcp line from /proc/net/snmp."); + break; + } + + words = procfile_linewords(ff, l); + if(words < 15) { + error("Cannot read /proc/net/snmp Tcp line. Expected 15 params, read %d.", words); + continue; + } + + unsigned long long RtoAlgorithm, RtoMin, RtoMax, MaxConn, ActiveOpens, PassiveOpens, AttemptFails, EstabResets, + CurrEstab, InSegs, OutSegs, RetransSegs, InErrs, OutRsts; + + RtoAlgorithm = strtoull(procfile_lineword(ff, l, 1), NULL, 10); + RtoMin = strtoull(procfile_lineword(ff, l, 2), NULL, 10); + RtoMax = strtoull(procfile_lineword(ff, l, 3), NULL, 10); + MaxConn = strtoull(procfile_lineword(ff, l, 4), NULL, 10); + ActiveOpens = strtoull(procfile_lineword(ff, l, 5), NULL, 10); + PassiveOpens = strtoull(procfile_lineword(ff, l, 6), NULL, 10); + AttemptFails = strtoull(procfile_lineword(ff, l, 7), NULL, 10); + EstabResets = strtoull(procfile_lineword(ff, l, 8), NULL, 10); + CurrEstab = strtoull(procfile_lineword(ff, l, 9), NULL, 10); + InSegs = strtoull(procfile_lineword(ff, l, 10), NULL, 10); + OutSegs = strtoull(procfile_lineword(ff, l, 11), NULL, 10); + RetransSegs = strtoull(procfile_lineword(ff, l, 12), NULL, 10); + InErrs = strtoull(procfile_lineword(ff, l, 13), NULL, 10); + OutRsts = strtoull(procfile_lineword(ff, l, 14), NULL, 10); + + // these are not counters + if(RtoAlgorithm) {}; + if(RtoMin) {}; + if(RtoMax) {}; + if(MaxConn) {}; + + // -------------------------------------------------------------------- + + // see http://net-snmp.sourceforge.net/docs/mibs/tcp.html + if(do_tcp_sockets) { + st = rrdset_find(RRD_TYPE_NET_SNMP ".tcpsock"); + if(!st) { + st = rrdset_create(RRD_TYPE_NET_SNMP, "tcpsock", NULL, "tcp", NULL, "IPv4 TCP Connections", "active connections", 2500, update_every, RRDSET_TYPE_LINE); + + rrddim_add(st, "connections", NULL, 1, 1, RRDDIM_ABSOLUTE); + } + else rrdset_next(st); + + rrddim_set(st, "connections", CurrEstab); + rrdset_done(st); + } + + // -------------------------------------------------------------------- + + if(do_tcp_packets) { + st = rrdset_find(RRD_TYPE_NET_SNMP ".tcppackets"); + if(!st) { + st = rrdset_create(RRD_TYPE_NET_SNMP, "tcppackets", NULL, "tcp", NULL, "IPv4 TCP Packets", "packets/s", 2600, update_every, RRDSET_TYPE_LINE); + + rrddim_add(st, "received", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "sent", NULL, -1, 1, RRDDIM_INCREMENTAL); + } + else rrdset_next(st); + + rrddim_set(st, "received", InSegs); + rrddim_set(st, "sent", OutSegs); + rrdset_done(st); + } + + // -------------------------------------------------------------------- + + if(do_tcp_errors) { + st = rrdset_find(RRD_TYPE_NET_SNMP ".tcperrors"); + if(!st) { + st = rrdset_create(RRD_TYPE_NET_SNMP, "tcperrors", NULL, "tcp", NULL, "IPv4 TCP Errors", "packets/s", 2700, update_every, RRDSET_TYPE_LINE); + st->isdetail = 1; + + rrddim_add(st, "InErrs", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "RetransSegs", NULL, -1, 1, RRDDIM_INCREMENTAL); + } + else rrdset_next(st); + + rrddim_set(st, "InErrs", InErrs); + rrddim_set(st, "RetransSegs", RetransSegs); + rrdset_done(st); + } + + // -------------------------------------------------------------------- + + if(do_tcp_handshake) { + st = rrdset_find(RRD_TYPE_NET_SNMP ".tcphandshake"); + if(!st) { + st = rrdset_create(RRD_TYPE_NET_SNMP, "tcphandshake", NULL, "tcp", NULL, "IPv4 TCP Handshake Issues", "events/s", 2900, update_every, RRDSET_TYPE_LINE); + st->isdetail = 1; + + rrddim_add(st, "EstabResets", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "OutRsts", NULL, -1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "ActiveOpens", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "PassiveOpens", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "AttemptFails", NULL, 1, 1, RRDDIM_INCREMENTAL); + } + else rrdset_next(st); + + rrddim_set(st, "EstabResets", EstabResets); + rrddim_set(st, "OutRsts", OutRsts); + rrddim_set(st, "ActiveOpens", ActiveOpens); + rrddim_set(st, "PassiveOpens", PassiveOpens); + rrddim_set(st, "AttemptFails", AttemptFails); + rrdset_done(st); + } + } + else if(strcmp(procfile_lineword(ff, l, 0), "Udp") == 0) { + l++; + + if(strcmp(procfile_lineword(ff, l, 0), "Udp") != 0) { + error("Cannot read Udp line from /proc/net/snmp."); + break; + } + + words = procfile_linewords(ff, l); + if(words < 7) { + error("Cannot read /proc/net/snmp Udp line. Expected 7 params, read %d.", words); + continue; + } + + unsigned long long InDatagrams, NoPorts, InErrors, OutDatagrams, RcvbufErrors, SndbufErrors; + + InDatagrams = strtoull(procfile_lineword(ff, l, 1), NULL, 10); + NoPorts = strtoull(procfile_lineword(ff, l, 2), NULL, 10); + InErrors = strtoull(procfile_lineword(ff, l, 3), NULL, 10); + OutDatagrams = strtoull(procfile_lineword(ff, l, 4), NULL, 10); + RcvbufErrors = strtoull(procfile_lineword(ff, l, 5), NULL, 10); + SndbufErrors = strtoull(procfile_lineword(ff, l, 6), NULL, 10); + + // -------------------------------------------------------------------- + + // see http://net-snmp.sourceforge.net/docs/mibs/udp.html + if(do_udp_packets) { + st = rrdset_find(RRD_TYPE_NET_SNMP ".udppackets"); + if(!st) { + st = rrdset_create(RRD_TYPE_NET_SNMP, "udppackets", NULL, "udp", NULL, "IPv4 UDP Packets", "packets/s", 2601, update_every, RRDSET_TYPE_LINE); + + rrddim_add(st, "received", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "sent", NULL, -1, 1, RRDDIM_INCREMENTAL); + } + else rrdset_next(st); + + rrddim_set(st, "received", InDatagrams); + rrddim_set(st, "sent", OutDatagrams); + rrdset_done(st); + } + + // -------------------------------------------------------------------- + + if(do_udp_errors) { + st = rrdset_find(RRD_TYPE_NET_SNMP ".udperrors"); + if(!st) { + st = rrdset_create(RRD_TYPE_NET_SNMP, "udperrors", NULL, "udp", NULL, "IPv4 UDP Errors", "events/s", 2701, update_every, RRDSET_TYPE_LINE); + st->isdetail = 1; + + rrddim_add(st, "RcvbufErrors", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "SndbufErrors", NULL, -1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "InErrors", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "NoPorts", NULL, 1, 1, RRDDIM_INCREMENTAL); + } + else rrdset_next(st); + + rrddim_set(st, "InErrors", InErrors); + rrddim_set(st, "NoPorts", NoPorts); + rrddim_set(st, "RcvbufErrors", RcvbufErrors); + rrddim_set(st, "SndbufErrors", SndbufErrors); + rrdset_done(st); + } + } + } + + return 0; +} + diff --git a/src/proc_net_stat_conntrack.c b/src/proc_net_stat_conntrack.c new file mode 100755 index 000000000..912c3eef0 --- /dev/null +++ b/src/proc_net_stat_conntrack.c @@ -0,0 +1,214 @@ +#ifdef HAVE_CONFIG_H +#include <config.h> +#endif +#include <stdio.h> +#include <stdlib.h> +#include <string.h> + +#include "common.h" +#include "log.h" +#include "appconfig.h" +#include "procfile.h" +#include "rrd.h" +#include "plugin_proc.h" + +#define RRD_TYPE_NET_STAT_CONNTRACK "netfilter" +#define RRD_TYPE_NET_STAT_CONNTRACK_LEN strlen(RRD_TYPE_NET_STAT_CONNTRACK) + +int do_proc_net_stat_conntrack(int update_every, unsigned long long dt) { + static procfile *ff = NULL; + static int do_sockets = -1, do_new = -1, do_changes = -1, do_expect = -1, do_search = -1, do_errors = -1; + + if(do_sockets == -1) do_sockets = config_get_boolean("plugin:proc:/proc/net/stat/nf_conntrack", "netfilter connections", 1); + if(do_new == -1) do_new = config_get_boolean("plugin:proc:/proc/net/stat/nf_conntrack", "netfilter new connections", 1); + if(do_changes == -1) do_changes = config_get_boolean("plugin:proc:/proc/net/stat/nf_conntrack", "netfilter connection changes", 1); + if(do_expect == -1) do_expect = config_get_boolean("plugin:proc:/proc/net/stat/nf_conntrack", "netfilter connection expectations", 1); + if(do_search == -1) do_search = config_get_boolean("plugin:proc:/proc/net/stat/nf_conntrack", "netfilter connection searches", 1); + if(do_errors == -1) do_errors = config_get_boolean("plugin:proc:/proc/net/stat/nf_conntrack", "netfilter errors", 1); + + if(dt) {}; + + if(!ff) { + char filename[FILENAME_MAX + 1]; + snprintf(filename, FILENAME_MAX, "%s%s", global_host_prefix, "/proc/net/stat/nf_conntrack"); + ff = procfile_open(config_get("plugin:proc:/proc/net/stat/nf_conntrack", "filename to monitor", filename), " \t:", PROCFILE_FLAG_DEFAULT); + } + if(!ff) return 1; + + ff = procfile_readall(ff); + if(!ff) return 0; // we return 0, so that we will retry to open it next time + + uint32_t lines = procfile_lines(ff), l; + uint32_t words; + + unsigned long long aentries = 0, asearched = 0, afound = 0, anew = 0, ainvalid = 0, aignore = 0, adelete = 0, adelete_list = 0, + ainsert = 0, ainsert_failed = 0, adrop = 0, aearly_drop = 0, aicmp_error = 0, aexpect_new = 0, aexpect_create = 0, aexpect_delete = 0, asearch_restart = 0; + + for(l = 1; l < lines ;l++) { + words = procfile_linewords(ff, l); + if(words < 17) { + if(words) error("Cannot read /proc/net/stat/nf_conntrack line. Expected 17 params, read %d.", words); + continue; + } + + unsigned long long tentries = 0, tsearched = 0, tfound = 0, tnew = 0, tinvalid = 0, tignore = 0, tdelete = 0, tdelete_list = 0, tinsert = 0, tinsert_failed = 0, tdrop = 0, tearly_drop = 0, ticmp_error = 0, texpect_new = 0, texpect_create = 0, texpect_delete = 0, tsearch_restart = 0; + + tentries = strtoull(procfile_lineword(ff, l, 0), NULL, 16); + tsearched = strtoull(procfile_lineword(ff, l, 1), NULL, 16); + tfound = strtoull(procfile_lineword(ff, l, 2), NULL, 16); + tnew = strtoull(procfile_lineword(ff, l, 3), NULL, 16); + tinvalid = strtoull(procfile_lineword(ff, l, 4), NULL, 16); + tignore = strtoull(procfile_lineword(ff, l, 5), NULL, 16); + tdelete = strtoull(procfile_lineword(ff, l, 6), NULL, 16); + tdelete_list = strtoull(procfile_lineword(ff, l, 7), NULL, 16); + tinsert = strtoull(procfile_lineword(ff, l, 8), NULL, 16); + tinsert_failed = strtoull(procfile_lineword(ff, l, 9), NULL, 16); + tdrop = strtoull(procfile_lineword(ff, l, 10), NULL, 16); + tearly_drop = strtoull(procfile_lineword(ff, l, 11), NULL, 16); + ticmp_error = strtoull(procfile_lineword(ff, l, 12), NULL, 16); + texpect_new = strtoull(procfile_lineword(ff, l, 13), NULL, 16); + texpect_create = strtoull(procfile_lineword(ff, l, 14), NULL, 16); + texpect_delete = strtoull(procfile_lineword(ff, l, 15), NULL, 16); + tsearch_restart = strtoull(procfile_lineword(ff, l, 16), NULL, 16); + + if(!aentries) aentries = tentries; + + // sum all the cpus together + asearched += tsearched; // conntrack.search + afound += tfound; // conntrack.search + anew += tnew; // conntrack.new + ainvalid += tinvalid; // conntrack.new + aignore += tignore; // conntrack.new + adelete += tdelete; // conntrack.changes + adelete_list += tdelete_list; // conntrack.changes + ainsert += tinsert; // conntrack.changes + ainsert_failed += tinsert_failed; // conntrack.errors + adrop += tdrop; // conntrack.errors + aearly_drop += tearly_drop; // conntrack.errors + aicmp_error += ticmp_error; // conntrack.errors + aexpect_new += texpect_new; // conntrack.expect + aexpect_create += texpect_create; // conntrack.expect + aexpect_delete += texpect_delete; // conntrack.expect + asearch_restart += tsearch_restart; // conntrack.search + } + + RRDSET *st; + + // -------------------------------------------------------------------- + + if(do_sockets) { + st = rrdset_find(RRD_TYPE_NET_STAT_CONNTRACK ".sockets"); + if(!st) { + st = rrdset_create(RRD_TYPE_NET_STAT_CONNTRACK, "sockets", NULL, RRD_TYPE_NET_STAT_CONNTRACK, NULL, "Netfilter Connections", "active connections", 1000, update_every, RRDSET_TYPE_LINE); + + rrddim_add(st, "connections", NULL, 1, 1, RRDDIM_ABSOLUTE); + } + else rrdset_next(st); + + rrddim_set(st, "connections", aentries); + rrdset_done(st); + } + + // -------------------------------------------------------------------- + + if(do_new) { + st = rrdset_find(RRD_TYPE_NET_STAT_CONNTRACK ".new"); + if(!st) { + st = rrdset_create(RRD_TYPE_NET_STAT_CONNTRACK, "new", NULL, RRD_TYPE_NET_STAT_CONNTRACK, NULL, "Netfilter New Connections", "connections/s", 1001, update_every, RRDSET_TYPE_LINE); + + rrddim_add(st, "new", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "ignore", NULL, -1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "invalid", NULL, -1, 1, RRDDIM_INCREMENTAL); + } + else rrdset_next(st); + + rrddim_set(st, "new", anew); + rrddim_set(st, "ignore", aignore); + rrddim_set(st, "invalid", ainvalid); + rrdset_done(st); + } + + // -------------------------------------------------------------------- + + if(do_changes) { + st = rrdset_find(RRD_TYPE_NET_STAT_CONNTRACK ".changes"); + if(!st) { + st = rrdset_create(RRD_TYPE_NET_STAT_CONNTRACK, "changes", NULL, RRD_TYPE_NET_STAT_CONNTRACK, NULL, "Netfilter Connection Changes", "changes/s", 1002, update_every, RRDSET_TYPE_LINE); + st->isdetail = 1; + + rrddim_add(st, "inserted", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "deleted", NULL, -1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "delete_list", NULL, -1, 1, RRDDIM_INCREMENTAL); + } + else rrdset_next(st); + + rrddim_set(st, "inserted", ainsert); + rrddim_set(st, "deleted", adelete); + rrddim_set(st, "delete_list", adelete_list); + rrdset_done(st); + } + + // -------------------------------------------------------------------- + + if(do_expect) { + st = rrdset_find(RRD_TYPE_NET_STAT_CONNTRACK ".expect"); + if(!st) { + st = rrdset_create(RRD_TYPE_NET_STAT_CONNTRACK, "expect", NULL, RRD_TYPE_NET_STAT_CONNTRACK, NULL, "Netfilter Connection Expectations", "expectations/s", 1003, update_every, RRDSET_TYPE_LINE); + st->isdetail = 1; + + rrddim_add(st, "created", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "deleted", NULL, -1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "new", NULL, 1, 1, RRDDIM_INCREMENTAL); + } + else rrdset_next(st); + + rrddim_set(st, "created", aexpect_create); + rrddim_set(st, "deleted", aexpect_delete); + rrddim_set(st, "new", aexpect_new); + rrdset_done(st); + } + + // -------------------------------------------------------------------- + + if(do_search) { + st = rrdset_find(RRD_TYPE_NET_STAT_CONNTRACK ".search"); + if(!st) { + st = rrdset_create(RRD_TYPE_NET_STAT_CONNTRACK, "search", NULL, RRD_TYPE_NET_STAT_CONNTRACK, NULL, "Netfilter Connection Searches", "searches/s", 1010, update_every, RRDSET_TYPE_LINE); + st->isdetail = 1; + + rrddim_add(st, "searched", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "restarted", NULL, -1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "found", NULL, 1, 1, RRDDIM_INCREMENTAL); + } + else rrdset_next(st); + + rrddim_set(st, "searched", asearched); + rrddim_set(st, "restarted", asearch_restart); + rrddim_set(st, "found", afound); + rrdset_done(st); + } + + // -------------------------------------------------------------------- + + if(do_errors) { + st = rrdset_find(RRD_TYPE_NET_STAT_CONNTRACK ".errors"); + if(!st) { + st = rrdset_create(RRD_TYPE_NET_STAT_CONNTRACK, "errors", NULL, RRD_TYPE_NET_STAT_CONNTRACK, NULL, "Netfilter Errors", "events/s", 1005, update_every, RRDSET_TYPE_LINE); + st->isdetail = 1; + + rrddim_add(st, "icmp_error", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "insert_failed", NULL, -1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "drop", NULL, -1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "early_drop", NULL, -1, 1, RRDDIM_INCREMENTAL); + } + else rrdset_next(st); + + rrddim_set(st, "icmp_error", aicmp_error); + rrddim_set(st, "insert_failed", ainsert_failed); + rrddim_set(st, "drop", adrop); + rrddim_set(st, "early_drop", aearly_drop); + rrdset_done(st); + } + + return 0; +} diff --git a/src/proc_softirqs.c b/src/proc_softirqs.c new file mode 100755 index 000000000..9373baaec --- /dev/null +++ b/src/proc_softirqs.c @@ -0,0 +1,160 @@ +#ifdef HAVE_CONFIG_H +#include <config.h> +#endif +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <ctype.h> + +#include "common.h" +#include "appconfig.h" +#include "procfile.h" +#include "rrd.h" +#include "plugin_proc.h" +#include "log.h" + +#define MAX_INTERRUPTS 256 +#define MAX_INTERRUPT_CPUS 256 +#define MAX_INTERRUPT_NAME 50 + +struct interrupt { + int used; + char *id; + char name[MAX_INTERRUPT_NAME + 1]; + unsigned long long value[MAX_INTERRUPT_CPUS]; + unsigned long long total; +}; + +int do_proc_softirqs(int update_every, unsigned long long dt) { + static procfile *ff = NULL; + static int cpus = -1, do_per_core = -1; + + if(dt) {}; + + if(do_per_core == -1) do_per_core = config_get_boolean("plugin:proc:/proc/softirqs", "interrupts per core", 1); + + if(!ff) { + char filename[FILENAME_MAX + 1]; + snprintf(filename, FILENAME_MAX, "%s%s", global_host_prefix, "/proc/softirqs"); + ff = procfile_open(config_get("plugin:proc:/proc/softirqs", "filename to monitor", filename), " \t", PROCFILE_FLAG_DEFAULT); + } + if(!ff) return 1; + + ff = procfile_readall(ff); + if(!ff) return 0; // we return 0, so that we will retry to open it next time + + uint32_t lines = procfile_lines(ff), l; + uint32_t words = procfile_linewords(ff, 0), w; + + // find how many CPUs are there + if(cpus == -1) { + cpus = 0; + for(w = 0; w < words ; w++) { + if(strncmp(procfile_lineword(ff, 0, w), "CPU", 3) == 0) + cpus++; + } + + if(cpus > MAX_INTERRUPT_CPUS) cpus = MAX_INTERRUPT_CPUS; + } + + if(!cpus) { + error("PLUGIN: PROC_SOFTIRQS: Cannot find the number of CPUs in /proc/softirqs"); + return 1; + } + + // allocate the size we need; + struct interrupt irrs[lines]; + irrs[0].used = 0; + + // loop through all lines + for(l = 1; l < lines ;l++) { + struct interrupt *irr = &irrs[l]; + irr->used = 0; + irr->total = 0; + + words = procfile_linewords(ff, l); + if(!words) continue; + + irr->id = procfile_lineword(ff, l, 0); + if(!irr->id || !irr->id[0]) continue; + + int idlen = strlen(irr->id); + if(irr->id[idlen - 1] == ':') + irr->id[idlen - 1] = '\0'; + + int c; + for(c = 0; c < cpus ;c++) { + if((c + 1) < (int)words) + irr->value[c] = strtoull(procfile_lineword(ff, l, (uint32_t)(c + 1)), NULL, 10); + else + irr->value[c] = 0; + + irr->total += irr->value[c]; + } + + strncpy(irr->name, irr->id, MAX_INTERRUPT_NAME); + irr->name[MAX_INTERRUPT_NAME] = '\0'; + + irr->used = 1; + } + + RRDSET *st; + + // -------------------------------------------------------------------- + + st = rrdset_find_bytype("system", "softirqs"); + if(!st) { + st = rrdset_create("system", "softirqs", NULL, "softirqs", NULL, "System softirqs", "softirqs/s", 950, update_every, RRDSET_TYPE_STACKED); + + for(l = 0; l < lines ;l++) { + if(!irrs[l].used) continue; + rrddim_add(st, irrs[l].id, irrs[l].name, 1, 1, RRDDIM_INCREMENTAL); + } + } + else rrdset_next(st); + + for(l = 0; l < lines ;l++) { + if(!irrs[l].used) continue; + rrddim_set(st, irrs[l].id, irrs[l].total); + } + rrdset_done(st); + + if(do_per_core) { + int c; + + for(c = 0; c < cpus ; c++) { + char id[256]; + snprintf(id, 256, "cpu%d_softirqs", c); + + st = rrdset_find_bytype("cpu", id); + if(!st) { + // find if everything is zero + unsigned long long core_sum = 0 ; + for(l = 0; l < lines ;l++) { + if(!irrs[l].used) continue; + core_sum += irrs[l].value[c]; + } + if(core_sum == 0) continue; // try next core + + char name[256], title[256]; + snprintf(name, 256, "cpu%d_softirqs", c); + snprintf(title, 256, "CPU%d softirqs", c); + st = rrdset_create("cpu", id, name, "softirqs", "cpu.softirqs", title, "softirqs/s", 3000 + c, update_every, RRDSET_TYPE_STACKED); + + for(l = 0; l < lines ;l++) { + if(!irrs[l].used) continue; + rrddim_add(st, irrs[l].id, irrs[l].name, 1, 1, RRDDIM_INCREMENTAL); + } + } + else rrdset_next(st); + + for(l = 0; l < lines ;l++) { + if(!irrs[l].used) continue; + rrddim_set(st, irrs[l].id, irrs[l].value[c]); + } + rrdset_done(st); + } + } + + return 0; +} diff --git a/src/proc_stat.c b/src/proc_stat.c new file mode 100755 index 000000000..3d090ed71 --- /dev/null +++ b/src/proc_stat.c @@ -0,0 +1,204 @@ +#ifdef HAVE_CONFIG_H +#include <config.h> +#endif +#include <stdio.h> +#include <stdlib.h> +#include <string.h> + +#include "common.h" +#include "log.h" +#include "appconfig.h" +#include "procfile.h" +#include "rrd.h" +#include "plugin_proc.h" + +#define RRD_TYPE_STAT "cpu" +#define RRD_TYPE_STAT_LEN strlen(RRD_TYPE_STAT) + +int do_proc_stat(int update_every, unsigned long long dt) { + static procfile *ff = NULL; + static int do_cpu = -1, do_cpu_cores = -1, do_interrupts = -1, do_context = -1, do_forks = -1, do_processes = -1; + + if(do_cpu == -1) do_cpu = config_get_boolean("plugin:proc:/proc/stat", "cpu utilization", 1); + if(do_cpu_cores == -1) do_cpu_cores = config_get_boolean("plugin:proc:/proc/stat", "per cpu core utilization", 1); + if(do_interrupts == -1) do_interrupts = config_get_boolean("plugin:proc:/proc/stat", "cpu interrupts", 1); + if(do_context == -1) do_context = config_get_boolean("plugin:proc:/proc/stat", "context switches", 1); + if(do_forks == -1) do_forks = config_get_boolean("plugin:proc:/proc/stat", "processes started", 1); + if(do_processes == -1) do_processes = config_get_boolean("plugin:proc:/proc/stat", "processes running", 1); + + if(dt) {}; + + if(!ff) { + char filename[FILENAME_MAX + 1]; + snprintf(filename, FILENAME_MAX, "%s%s", global_host_prefix, "/proc/stat"); + ff = procfile_open(config_get("plugin:proc:/proc/stat", "filename to monitor", filename), " \t:", PROCFILE_FLAG_DEFAULT); + } + if(!ff) return 1; + + ff = procfile_readall(ff); + if(!ff) return 0; // we return 0, so that we will retry to open it next time + + uint32_t lines = procfile_lines(ff), l; + uint32_t words; + + unsigned long long processes = 0, running = 0 , blocked = 0; + RRDSET *st; + + for(l = 0; l < lines ;l++) { + if(strncmp(procfile_lineword(ff, l, 0), "cpu", 3) == 0) { + words = procfile_linewords(ff, l); + if(words < 10) { + error("Cannot read /proc/stat cpu line. Expected 10 params, read %d.", words); + continue; + } + + char *id; + unsigned long long user = 0, nice = 0, system = 0, idle = 0, iowait = 0, irq = 0, softirq = 0, steal = 0, guest = 0, guest_nice = 0; + + id = procfile_lineword(ff, l, 0); + user = strtoull(procfile_lineword(ff, l, 1), NULL, 10); + nice = strtoull(procfile_lineword(ff, l, 2), NULL, 10); + system = strtoull(procfile_lineword(ff, l, 3), NULL, 10); + idle = strtoull(procfile_lineword(ff, l, 4), NULL, 10); + iowait = strtoull(procfile_lineword(ff, l, 5), NULL, 10); + irq = strtoull(procfile_lineword(ff, l, 6), NULL, 10); + softirq = strtoull(procfile_lineword(ff, l, 7), NULL, 10); + steal = strtoull(procfile_lineword(ff, l, 8), NULL, 10); + guest = strtoull(procfile_lineword(ff, l, 9), NULL, 10); + if(words >= 11) guest_nice = strtoull(procfile_lineword(ff, l, 10), NULL, 10); + + char *title = "Core utilization"; + char *type = RRD_TYPE_STAT; + char *context = "cpu.cpu"; + char *family = "utilization"; + long priority = 1000; + int isthistotal = 0; + + if(strcmp(id, "cpu") == 0) { + isthistotal = 1; + type = "system"; + title = "Total CPU utilization"; + context = "system.cpu"; + family = id; + priority = 100; + } + + if((isthistotal && do_cpu) || (!isthistotal && do_cpu_cores)) { + st = rrdset_find_bytype(type, id); + if(!st) { + st = rrdset_create(type, id, NULL, family, context, title, "percentage", priority, update_every, RRDSET_TYPE_STACKED); + + long multiplier = 1; + long divisor = 1; // sysconf(_SC_CLK_TCK); + + rrddim_add(st, "guest_nice", NULL, multiplier, divisor, RRDDIM_PCENT_OVER_DIFF_TOTAL); + rrddim_add(st, "guest", NULL, multiplier, divisor, RRDDIM_PCENT_OVER_DIFF_TOTAL); + rrddim_add(st, "steal", NULL, multiplier, divisor, RRDDIM_PCENT_OVER_DIFF_TOTAL); + rrddim_add(st, "softirq", NULL, multiplier, divisor, RRDDIM_PCENT_OVER_DIFF_TOTAL); + rrddim_add(st, "irq", NULL, multiplier, divisor, RRDDIM_PCENT_OVER_DIFF_TOTAL); + rrddim_add(st, "user", NULL, multiplier, divisor, RRDDIM_PCENT_OVER_DIFF_TOTAL); + rrddim_add(st, "system", NULL, multiplier, divisor, RRDDIM_PCENT_OVER_DIFF_TOTAL); + rrddim_add(st, "nice", NULL, multiplier, divisor, RRDDIM_PCENT_OVER_DIFF_TOTAL); + rrddim_add(st, "iowait", NULL, multiplier, divisor, RRDDIM_PCENT_OVER_DIFF_TOTAL); + + rrddim_add(st, "idle", NULL, multiplier, divisor, RRDDIM_PCENT_OVER_DIFF_TOTAL); + rrddim_hide(st, "idle"); + } + else rrdset_next(st); + + rrddim_set(st, "user", user); + rrddim_set(st, "nice", nice); + rrddim_set(st, "system", system); + rrddim_set(st, "idle", idle); + rrddim_set(st, "iowait", iowait); + rrddim_set(st, "irq", irq); + rrddim_set(st, "softirq", softirq); + rrddim_set(st, "steal", steal); + rrddim_set(st, "guest", guest); + rrddim_set(st, "guest_nice", guest_nice); + rrdset_done(st); + } + } + else if(strcmp(procfile_lineword(ff, l, 0), "intr") == 0) { + unsigned long long value = strtoull(procfile_lineword(ff, l, 1), NULL, 10); + + // -------------------------------------------------------------------- + + if(do_interrupts) { + st = rrdset_find_bytype("system", "intr"); + if(!st) { + st = rrdset_create("system", "intr", NULL, "interrupts", NULL, "CPU Interrupts", "interrupts/s", 900, update_every, RRDSET_TYPE_LINE); + st->isdetail = 1; + + rrddim_add(st, "interrupts", NULL, 1, 1, RRDDIM_INCREMENTAL); + } + else rrdset_next(st); + + rrddim_set(st, "interrupts", value); + rrdset_done(st); + } + } + else if(strcmp(procfile_lineword(ff, l, 0), "ctxt") == 0) { + unsigned long long value = strtoull(procfile_lineword(ff, l, 1), NULL, 10); + + // -------------------------------------------------------------------- + + if(do_context) { + st = rrdset_find_bytype("system", "ctxt"); + if(!st) { + st = rrdset_create("system", "ctxt", NULL, "processes", NULL, "CPU Context Switches", "context switches/s", 800, update_every, RRDSET_TYPE_LINE); + + rrddim_add(st, "switches", NULL, 1, 1, RRDDIM_INCREMENTAL); + } + else rrdset_next(st); + + rrddim_set(st, "switches", value); + rrdset_done(st); + } + } + else if(!processes && strcmp(procfile_lineword(ff, l, 0), "processes") == 0) { + processes = strtoull(procfile_lineword(ff, l, 1), NULL, 10); + } + else if(!running && strcmp(procfile_lineword(ff, l, 0), "procs_running") == 0) { + running = strtoull(procfile_lineword(ff, l, 1), NULL, 10); + } + else if(!blocked && strcmp(procfile_lineword(ff, l, 0), "procs_blocked") == 0) { + blocked = strtoull(procfile_lineword(ff, l, 1), NULL, 10); + } + } + + // -------------------------------------------------------------------- + + if(do_forks) { + st = rrdset_find_bytype("system", "forks"); + if(!st) { + st = rrdset_create("system", "forks", NULL, "processes", NULL, "Started Processes", "processes/s", 700, update_every, RRDSET_TYPE_LINE); + st->isdetail = 1; + + rrddim_add(st, "started", NULL, 1, 1, RRDDIM_INCREMENTAL); + } + else rrdset_next(st); + + rrddim_set(st, "started", processes); + rrdset_done(st); + } + + // -------------------------------------------------------------------- + + if(do_processes) { + st = rrdset_find_bytype("system", "processes"); + if(!st) { + st = rrdset_create("system", "processes", NULL, "processes", NULL, "System Processes", "processes", 600, update_every, RRDSET_TYPE_LINE); + + rrddim_add(st, "running", NULL, 1, 1, RRDDIM_ABSOLUTE); + rrddim_add(st, "blocked", NULL, -1, 1, RRDDIM_ABSOLUTE); + } + else rrdset_next(st); + + rrddim_set(st, "running", running); + rrddim_set(st, "blocked", blocked); + rrdset_done(st); + } + + return 0; +} diff --git a/src/proc_sys_kernel_random_entropy_avail.c b/src/proc_sys_kernel_random_entropy_avail.c new file mode 100755 index 000000000..be9070aca --- /dev/null +++ b/src/proc_sys_kernel_random_entropy_avail.c @@ -0,0 +1,41 @@ +#ifdef HAVE_CONFIG_H +#include <config.h> +#endif +#include <stdio.h> +#include <stdlib.h> + +#include "common.h" +#include "appconfig.h" +#include "procfile.h" +#include "rrd.h" +#include "plugin_proc.h" + +int do_proc_sys_kernel_random_entropy_avail(int update_every, unsigned long long dt) { + static procfile *ff = NULL; + + if(dt) {} ; + + if(!ff) { + char filename[FILENAME_MAX + 1]; + snprintf(filename, FILENAME_MAX, "%s%s", global_host_prefix, "/proc/sys/kernel/random/entropy_avail"); + ff = procfile_open(config_get("plugin:proc:/proc/sys/kernel/random/entropy_avail", "filename to monitor", filename), "", PROCFILE_FLAG_DEFAULT); + } + if(!ff) return 1; + + ff = procfile_readall(ff); + if(!ff) return 0; // we return 0, so that we will retry to open it next time + + unsigned long long entropy = strtoull(procfile_lineword(ff, 0, 0), NULL, 10); + + RRDSET *st = rrdset_find_bytype("system", "entropy"); + if(!st) { + st = rrdset_create("system", "entropy", NULL, "entropy", NULL, "Available Entropy", "entropy", 1000, update_every, RRDSET_TYPE_LINE); + rrddim_add(st, "entropy", NULL, 1, 1, RRDDIM_ABSOLUTE); + } + else rrdset_next(st); + + rrddim_set(st, "entropy", entropy); + rrdset_done(st); + + return 0; +} diff --git a/src/proc_vmstat.c b/src/proc_vmstat.c new file mode 100755 index 000000000..46ddbae7f --- /dev/null +++ b/src/proc_vmstat.c @@ -0,0 +1,486 @@ +#ifdef HAVE_CONFIG_H +#include <config.h> +#endif +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <unistd.h> + +#include "common.h" +#include "log.h" +#include "appconfig.h" +#include "procfile.h" +#include "rrd.h" +#include "plugin_proc.h" + +#define MAX_PROC_VMSTAT_LINE 4096 +#define MAX_PROC_VMSTAT_NAME 1024 + +int do_proc_vmstat(int update_every, unsigned long long dt) { + static procfile *ff = NULL; + static int do_swapio = -1, do_io = -1, do_pgfaults = -1, gen_hashes = -1; + + // static uint32_t hash_allocstall = -1; + // static uint32_t hash_compact_blocks_moved = -1; + // static uint32_t hash_compact_fail = -1; + // static uint32_t hash_compact_pagemigrate_failed = -1; + // static uint32_t hash_compact_pages_moved = -1; + // static uint32_t hash_compact_stall = -1; + // static uint32_t hash_compact_success = -1; + // static uint32_t hash_htlb_buddy_alloc_fail = -1; + // static uint32_t hash_htlb_buddy_alloc_success = -1; + // static uint32_t hash_kswapd_high_wmark_hit_quickly = -1; + // static uint32_t hash_kswapd_inodesteal = -1; + // static uint32_t hash_kswapd_low_wmark_hit_quickly = -1; + // static uint32_t hash_kswapd_skip_congestion_wait = -1; + // static uint32_t hash_nr_active_anon = -1; + // static uint32_t hash_nr_active_file = -1; + // static uint32_t hash_nr_anon_pages = -1; + // static uint32_t hash_nr_anon_transparent_hugepages = -1; + // static uint32_t hash_nr_bounce = -1; + // static uint32_t hash_nr_dirtied = -1; + // static uint32_t hash_nr_dirty = -1; + // static uint32_t hash_nr_dirty_background_threshold = -1; + // static uint32_t hash_nr_dirty_threshold = -1; + // static uint32_t hash_nr_file_pages = -1; + // static uint32_t hash_nr_free_pages = -1; + // static uint32_t hash_nr_inactive_anon = -1; + // static uint32_t hash_nr_inactive_file = -1; + // static uint32_t hash_nr_isolated_anon = -1; + // static uint32_t hash_nr_isolated_file = -1; + // static uint32_t hash_nr_kernel_stack = -1; + // static uint32_t hash_nr_mapped = -1; + // static uint32_t hash_nr_mlock = -1; + // static uint32_t hash_nr_page_table_pages = -1; + // static uint32_t hash_nr_shmem = -1; + // static uint32_t hash_nr_slab_reclaimable = -1; + // static uint32_t hash_nr_slab_unreclaimable = -1; + // static uint32_t hash_nr_unevictable = -1; + // static uint32_t hash_nr_unstable = -1; + // static uint32_t hash_nr_vmscan_immediate_reclaim = -1; + // static uint32_t hash_nr_vmscan_write = -1; + // static uint32_t hash_nr_writeback = -1; + // static uint32_t hash_nr_writeback_temp = -1; + // static uint32_t hash_nr_written = -1; + // static uint32_t hash_pageoutrun = -1; + // static uint32_t hash_pgactivate = -1; + // static uint32_t hash_pgalloc_dma = -1; + // static uint32_t hash_pgalloc_dma32 = -1; + // static uint32_t hash_pgalloc_movable = -1; + // static uint32_t hash_pgalloc_normal = -1; + // static uint32_t hash_pgdeactivate = -1; + static uint32_t hash_pgfault = -1; + // static uint32_t hash_pgfree = -1; + // static uint32_t hash_pginodesteal = -1; + static uint32_t hash_pgmajfault = -1; + static uint32_t hash_pgpgin = -1; + static uint32_t hash_pgpgout = -1; + // static uint32_t hash_pgrefill_dma = -1; + // static uint32_t hash_pgrefill_dma32 = -1; + // static uint32_t hash_pgrefill_movable = -1; + // static uint32_t hash_pgrefill_normal = -1; + // static uint32_t hash_pgrotated = -1; + // static uint32_t hash_pgscan_direct_dma = -1; + // static uint32_t hash_pgscan_direct_dma32 = -1; + // static uint32_t hash_pgscan_direct_movable = -1; + // static uint32_t hash_pgscan_direct_normal = -1; + // static uint32_t hash_pgscan_kswapd_dma = -1; + // static uint32_t hash_pgscan_kswapd_dma32 = -1; + // static uint32_t hash_pgscan_kswapd_movable = -1; + // static uint32_t hash_pgscan_kswapd_normal = -1; + // static uint32_t hash_pgsteal_direct_dma = -1; + // static uint32_t hash_pgsteal_direct_dma32 = -1; + // static uint32_t hash_pgsteal_direct_movable = -1; + // static uint32_t hash_pgsteal_direct_normal = -1; + // static uint32_t hash_pgsteal_kswapd_dma = -1; + // static uint32_t hash_pgsteal_kswapd_dma32 = -1; + // static uint32_t hash_pgsteal_kswapd_movable = -1; + // static uint32_t hash_pgsteal_kswapd_normal = -1; + static uint32_t hash_pswpin = -1; + static uint32_t hash_pswpout = -1; + // static uint32_t hash_slabs_scanned = -1; + // static uint32_t hash_thp_collapse_alloc = -1; + // static uint32_t hash_thp_collapse_alloc_failed = -1; + // static uint32_t hash_thp_fault_alloc = -1; + // static uint32_t hash_thp_fault_fallback = -1; + // static uint32_t hash_thp_split = -1; + // static uint32_t hash_unevictable_pgs_cleared = -1; + // static uint32_t hash_unevictable_pgs_culled = -1; + // static uint32_t hash_unevictable_pgs_mlocked = -1; + // static uint32_t hash_unevictable_pgs_mlockfreed = -1; + // static uint32_t hash_unevictable_pgs_munlocked = -1; + // static uint32_t hash_unevictable_pgs_rescued = -1; + // static uint32_t hash_unevictable_pgs_scanned = -1; + // static uint32_t hash_unevictable_pgs_stranded = -1; + + if(gen_hashes != 1) { + gen_hashes = 1; + // hash_allocstall = simple_hash("allocstall"); + // hash_compact_blocks_moved = simple_hash("compact_blocks_moved"); + // hash_compact_fail = simple_hash("compact_fail"); + // hash_compact_pagemigrate_failed = simple_hash("compact_pagemigrate_failed"); + // hash_compact_pages_moved = simple_hash("compact_pages_moved"); + // hash_compact_stall = simple_hash("compact_stall"); + // hash_compact_success = simple_hash("compact_success"); + // hash_htlb_buddy_alloc_fail = simple_hash("htlb_buddy_alloc_fail"); + // hash_htlb_buddy_alloc_success = simple_hash("htlb_buddy_alloc_success"); + // hash_kswapd_high_wmark_hit_quickly = simple_hash("kswapd_high_wmark_hit_quickly"); + // hash_kswapd_inodesteal = simple_hash("kswapd_inodesteal"); + // hash_kswapd_low_wmark_hit_quickly = simple_hash("kswapd_low_wmark_hit_quickly"); + // hash_kswapd_skip_congestion_wait = simple_hash("kswapd_skip_congestion_wait"); + // hash_nr_active_anon = simple_hash("nr_active_anon"); + // hash_nr_active_file = simple_hash("nr_active_file"); + // hash_nr_anon_pages = simple_hash("nr_anon_pages"); + // hash_nr_anon_transparent_hugepages = simple_hash("nr_anon_transparent_hugepages"); + // hash_nr_bounce = simple_hash("nr_bounce"); + // hash_nr_dirtied = simple_hash("nr_dirtied"); + // hash_nr_dirty = simple_hash("nr_dirty"); + // hash_nr_dirty_background_threshold = simple_hash("nr_dirty_background_threshold"); + // hash_nr_dirty_threshold = simple_hash("nr_dirty_threshold"); + // hash_nr_file_pages = simple_hash("nr_file_pages"); + // hash_nr_free_pages = simple_hash("nr_free_pages"); + // hash_nr_inactive_anon = simple_hash("nr_inactive_anon"); + // hash_nr_inactive_file = simple_hash("nr_inactive_file"); + // hash_nr_isolated_anon = simple_hash("nr_isolated_anon"); + // hash_nr_isolated_file = simple_hash("nr_isolated_file"); + // hash_nr_kernel_stack = simple_hash("nr_kernel_stack"); + // hash_nr_mapped = simple_hash("nr_mapped"); + // hash_nr_mlock = simple_hash("nr_mlock"); + // hash_nr_page_table_pages = simple_hash("nr_page_table_pages"); + // hash_nr_shmem = simple_hash("nr_shmem"); + // hash_nr_slab_reclaimable = simple_hash("nr_slab_reclaimable"); + // hash_nr_slab_unreclaimable = simple_hash("nr_slab_unreclaimable"); + // hash_nr_unevictable = simple_hash("nr_unevictable"); + // hash_nr_unstable = simple_hash("nr_unstable"); + // hash_nr_vmscan_immediate_reclaim = simple_hash("nr_vmscan_immediate_reclaim"); + // hash_nr_vmscan_write = simple_hash("nr_vmscan_write"); + // hash_nr_writeback = simple_hash("nr_writeback"); + // hash_nr_writeback_temp = simple_hash("nr_writeback_temp"); + // hash_nr_written = simple_hash("nr_written"); + // hash_pageoutrun = simple_hash("pageoutrun"); + // hash_pgactivate = simple_hash("pgactivate"); + // hash_pgalloc_dma = simple_hash("pgalloc_dma"); + // hash_pgalloc_dma32 = simple_hash("pgalloc_dma32"); + // hash_pgalloc_movable = simple_hash("pgalloc_movable"); + // hash_pgalloc_normal = simple_hash("pgalloc_normal"); + // hash_pgdeactivate = simple_hash("pgdeactivate"); + hash_pgfault = simple_hash("pgfault"); + // hash_pgfree = simple_hash("pgfree"); + // hash_pginodesteal = simple_hash("pginodesteal"); + hash_pgmajfault = simple_hash("pgmajfault"); + hash_pgpgin = simple_hash("pgpgin"); + hash_pgpgout = simple_hash("pgpgout"); + // hash_pgrefill_dma = simple_hash("pgrefill_dma"); + // hash_pgrefill_dma32 = simple_hash("pgrefill_dma32"); + // hash_pgrefill_movable = simple_hash("pgrefill_movable"); + // hash_pgrefill_normal = simple_hash("pgrefill_normal"); + // hash_pgrotated = simple_hash("pgrotated"); + // hash_pgscan_direct_dma = simple_hash("pgscan_direct_dma"); + // hash_pgscan_direct_dma32 = simple_hash("pgscan_direct_dma32"); + // hash_pgscan_direct_movable = simple_hash("pgscan_direct_movable"); + // hash_pgscan_direct_normal = simple_hash("pgscan_direct_normal"); + // hash_pgscan_kswapd_dma = simple_hash("pgscan_kswapd_dma"); + // hash_pgscan_kswapd_dma32 = simple_hash("pgscan_kswapd_dma32"); + // hash_pgscan_kswapd_movable = simple_hash("pgscan_kswapd_movable"); + // hash_pgscan_kswapd_normal = simple_hash("pgscan_kswapd_normal"); + // hash_pgsteal_direct_dma = simple_hash("pgsteal_direct_dma"); + // hash_pgsteal_direct_dma32 = simple_hash("pgsteal_direct_dma32"); + // hash_pgsteal_direct_movable = simple_hash("pgsteal_direct_movable"); + // hash_pgsteal_direct_normal = simple_hash("pgsteal_direct_normal"); + // hash_pgsteal_kswapd_dma = simple_hash("pgsteal_kswapd_dma"); + // hash_pgsteal_kswapd_dma32 = simple_hash("pgsteal_kswapd_dma32"); + // hash_pgsteal_kswapd_movable = simple_hash("pgsteal_kswapd_movable"); + // hash_pgsteal_kswapd_normal = simple_hash("pgsteal_kswapd_normal"); + hash_pswpin = simple_hash("pswpin"); + hash_pswpout = simple_hash("pswpout"); + // hash_slabs_scanned = simple_hash("slabs_scanned"); + // hash_thp_collapse_alloc = simple_hash("thp_collapse_alloc"); + // hash_thp_collapse_alloc_failed = simple_hash("thp_collapse_alloc_failed"); + // hash_thp_fault_alloc = simple_hash("thp_fault_alloc"); + // hash_thp_fault_fallback = simple_hash("thp_fault_fallback"); + // hash_thp_split = simple_hash("thp_split"); + // hash_unevictable_pgs_cleared = simple_hash("unevictable_pgs_cleared"); + // hash_unevictable_pgs_culled = simple_hash("unevictable_pgs_culled"); + // hash_unevictable_pgs_mlocked = simple_hash("unevictable_pgs_mlocked"); + // hash_unevictable_pgs_mlockfreed = simple_hash("unevictable_pgs_mlockfreed"); + // hash_unevictable_pgs_munlocked = simple_hash("unevictable_pgs_munlocked"); + // hash_unevictable_pgs_rescued = simple_hash("unevictable_pgs_rescued"); + // hash_unevictable_pgs_scanned = simple_hash("unevictable_pgs_scanned"); + // hash_unevictable_pgs_stranded = simple_hash("unevictable_pgs_stranded"); + } + + if(do_swapio == -1) do_swapio = config_get_boolean("plugin:proc:/proc/vmstat", "swap i/o", 1); + if(do_io == -1) do_io = config_get_boolean("plugin:proc:/proc/vmstat", "disk i/o", 1); + if(do_pgfaults == -1) do_pgfaults = config_get_boolean("plugin:proc:/proc/vmstat", "memory page faults", 1); + + if(dt) {}; + + if(!ff) { + char filename[FILENAME_MAX + 1]; + snprintf(filename, FILENAME_MAX, "%s%s", global_host_prefix, "/proc/vmstat"); + ff = procfile_open(config_get("plugin:proc:/proc/vmstat", "filename to monitor", filename), " \t:", PROCFILE_FLAG_DEFAULT); + } + if(!ff) return 1; + + ff = procfile_readall(ff); + if(!ff) return 0; // we return 0, so that we will retry to open it next time + + uint32_t lines = procfile_lines(ff), l; + uint32_t words; + + // unsigned long long allocstall = 0ULL; + // unsigned long long compact_blocks_moved = 0ULL; + // unsigned long long compact_fail = 0ULL; + // unsigned long long compact_pagemigrate_failed = 0ULL; + // unsigned long long compact_pages_moved = 0ULL; + // unsigned long long compact_stall = 0ULL; + // unsigned long long compact_success = 0ULL; + // unsigned long long htlb_buddy_alloc_fail = 0ULL; + // unsigned long long htlb_buddy_alloc_success = 0ULL; + // unsigned long long kswapd_high_wmark_hit_quickly = 0ULL; + // unsigned long long kswapd_inodesteal = 0ULL; + // unsigned long long kswapd_low_wmark_hit_quickly = 0ULL; + // unsigned long long kswapd_skip_congestion_wait = 0ULL; + // unsigned long long nr_active_anon = 0ULL; + // unsigned long long nr_active_file = 0ULL; + // unsigned long long nr_anon_pages = 0ULL; + // unsigned long long nr_anon_transparent_hugepages = 0ULL; + // unsigned long long nr_bounce = 0ULL; + // unsigned long long nr_dirtied = 0ULL; + // unsigned long long nr_dirty = 0ULL; + // unsigned long long nr_dirty_background_threshold = 0ULL; + // unsigned long long nr_dirty_threshold = 0ULL; + // unsigned long long nr_file_pages = 0ULL; + // unsigned long long nr_free_pages = 0ULL; + // unsigned long long nr_inactive_anon = 0ULL; + // unsigned long long nr_inactive_file = 0ULL; + // unsigned long long nr_isolated_anon = 0ULL; + // unsigned long long nr_isolated_file = 0ULL; + // unsigned long long nr_kernel_stack = 0ULL; + // unsigned long long nr_mapped = 0ULL; + // unsigned long long nr_mlock = 0ULL; + // unsigned long long nr_page_table_pages = 0ULL; + // unsigned long long nr_shmem = 0ULL; + // unsigned long long nr_slab_reclaimable = 0ULL; + // unsigned long long nr_slab_unreclaimable = 0ULL; + // unsigned long long nr_unevictable = 0ULL; + // unsigned long long nr_unstable = 0ULL; + // unsigned long long nr_vmscan_immediate_reclaim = 0ULL; + // unsigned long long nr_vmscan_write = 0ULL; + // unsigned long long nr_writeback = 0ULL; + // unsigned long long nr_writeback_temp = 0ULL; + // unsigned long long nr_written = 0ULL; + // unsigned long long pageoutrun = 0ULL; + // unsigned long long pgactivate = 0ULL; + // unsigned long long pgalloc_dma = 0ULL; + // unsigned long long pgalloc_dma32 = 0ULL; + // unsigned long long pgalloc_movable = 0ULL; + // unsigned long long pgalloc_normal = 0ULL; + // unsigned long long pgdeactivate = 0ULL; + unsigned long long pgfault = 0ULL; + // unsigned long long pgfree = 0ULL; + // unsigned long long pginodesteal = 0ULL; + unsigned long long pgmajfault = 0ULL; + unsigned long long pgpgin = 0ULL; + unsigned long long pgpgout = 0ULL; + // unsigned long long pgrefill_dma = 0ULL; + // unsigned long long pgrefill_dma32 = 0ULL; + // unsigned long long pgrefill_movable = 0ULL; + // unsigned long long pgrefill_normal = 0ULL; + // unsigned long long pgrotated = 0ULL; + // unsigned long long pgscan_direct_dma = 0ULL; + // unsigned long long pgscan_direct_dma32 = 0ULL; + // unsigned long long pgscan_direct_movable = 0ULL; + // unsigned long long pgscan_direct_normal = 0ULL; + // unsigned long long pgscan_kswapd_dma = 0ULL; + // unsigned long long pgscan_kswapd_dma32 = 0ULL; + // unsigned long long pgscan_kswapd_movable = 0ULL; + // unsigned long long pgscan_kswapd_normal = 0ULL; + // unsigned long long pgsteal_direct_dma = 0ULL; + // unsigned long long pgsteal_direct_dma32 = 0ULL; + // unsigned long long pgsteal_direct_movable = 0ULL; + // unsigned long long pgsteal_direct_normal = 0ULL; + // unsigned long long pgsteal_kswapd_dma = 0ULL; + // unsigned long long pgsteal_kswapd_dma32 = 0ULL; + // unsigned long long pgsteal_kswapd_movable = 0ULL; + // unsigned long long pgsteal_kswapd_normal = 0ULL; + unsigned long long pswpin = 0ULL; + unsigned long long pswpout = 0ULL; + // unsigned long long slabs_scanned = 0ULL; + // unsigned long long thp_collapse_alloc = 0ULL; + // unsigned long long thp_collapse_alloc_failed = 0ULL; + // unsigned long long thp_fault_alloc = 0ULL; + // unsigned long long thp_fault_fallback = 0ULL; + // unsigned long long thp_split = 0ULL; + // unsigned long long unevictable_pgs_cleared = 0ULL; + // unsigned long long unevictable_pgs_culled = 0ULL; + // unsigned long long unevictable_pgs_mlocked = 0ULL; + // unsigned long long unevictable_pgs_mlockfreed = 0ULL; + // unsigned long long unevictable_pgs_munlocked = 0ULL; + // unsigned long long unevictable_pgs_rescued = 0ULL; + // unsigned long long unevictable_pgs_scanned = 0ULL; + // unsigned long long unevictable_pgs_stranded = 0ULL; + + for(l = 0; l < lines ;l++) { + words = procfile_linewords(ff, l); + if(words < 2) { + if(words) error("Cannot read /proc/vmstat line %d. Expected 2 params, read %d.", l, words); + continue; + } + + char *name = procfile_lineword(ff, l, 0); + char * value = procfile_lineword(ff, l, 1); + if(!name || !*name || !value || !*value) continue; + + uint32_t hash = simple_hash(name); + + if(0) ; + // else if(hash == hash_allocstall && strcmp(name, "allocstall") == 0) allocstall = strtoull(value, NULL, 10); + // else if(hash == hash_compact_blocks_moved && strcmp(name, "compact_blocks_moved") == 0) compact_blocks_moved = strtoull(value, NULL, 10); + // else if(hash == hash_compact_fail && strcmp(name, "compact_fail") == 0) compact_fail = strtoull(value, NULL, 10); + // else if(hash == hash_compact_pagemigrate_failed && strcmp(name, "compact_pagemigrate_failed") == 0) compact_pagemigrate_failed = strtoull(value, NULL, 10); + // else if(hash == hash_compact_pages_moved && strcmp(name, "compact_pages_moved") == 0) compact_pages_moved = strtoull(value, NULL, 10); + // else if(hash == hash_compact_stall && strcmp(name, "compact_stall") == 0) compact_stall = strtoull(value, NULL, 10); + // else if(hash == hash_compact_success && strcmp(name, "compact_success") == 0) compact_success = strtoull(value, NULL, 10); + // else if(hash == hash_htlb_buddy_alloc_fail && strcmp(name, "htlb_buddy_alloc_fail") == 0) htlb_buddy_alloc_fail = strtoull(value, NULL, 10); + // else if(hash == hash_htlb_buddy_alloc_success && strcmp(name, "htlb_buddy_alloc_success") == 0) htlb_buddy_alloc_success = strtoull(value, NULL, 10); + // else if(hash == hash_kswapd_high_wmark_hit_quickly && strcmp(name, "kswapd_high_wmark_hit_quickly") == 0) kswapd_high_wmark_hit_quickly = strtoull(value, NULL, 10); + // else if(hash == hash_kswapd_inodesteal && strcmp(name, "kswapd_inodesteal") == 0) kswapd_inodesteal = strtoull(value, NULL, 10); + // else if(hash == hash_kswapd_low_wmark_hit_quickly && strcmp(name, "kswapd_low_wmark_hit_quickly") == 0) kswapd_low_wmark_hit_quickly = strtoull(value, NULL, 10); + // else if(hash == hash_kswapd_skip_congestion_wait && strcmp(name, "kswapd_skip_congestion_wait") == 0) kswapd_skip_congestion_wait = strtoull(value, NULL, 10); + // else if(hash == hash_nr_active_anon && strcmp(name, "nr_active_anon") == 0) nr_active_anon = strtoull(value, NULL, 10); + // else if(hash == hash_nr_active_file && strcmp(name, "nr_active_file") == 0) nr_active_file = strtoull(value, NULL, 10); + // else if(hash == hash_nr_anon_pages && strcmp(name, "nr_anon_pages") == 0) nr_anon_pages = strtoull(value, NULL, 10); + // else if(hash == hash_nr_anon_transparent_hugepages && strcmp(name, "nr_anon_transparent_hugepages") == 0) nr_anon_transparent_hugepages = strtoull(value, NULL, 10); + // else if(hash == hash_nr_bounce && strcmp(name, "nr_bounce") == 0) nr_bounce = strtoull(value, NULL, 10); + // else if(hash == hash_nr_dirtied && strcmp(name, "nr_dirtied") == 0) nr_dirtied = strtoull(value, NULL, 10); + // else if(hash == hash_nr_dirty && strcmp(name, "nr_dirty") == 0) nr_dirty = strtoull(value, NULL, 10); + // else if(hash == hash_nr_dirty_background_threshold && strcmp(name, "nr_dirty_background_threshold") == 0) nr_dirty_background_threshold = strtoull(value, NULL, 10); + // else if(hash == hash_nr_dirty_threshold && strcmp(name, "nr_dirty_threshold") == 0) nr_dirty_threshold = strtoull(value, NULL, 10); + // else if(hash == hash_nr_file_pages && strcmp(name, "nr_file_pages") == 0) nr_file_pages = strtoull(value, NULL, 10); + // else if(hash == hash_nr_free_pages && strcmp(name, "nr_free_pages") == 0) nr_free_pages = strtoull(value, NULL, 10); + // else if(hash == hash_nr_inactive_anon && strcmp(name, "nr_inactive_anon") == 0) nr_inactive_anon = strtoull(value, NULL, 10); + // else if(hash == hash_nr_inactive_file && strcmp(name, "nr_inactive_file") == 0) nr_inactive_file = strtoull(value, NULL, 10); + // else if(hash == hash_nr_isolated_anon && strcmp(name, "nr_isolated_anon") == 0) nr_isolated_anon = strtoull(value, NULL, 10); + // else if(hash == hash_nr_isolated_file && strcmp(name, "nr_isolated_file") == 0) nr_isolated_file = strtoull(value, NULL, 10); + // else if(hash == hash_nr_kernel_stack && strcmp(name, "nr_kernel_stack") == 0) nr_kernel_stack = strtoull(value, NULL, 10); + // else if(hash == hash_nr_mapped && strcmp(name, "nr_mapped") == 0) nr_mapped = strtoull(value, NULL, 10); + // else if(hash == hash_nr_mlock && strcmp(name, "nr_mlock") == 0) nr_mlock = strtoull(value, NULL, 10); + // else if(hash == hash_nr_page_table_pages && strcmp(name, "nr_page_table_pages") == 0) nr_page_table_pages = strtoull(value, NULL, 10); + // else if(hash == hash_nr_shmem && strcmp(name, "nr_shmem") == 0) nr_shmem = strtoull(value, NULL, 10); + // else if(hash == hash_nr_slab_reclaimable && strcmp(name, "nr_slab_reclaimable") == 0) nr_slab_reclaimable = strtoull(value, NULL, 10); + // else if(hash == hash_nr_slab_unreclaimable && strcmp(name, "nr_slab_unreclaimable") == 0) nr_slab_unreclaimable = strtoull(value, NULL, 10); + // else if(hash == hash_nr_unevictable && strcmp(name, "nr_unevictable") == 0) nr_unevictable = strtoull(value, NULL, 10); + // else if(hash == hash_nr_unstable && strcmp(name, "nr_unstable") == 0) nr_unstable = strtoull(value, NULL, 10); + // else if(hash == hash_nr_vmscan_immediate_reclaim && strcmp(name, "nr_vmscan_immediate_reclaim") == 0) nr_vmscan_immediate_reclaim = strtoull(value, NULL, 10); + // else if(hash == hash_nr_vmscan_write && strcmp(name, "nr_vmscan_write") == 0) nr_vmscan_write = strtoull(value, NULL, 10); + // else if(hash == hash_nr_writeback && strcmp(name, "nr_writeback") == 0) nr_writeback = strtoull(value, NULL, 10); + // else if(hash == hash_nr_writeback_temp && strcmp(name, "nr_writeback_temp") == 0) nr_writeback_temp = strtoull(value, NULL, 10); + // else if(hash == hash_nr_written && strcmp(name, "nr_written") == 0) nr_written = strtoull(value, NULL, 10); + // else if(hash == hash_pageoutrun && strcmp(name, "pageoutrun") == 0) pageoutrun = strtoull(value, NULL, 10); + // else if(hash == hash_pgactivate && strcmp(name, "pgactivate") == 0) pgactivate = strtoull(value, NULL, 10); + // else if(hash == hash_pgalloc_dma && strcmp(name, "pgalloc_dma") == 0) pgalloc_dma = strtoull(value, NULL, 10); + // else if(hash == hash_pgalloc_dma32 && strcmp(name, "pgalloc_dma32") == 0) pgalloc_dma32 = strtoull(value, NULL, 10); + // else if(hash == hash_pgalloc_movable && strcmp(name, "pgalloc_movable") == 0) pgalloc_movable = strtoull(value, NULL, 10); + // else if(hash == hash_pgalloc_normal && strcmp(name, "pgalloc_normal") == 0) pgalloc_normal = strtoull(value, NULL, 10); + // else if(hash == hash_pgdeactivate && strcmp(name, "pgdeactivate") == 0) pgdeactivate = strtoull(value, NULL, 10); + else if(hash == hash_pgfault && strcmp(name, "pgfault") == 0) pgfault = strtoull(value, NULL, 10); + // else if(hash == hash_pgfree && strcmp(name, "pgfree") == 0) pgfree = strtoull(value, NULL, 10); + // else if(hash == hash_pginodesteal && strcmp(name, "pginodesteal") == 0) pginodesteal = strtoull(value, NULL, 10); + else if(hash == hash_pgmajfault && strcmp(name, "pgmajfault") == 0) pgmajfault = strtoull(value, NULL, 10); + else if(hash == hash_pgpgin && strcmp(name, "pgpgin") == 0) pgpgin = strtoull(value, NULL, 10); + else if(hash == hash_pgpgout && strcmp(name, "pgpgout") == 0) pgpgout = strtoull(value, NULL, 10); + // else if(hash == hash_pgrefill_dma && strcmp(name, "pgrefill_dma") == 0) pgrefill_dma = strtoull(value, NULL, 10); + // else if(hash == hash_pgrefill_dma32 && strcmp(name, "pgrefill_dma32") == 0) pgrefill_dma32 = strtoull(value, NULL, 10); + // else if(hash == hash_pgrefill_movable && strcmp(name, "pgrefill_movable") == 0) pgrefill_movable = strtoull(value, NULL, 10); + // else if(hash == hash_pgrefill_normal && strcmp(name, "pgrefill_normal") == 0) pgrefill_normal = strtoull(value, NULL, 10); + // else if(hash == hash_pgrotated && strcmp(name, "pgrotated") == 0) pgrotated = strtoull(value, NULL, 10); + // else if(hash == hash_pgscan_direct_dma && strcmp(name, "pgscan_direct_dma") == 0) pgscan_direct_dma = strtoull(value, NULL, 10); + // else if(hash == hash_pgscan_direct_dma32 && strcmp(name, "pgscan_direct_dma32") == 0) pgscan_direct_dma32 = strtoull(value, NULL, 10); + // else if(hash == hash_pgscan_direct_movable && strcmp(name, "pgscan_direct_movable") == 0) pgscan_direct_movable = strtoull(value, NULL, 10); + // else if(hash == hash_pgscan_direct_normal && strcmp(name, "pgscan_direct_normal") == 0) pgscan_direct_normal = strtoull(value, NULL, 10); + // else if(hash == hash_pgscan_kswapd_dma && strcmp(name, "pgscan_kswapd_dma") == 0) pgscan_kswapd_dma = strtoull(value, NULL, 10); + // else if(hash == hash_pgscan_kswapd_dma32 && strcmp(name, "pgscan_kswapd_dma32") == 0) pgscan_kswapd_dma32 = strtoull(value, NULL, 10); + // else if(hash == hash_pgscan_kswapd_movable && strcmp(name, "pgscan_kswapd_movable") == 0) pgscan_kswapd_movable = strtoull(value, NULL, 10); + // else if(hash == hash_pgscan_kswapd_normal && strcmp(name, "pgscan_kswapd_normal") == 0) pgscan_kswapd_normal = strtoull(value, NULL, 10); + // else if(hash == hash_pgsteal_direct_dma && strcmp(name, "pgsteal_direct_dma") == 0) pgsteal_direct_dma = strtoull(value, NULL, 10); + // else if(hash == hash_pgsteal_direct_dma32 && strcmp(name, "pgsteal_direct_dma32") == 0) pgsteal_direct_dma32 = strtoull(value, NULL, 10); + // else if(hash == hash_pgsteal_direct_movable && strcmp(name, "pgsteal_direct_movable") == 0) pgsteal_direct_movable = strtoull(value, NULL, 10); + // else if(hash == hash_pgsteal_direct_normal && strcmp(name, "pgsteal_direct_normal") == 0) pgsteal_direct_normal = strtoull(value, NULL, 10); + // else if(hash == hash_pgsteal_kswapd_dma && strcmp(name, "pgsteal_kswapd_dma") == 0) pgsteal_kswapd_dma = strtoull(value, NULL, 10); + // else if(hash == hash_pgsteal_kswapd_dma32 && strcmp(name, "pgsteal_kswapd_dma32") == 0) pgsteal_kswapd_dma32 = strtoull(value, NULL, 10); + // else if(hash == hash_pgsteal_kswapd_movable && strcmp(name, "pgsteal_kswapd_movable") == 0) pgsteal_kswapd_movable = strtoull(value, NULL, 10); + // else if(hash == hash_pgsteal_kswapd_normal && strcmp(name, "pgsteal_kswapd_normal") == 0) pgsteal_kswapd_normal = strtoull(value, NULL, 10); + else if(hash == hash_pswpin && strcmp(name, "pswpin") == 0) pswpin = strtoull(value, NULL, 10); + else if(hash == hash_pswpout && strcmp(name, "pswpout") == 0) pswpout = strtoull(value, NULL, 10); + // else if(hash == hash_slabs_scanned && strcmp(name, "slabs_scanned") == 0) slabs_scanned = strtoull(value, NULL, 10); + // else if(hash == hash_thp_collapse_alloc && strcmp(name, "thp_collapse_alloc") == 0) thp_collapse_alloc = strtoull(value, NULL, 10); + // else if(hash == hash_thp_collapse_alloc_failed && strcmp(name, "thp_collapse_alloc_failed") == 0) thp_collapse_alloc_failed = strtoull(value, NULL, 10); + // else if(hash == hash_thp_fault_alloc && strcmp(name, "thp_fault_alloc") == 0) thp_fault_alloc = strtoull(value, NULL, 10); + // else if(hash == hash_thp_fault_fallback && strcmp(name, "thp_fault_fallback") == 0) thp_fault_fallback = strtoull(value, NULL, 10); + // else if(hash == hash_thp_split && strcmp(name, "thp_split") == 0) thp_split = strtoull(value, NULL, 10); + // else if(hash == hash_unevictable_pgs_cleared && strcmp(name, "unevictable_pgs_cleared") == 0) unevictable_pgs_cleared = strtoull(value, NULL, 10); + // else if(hash == hash_unevictable_pgs_culled && strcmp(name, "unevictable_pgs_culled") == 0) unevictable_pgs_culled = strtoull(value, NULL, 10); + // else if(hash == hash_unevictable_pgs_mlocked && strcmp(name, "unevictable_pgs_mlocked") == 0) unevictable_pgs_mlocked = strtoull(value, NULL, 10); + // else if(hash == hash_unevictable_pgs_mlockfreed && strcmp(name, "unevictable_pgs_mlockfreed") == 0) unevictable_pgs_mlockfreed = strtoull(value, NULL, 10); + // else if(hash == hash_unevictable_pgs_munlocked && strcmp(name, "unevictable_pgs_munlocked") == 0) unevictable_pgs_munlocked = strtoull(value, NULL, 10); + // else if(hash == hash_unevictable_pgs_rescued && strcmp(name, "unevictable_pgs_rescued") == 0) unevictable_pgs_rescued = strtoull(value, NULL, 10); + // else if(hash == hash_unevictable_pgs_scanned && strcmp(name, "unevictable_pgs_scanned") == 0) unevictable_pgs_scanned = strtoull(value, NULL, 10); + // else if(hash == hash_unevictable_pgs_stranded && strcmp(name, "unevictable_pgs_stranded") == 0) unevictable_pgs_stranded = strtoull(value, NULL, 10); + } + + // -------------------------------------------------------------------- + + if(do_swapio) { + static RRDSET *st_swapio = NULL; + if(!st_swapio) { + st_swapio = rrdset_create("system", "swapio", NULL, "swap", NULL, "Swap I/O", "kilobytes/s", 250, update_every, RRDSET_TYPE_AREA); + + rrddim_add(st_swapio, "in", NULL, sysconf(_SC_PAGESIZE), 1024, RRDDIM_INCREMENTAL); + rrddim_add(st_swapio, "out", NULL, -sysconf(_SC_PAGESIZE), 1024, RRDDIM_INCREMENTAL); + } + else rrdset_next(st_swapio); + + rrddim_set(st_swapio, "in", pswpin); + rrddim_set(st_swapio, "out", pswpout); + rrdset_done(st_swapio); + } + + // -------------------------------------------------------------------- + + if(do_io) { + static RRDSET *st_io = NULL; + if(!st_io) { + st_io = rrdset_create("system", "io", NULL, "disk", NULL, "Disk I/O", "kilobytes/s", 150, update_every, RRDSET_TYPE_AREA); + + rrddim_add(st_io, "in", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st_io, "out", NULL, -1, 1, RRDDIM_INCREMENTAL); + } + else rrdset_next(st_io); + + rrddim_set(st_io, "in", pgpgin); + rrddim_set(st_io, "out", pgpgout); + rrdset_done(st_io); + } + + // -------------------------------------------------------------------- + + if(do_pgfaults) { + static RRDSET *st_pgfaults = NULL; + if(!st_pgfaults) { + st_pgfaults = rrdset_create("mem", "pgfaults", NULL, "system", NULL, "Memory Page Faults", "page faults/s", 500, update_every, RRDSET_TYPE_LINE); + st_pgfaults->isdetail = 1; + + rrddim_add(st_pgfaults, "minor", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st_pgfaults, "major", NULL, -1, 1, RRDDIM_INCREMENTAL); + } + else rrdset_next(st_pgfaults); + + rrddim_set(st_pgfaults, "minor", pgfault); + rrddim_set(st_pgfaults, "major", pgmajfault); + rrdset_done(st_pgfaults); + } + + return 0; +} + diff --git a/src/procfile.c b/src/procfile.c new file mode 100755 index 000000000..7a4857959 --- /dev/null +++ b/src/procfile.c @@ -0,0 +1,400 @@ +#ifdef HAVE_CONFIG_H +#include <config.h> +#endif +#include <stdio.h> +#include <stdlib.h> +#include <unistd.h> +#include <errno.h> +#include <fcntl.h> +#include <string.h> +#include <malloc.h> +#include <ctype.h> +#include <time.h> +#include <sys/time.h> +#include <sys/wait.h> + +#include <sys/types.h> +#include <sys/stat.h> +#include <sys/mman.h> + +#include "common.h" +#include "log.h" +#include "procfile.h" + +#define PF_PREFIX "PROCFILE" + +#define PFWORDS_INCREASE_STEP 200 +#define PFLINES_INCREASE_STEP 10 +#define PROCFILE_INCREMENT_BUFFER 512 + +int procfile_adaptive_initial_allocation = 0; + +// if adaptive allocation is set, these store the +// max values we have seen so far +uint32_t procfile_max_lines = PFLINES_INCREASE_STEP; +uint32_t procfile_max_words = PFWORDS_INCREASE_STEP; +size_t procfile_max_allocation = PROCFILE_INCREMENT_BUFFER; + +// ---------------------------------------------------------------------------- +// An array of words + + +pfwords *pfwords_add(pfwords *fw, char *str) { + // debug(D_PROCFILE, PF_PREFIX ": adding word No %d: '%s'", fw->len, str); + + if(unlikely(fw->len == fw->size)) { + // debug(D_PROCFILE, PF_PREFIX ": expanding words"); + + pfwords *new = realloc(fw, sizeof(pfwords) + (fw->size + PFWORDS_INCREASE_STEP) * sizeof(char *)); + if(unlikely(!new)) { + error(PF_PREFIX ": failed to expand words"); + free(fw); + return NULL; + } + fw = new; + fw->size += PFWORDS_INCREASE_STEP; + } + + fw->words[fw->len++] = str; + + return fw; +} + +pfwords *pfwords_new(void) { + // debug(D_PROCFILE, PF_PREFIX ": initializing words"); + + uint32_t size = (procfile_adaptive_initial_allocation) ? procfile_max_words : PFWORDS_INCREASE_STEP; + + pfwords *new = malloc(sizeof(pfwords) + size * sizeof(char *)); + if(unlikely(!new)) return NULL; + + new->len = 0; + new->size = size; + return new; +} + +void pfwords_reset(pfwords *fw) { + // debug(D_PROCFILE, PF_PREFIX ": reseting words"); + fw->len = 0; +} + +void pfwords_free(pfwords *fw) { + // debug(D_PROCFILE, PF_PREFIX ": freeing words"); + + free(fw); +} + + +// ---------------------------------------------------------------------------- +// An array of lines + +pflines *pflines_add(pflines *fl, uint32_t first_word) { + // debug(D_PROCFILE, PF_PREFIX ": adding line %d at word %d", fl->len, first_word); + + if(unlikely(fl->len == fl->size)) { + // debug(D_PROCFILE, PF_PREFIX ": expanding lines"); + + pflines *new = realloc(fl, sizeof(pflines) + (fl->size + PFLINES_INCREASE_STEP) * sizeof(ffline)); + if(unlikely(!new)) { + error(PF_PREFIX ": failed to expand lines"); + free(fl); + return NULL; + } + fl = new; + fl->size += PFLINES_INCREASE_STEP; + } + + fl->lines[fl->len].words = 0; + fl->lines[fl->len++].first = first_word; + + return fl; +} + +pflines *pflines_new(void) { + // debug(D_PROCFILE, PF_PREFIX ": initializing lines"); + + uint32_t size = (unlikely(procfile_adaptive_initial_allocation)) ? procfile_max_words : PFLINES_INCREASE_STEP; + + pflines *new = malloc(sizeof(pflines) + size * sizeof(ffline)); + if(unlikely(!new)) return NULL; + + new->len = 0; + new->size = size; + return new; +} + +void pflines_reset(pflines *fl) { + // debug(D_PROCFILE, PF_PREFIX ": reseting lines"); + + fl->len = 0; +} + +void pflines_free(pflines *fl) { + // debug(D_PROCFILE, PF_PREFIX ": freeing lines"); + + free(fl); +} + + +// ---------------------------------------------------------------------------- +// The procfile + +#define PF_CHAR_IS_SEPARATOR ' ' +#define PF_CHAR_IS_NEWLINE 'N' +#define PF_CHAR_IS_WORD 'W' + +void procfile_close(procfile *ff) { + debug(D_PROCFILE, PF_PREFIX ": Closing file '%s'", ff->filename); + + if(likely(ff->lines)) pflines_free(ff->lines); + if(likely(ff->words)) pfwords_free(ff->words); + + if(likely(ff->fd != -1)) close(ff->fd); + free(ff); +} + +procfile *procfile_parser(procfile *ff) { + debug(D_PROCFILE, PF_PREFIX ": Parsing file '%s'", ff->filename); + + char *s = ff->data, *e = ff->data, *t = ff->data; + uint32_t l = 0, w = 0; + e += ff->len; + + ff->lines = pflines_add(ff->lines, w); + if(unlikely(!ff->lines)) goto cleanup; + + while(likely(s < e)) { + switch(ff->separators[(int)(*s)]) { + case PF_CHAR_IS_SEPARATOR: + if(likely(s == t)) { + // skip all leading white spaces + t = ++s; + continue; + } + + // end of word + *s = '\0'; + + ff->words = pfwords_add(ff->words, t); + if(unlikely(!ff->words)) goto cleanup; + + ff->lines->lines[l].words++; + w++; + + t = ++s; + continue; + + case PF_CHAR_IS_NEWLINE: + // end of line + *s = '\0'; + + ff->words = pfwords_add(ff->words, t); + if(unlikely(!ff->words)) goto cleanup; + + ff->lines->lines[l].words++; + w++; + + // debug(D_PROCFILE, PF_PREFIX ": ended line %d with %d words", l, ff->lines->lines[l].words); + + ff->lines = pflines_add(ff->lines, w); + if(unlikely(!ff->lines)) goto cleanup; + l++; + + t = ++s; + continue; + + default: + s++; + continue; + } + } + + if(likely(s != t)) { + // the last word + if(likely(ff->len < ff->size)) *s = '\0'; + else { + // we are going to loose the last byte + ff->data[ff->size - 1] = '\0'; + } + + ff->words = pfwords_add(ff->words, t); + if(unlikely(!ff->words)) goto cleanup; + + ff->lines->lines[l].words++; + w++; + } + + return ff; + +cleanup: + error(PF_PREFIX ": Failed to parse file '%s'", ff->filename); + procfile_close(ff); + return NULL; +} + +procfile *procfile_readall(procfile *ff) { + debug(D_PROCFILE, PF_PREFIX ": Reading file '%s'.", ff->filename); + + ssize_t s, r = 1, x; + ff->len = 0; + + while(likely(r > 0)) { + s = ff->len; + x = ff->size - s; + + if(!x) { + debug(D_PROCFILE, PF_PREFIX ": Expanding data buffer for file '%s'.", ff->filename); + + procfile *new = realloc(ff, sizeof(procfile) + ff->size + PROCFILE_INCREMENT_BUFFER); + if(unlikely(!new)) { + error(PF_PREFIX ": Cannot allocate memory for file '%s'", ff->filename); + procfile_close(ff); + return NULL; + } + ff = new; + ff->size += PROCFILE_INCREMENT_BUFFER; + } + + debug(D_PROCFILE, "Reading file '%s', from position %ld with length %ld", ff->filename, s, ff->size - s); + r = read(ff->fd, &ff->data[s], ff->size - s); + if(unlikely(r == -1)) { + if(unlikely(!(ff->flags & PROCFILE_FLAG_NO_ERROR_ON_FILE_IO))) error(PF_PREFIX ": Cannot read from file '%s'", ff->filename); + procfile_close(ff); + return NULL; + } + + ff->len += r; + } + + debug(D_PROCFILE, "Rewinding file '%s'", ff->filename); + if(unlikely(lseek(ff->fd, 0, SEEK_SET) == -1)) { + if(unlikely(!(ff->flags & PROCFILE_FLAG_NO_ERROR_ON_FILE_IO))) error(PF_PREFIX ": Cannot rewind on file '%s'.", ff->filename); + procfile_close(ff); + return NULL; + } + + pflines_reset(ff->lines); + pfwords_reset(ff->words); + + ff = procfile_parser(ff); + + if(unlikely(procfile_adaptive_initial_allocation)) { + if(unlikely(ff->len > procfile_max_allocation)) procfile_max_allocation = ff->len; + if(unlikely(ff->lines->len > procfile_max_lines)) procfile_max_lines = ff->lines->len; + if(unlikely(ff->words->len > procfile_max_words)) procfile_max_words = ff->words->len; + } + + debug(D_PROCFILE, "File '%s' updated.", ff->filename); + return ff; +} + +static void procfile_set_separators(procfile *ff, const char *separators) { + static char def[256] = { [0 ... 255] = 0 }; + int i; + + if(unlikely(!def[255])) { + // this is thread safe + // we check that the last byte is non-zero + // if it is zero, multiple threads may be executing this at the same time + // setting in def[] the exact same values + for(i = 0; likely(i < 256) ;i++) { + if(unlikely(i == '\n' || i == '\r')) def[i] = PF_CHAR_IS_NEWLINE; + else if(unlikely(isspace(i) || !isprint(i))) def[i] = PF_CHAR_IS_SEPARATOR; + else def[i] = PF_CHAR_IS_WORD; + } + } + + // copy the default + char *ffs = ff->separators, *ffd = def, *ffe = &def[256]; + while(likely(ffd != ffe)) *ffs++ = *ffd++; + + // set the separators + if(unlikely(!separators)) separators = " \t=|"; + ffs = ff->separators; + const char *s = separators; + while(likely(*s)) ffs[(int)*s++] = PF_CHAR_IS_SEPARATOR; +} + +procfile *procfile_open(const char *filename, const char *separators, uint32_t flags) { + debug(D_PROCFILE, PF_PREFIX ": Opening file '%s'", filename); + + int fd = open(filename, O_RDONLY, 0666); + if(unlikely(fd == -1)) { + if(unlikely(!(flags & PROCFILE_FLAG_NO_ERROR_ON_FILE_IO))) error(PF_PREFIX ": Cannot open file '%s'", filename); + return NULL; + } + + size_t size = (unlikely(procfile_adaptive_initial_allocation)) ? procfile_max_allocation : PROCFILE_INCREMENT_BUFFER; + procfile *ff = malloc(sizeof(procfile) + size); + if(unlikely(!ff)) { + error(PF_PREFIX ": Cannot allocate memory for file '%s'", filename); + close(fd); + return NULL; + } + + strncpy(ff->filename, filename, FILENAME_MAX); + ff->filename[FILENAME_MAX] = '\0'; + + ff->fd = fd; + ff->size = size; + ff->len = 0; + ff->flags = flags; + + ff->lines = pflines_new(); + ff->words = pfwords_new(); + + if(unlikely(!ff->lines || !ff->words)) { + error(PF_PREFIX ": Cannot initialize parser for file '%s'", filename); + procfile_close(ff); + return NULL; + } + + procfile_set_separators(ff, separators); + + debug(D_PROCFILE, "File '%s' opened.", filename); + return ff; +} + +procfile *procfile_reopen(procfile *ff, const char *filename, const char *separators, uint32_t flags) { + if(unlikely(!ff)) return procfile_open(filename, separators, flags); + + if(likely(ff->fd != -1)) close(ff->fd); + + ff->fd = open(filename, O_RDONLY, 0666); + if(unlikely(ff->fd == -1)) { + procfile_close(ff); + return NULL; + } + + strncpy(ff->filename, filename, FILENAME_MAX); + ff->filename[FILENAME_MAX] = '\0'; + + ff->flags = flags; + + // do not do the separators again if NULL is given + if(likely(separators)) procfile_set_separators(ff, separators); + + return ff; +} + +// ---------------------------------------------------------------------------- +// example parsing of procfile data + +void procfile_print(procfile *ff) { + uint32_t lines = procfile_lines(ff), l; + uint32_t words, w; + char *s; + + debug(D_PROCFILE, "File '%s' with %d lines and %d words", ff->filename, ff->lines->len, ff->words->len); + + for(l = 0; likely(l < lines) ;l++) { + words = procfile_linewords(ff, l); + + debug(D_PROCFILE, " line %d starts at word %d and has %d words", l, ff->lines->lines[l].first, ff->lines->lines[l].words); + + for(w = 0; likely(w < words) ;w++) { + s = procfile_lineword(ff, l, w); + debug(D_PROCFILE, " [%d.%d] '%s'", l, w, s); + } + } +} diff --git a/src/procfile.h b/src/procfile.h new file mode 100755 index 000000000..ce2f9bc92 --- /dev/null +++ b/src/procfile.h @@ -0,0 +1,109 @@ +/* + * procfile is a library for reading kernel files from /proc + * + * The idea is this: + * + * - every file is opened once with procfile_open(). + * + * - to read updated contents, we rewind it (lseek() to 0) and read again + * with procfile_readall(). + * + * - for every file, we use a buffer that is adjusted to fit its entire + * contents in memory, allowing us to read it with a single read() call. + * (this provides atomicity / consistency on the data read from the kernel) + * + * - once the data are read, we update two arrays of pointers: + * - a words array, pointing to each word in the data read + * - a lines array, pointing to the first word for each line + * + * This is highly optimized. Both arrays are automatically adjusted to + * fit all contents and are updated in a single pass on the data: + * - a raspberry Pi can process 5.000+ files / sec. + * - a J1900 celeron processor can process 23.000+ files / sec. +*/ + + +#ifndef NETDATA_PROCFILE_H +#define NETDATA_PROCFILE_H 1 + +// ---------------------------------------------------------------------------- +// An array of words + +typedef struct { + uint32_t len; // used entries + uint32_t size; // capacity + char *words[]; // array of pointers +} pfwords; + + +// ---------------------------------------------------------------------------- +// An array of lines + +typedef struct { + uint32_t words; // how many words this line has + uint32_t first; // the id of the first word of this line + // in the words array +} ffline; + +typedef struct { + uint32_t len; // used entries + uint32_t size; // capacity + ffline lines[]; // array of lines +} pflines; + + +// ---------------------------------------------------------------------------- +// The procfile + +#define PROCFILE_FLAG_DEFAULT 0x00000000 +#define PROCFILE_FLAG_NO_ERROR_ON_FILE_IO 0x00000001 + +typedef struct { + char filename[FILENAME_MAX + 1]; + uint32_t flags; + int fd; // the file desriptor + size_t len; // the bytes we have placed into data + size_t size; // the bytes we have allocated for data + pflines *lines; + pfwords *words; + char separators[256]; + char data[]; // allocated buffer to keep file contents +} procfile; + +// close the proc file and free all related memory +extern void procfile_close(procfile *ff); + +// (re)read and parse the proc file +extern procfile *procfile_readall(procfile *ff); + +// open a /proc or /sys file +extern procfile *procfile_open(const char *filename, const char *separators, uint32_t flags); + +// re-open a file +// if separators == NULL, the last separators are used +extern procfile *procfile_reopen(procfile *ff, const char *filename, const char *separators, uint32_t flags); + +// example walk-through a procfile parsed file +extern void procfile_print(procfile *ff); + +// ---------------------------------------------------------------------------- + +// set this to 1, to have procfile adapt its initial buffer allocation to the max allocation used so far +extern int procfile_adaptive_initial_allocation; + +// return the number of lines present +#define procfile_lines(ff) (ff->lines->len) + +// return the number of words of the Nth line +#define procfile_linewords(ff, line) (((line) < procfile_lines(ff)) ? (ff)->lines->lines[(line)].words : 0) + +// return the Nth word of the file, or empty string +#define procfile_word(ff, word) (((word) < (ff)->words->len) ? (ff)->words->words[(word)] : "") + +// return the first word of the Nth line, or empty string +#define procfile_line(ff, line) (((line) < procfile_lines(ff)) ? procfile_word((ff), (ff)->lines->lines[(line)].first) : "") + +// return the Nth word of the current line +#define procfile_lineword(ff, line, word) (((line) < procfile_lines(ff) && (word) < procfile_linewords(ff, (line))) ? procfile_word((ff), (ff)->lines->lines[(line)].first + word) : "") + +#endif /* NETDATA_PROCFILE_H */ diff --git a/src/rrd.c b/src/rrd.c new file mode 100755 index 000000000..2dce02e66 --- /dev/null +++ b/src/rrd.c @@ -0,0 +1,1338 @@ +#ifdef HAVE_CONFIG_H +#include <config.h> +#endif +#include <stddef.h> +#include <string.h> +#include <unistd.h> +#include <time.h> +#include <sys/time.h> +#include <sys/mman.h> +#include <pthread.h> +#include <errno.h> +#include <ctype.h> +#include <sys/stat.h> +#include <sys/types.h> +#include <stdlib.h> + +#include "common.h" +#include "log.h" +#include "appconfig.h" + +#include "rrd.h" + +#define RRD_DEFAULT_GAP_INTERPOLATIONS 1 + +// ---------------------------------------------------------------------------- +// globals + +// if not zero it gives the time (in seconds) to remove un-updated dimensions +// DO NOT ENABLE +// if dimensions are removed, the chart generation will have to run again +int rrd_delete_unupdated_dimensions = 0; + +int rrd_update_every = UPDATE_EVERY; +int rrd_default_history_entries = RRD_DEFAULT_HISTORY_ENTRIES; + +RRDSET *rrdset_root = NULL; +pthread_rwlock_t rrdset_root_rwlock = PTHREAD_RWLOCK_INITIALIZER; + +int rrd_memory_mode = RRD_MEMORY_MODE_SAVE; + + +// ---------------------------------------------------------------------------- +// RRDSET index + +static int rrdset_iterator(avl *a) { if(a) {}; return 0; } + +static int rrdset_compare(void* a, void* b) { + if(((RRDSET *)a)->hash < ((RRDSET *)b)->hash) return -1; + else if(((RRDSET *)a)->hash > ((RRDSET *)b)->hash) return 1; + else return strcmp(((RRDSET *)a)->id, ((RRDSET *)b)->id); +} + +avl_tree rrdset_root_index = { + NULL, + rrdset_compare, +#ifdef AVL_LOCK_WITH_MUTEX + PTHREAD_MUTEX_INITIALIZER +#else + PTHREAD_RWLOCK_INITIALIZER +#endif +}; + +#define rrdset_index_add(st) avl_insert(&rrdset_root_index, (avl *)(st)) +#define rrdset_index_del(st) avl_remove(&rrdset_root_index, (avl *)(st)) + +static RRDSET *rrdset_index_find(const char *id, uint32_t hash) { + RRDSET *result = NULL, tmp; + strncpy(tmp.id, id, RRD_ID_LENGTH_MAX); + tmp.id[RRD_ID_LENGTH_MAX] = '\0'; + tmp.hash = (hash)?hash:simple_hash(tmp.id); + + avl_search(&(rrdset_root_index), (avl *)&tmp, rrdset_iterator, (avl **)&result); + return result; +} + +// ---------------------------------------------------------------------------- +// RRDSET name index + +#define rrdset_from_avlname(avlname_ptr) ((RRDSET *)((avlname_ptr) - offsetof(RRDSET, avlname))) + +static int rrdset_iterator_name(avl *a) { if(a) {}; return 0; } + +static int rrdset_compare_name(void* a, void* b) { + RRDSET *A = rrdset_from_avlname(a); + RRDSET *B = rrdset_from_avlname(b); + + // fprintf(stderr, "COMPARING: %s with %s\n", A->name, B->name); + + if(A->hash_name < B->hash_name) return -1; + else if(A->hash_name > B->hash_name) return 1; + else return strcmp(A->name, B->name); +} + +avl_tree rrdset_root_index_name = { + NULL, + rrdset_compare_name, +#ifdef AVL_LOCK_WITH_MUTEX + PTHREAD_MUTEX_INITIALIZER +#else + PTHREAD_RWLOCK_INITIALIZER +#endif +}; + +int rrdset_index_add_name(RRDSET *st) { + // fprintf(stderr, "ADDING: %s (name: %s)\n", st->id, st->name); + return avl_insert(&rrdset_root_index_name, (avl *)(&st->avlname)); +} + +#define rrdset_index_del_name(st) avl_remove(&rrdset_root_index_name, (avl *)(&st->avlname)) + +static RRDSET *rrdset_index_find_name(const char *name, uint32_t hash) { + void *result = NULL; + RRDSET tmp; + tmp.name = name; + tmp.hash_name = (hash)?hash:simple_hash(tmp.name); + + // fprintf(stderr, "SEARCHING: %s\n", name); + avl_search(&(rrdset_root_index_name), (avl *)(&(tmp.avlname)), rrdset_iterator_name, (avl **)&result); + if(result) { + RRDSET *st = rrdset_from_avlname(result); + if(strcmp(st->magic, RRDSET_MAGIC)) + error("Search for RRDSET %s returned an invalid RRDSET %s (name %s)", name, st->id, st->name); + + // fprintf(stderr, "FOUND: %s\n", name); + return rrdset_from_avlname(result); + } + // fprintf(stderr, "NOT FOUND: %s\n", name); + return NULL; +} + + +// ---------------------------------------------------------------------------- +// RRDDIM index + +static int rrddim_iterator(avl *a) { if(a) {}; return 0; } + +static int rrddim_compare(void* a, void* b) { + if(((RRDDIM *)a)->hash < ((RRDDIM *)b)->hash) return -1; + else if(((RRDDIM *)a)->hash > ((RRDDIM *)b)->hash) return 1; + else return strcmp(((RRDDIM *)a)->id, ((RRDDIM *)b)->id); +} + +#define rrddim_index_add(st, rd) avl_insert(&((st)->dimensions_index), (avl *)(rd)) +#define rrddim_index_del(st,rd ) avl_remove(&((st)->dimensions_index), (avl *)(rd)) + +static RRDDIM *rrddim_index_find(RRDSET *st, const char *id, uint32_t hash) { + RRDDIM *result = NULL, tmp; + strncpy(tmp.id, id, RRD_ID_LENGTH_MAX); + tmp.id[RRD_ID_LENGTH_MAX] = '\0'; + tmp.hash = (hash)?hash:simple_hash(tmp.id); + + avl_search(&(st->dimensions_index), (avl *)&tmp, rrddim_iterator, (avl **)&result); + return result; +} + +// ---------------------------------------------------------------------------- +// chart types + +int rrdset_type_id(const char *name) +{ + if(unlikely(strcmp(name, RRDSET_TYPE_AREA_NAME) == 0)) return RRDSET_TYPE_AREA; + else if(unlikely(strcmp(name, RRDSET_TYPE_STACKED_NAME) == 0)) return RRDSET_TYPE_STACKED; + else if(unlikely(strcmp(name, RRDSET_TYPE_LINE_NAME) == 0)) return RRDSET_TYPE_LINE; + return RRDSET_TYPE_LINE; +} + +const char *rrdset_type_name(int chart_type) +{ + static char line[] = RRDSET_TYPE_LINE_NAME; + static char area[] = RRDSET_TYPE_AREA_NAME; + static char stacked[] = RRDSET_TYPE_STACKED_NAME; + + switch(chart_type) { + case RRDSET_TYPE_LINE: + return line; + + case RRDSET_TYPE_AREA: + return area; + + case RRDSET_TYPE_STACKED: + return stacked; + } + return line; +} + +// ---------------------------------------------------------------------------- +// load / save + +const char *rrd_memory_mode_name(int id) +{ + static const char ram[] = RRD_MEMORY_MODE_RAM_NAME; + static const char map[] = RRD_MEMORY_MODE_MAP_NAME; + static const char save[] = RRD_MEMORY_MODE_SAVE_NAME; + + switch(id) { + case RRD_MEMORY_MODE_RAM: + return ram; + + case RRD_MEMORY_MODE_MAP: + return map; + + case RRD_MEMORY_MODE_SAVE: + default: + return save; + } + + return save; +} + +int rrd_memory_mode_id(const char *name) +{ + if(unlikely(!strcmp(name, RRD_MEMORY_MODE_RAM_NAME))) + return RRD_MEMORY_MODE_RAM; + else if(unlikely(!strcmp(name, RRD_MEMORY_MODE_MAP_NAME))) + return RRD_MEMORY_MODE_MAP; + + return RRD_MEMORY_MODE_SAVE; +} + +// ---------------------------------------------------------------------------- +// algorithms types + +int rrddim_algorithm_id(const char *name) +{ + if(strcmp(name, RRDDIM_ABSOLUTE_NAME) == 0) return RRDDIM_ABSOLUTE; + if(strcmp(name, RRDDIM_INCREMENTAL_NAME) == 0) return RRDDIM_INCREMENTAL; + if(strcmp(name, RRDDIM_PCENT_OVER_ROW_TOTAL_NAME) == 0) return RRDDIM_PCENT_OVER_ROW_TOTAL; + if(strcmp(name, RRDDIM_PCENT_OVER_DIFF_TOTAL_NAME) == 0) return RRDDIM_PCENT_OVER_DIFF_TOTAL; + return RRDDIM_ABSOLUTE; +} + +const char *rrddim_algorithm_name(int chart_type) +{ + static char absolute[] = RRDDIM_ABSOLUTE_NAME; + static char incremental[] = RRDDIM_INCREMENTAL_NAME; + static char percentage_of_absolute_row[] = RRDDIM_PCENT_OVER_ROW_TOTAL_NAME; + static char percentage_of_incremental_row[] = RRDDIM_PCENT_OVER_DIFF_TOTAL_NAME; + + switch(chart_type) { + case RRDDIM_ABSOLUTE: + return absolute; + + case RRDDIM_INCREMENTAL: + return incremental; + + case RRDDIM_PCENT_OVER_ROW_TOTAL: + return percentage_of_absolute_row; + + case RRDDIM_PCENT_OVER_DIFF_TOTAL: + return percentage_of_incremental_row; + } + return absolute; +} + +// ---------------------------------------------------------------------------- +// chart names + +char *rrdset_strncpy_name(char *to, const char *from, int length) +{ + int i; + for(i = 0; i < length && from[i] ;i++) { + if(from[i] == '.' || isalpha(from[i]) || isdigit(from[i])) to[i] = from[i]; + else to[i] = '_'; + } + if(i < length) to[i] = '\0'; + to[length - 1] = '\0'; + + return to; +} + +void rrdset_set_name(RRDSET *st, const char *name) +{ + debug(D_RRD_CALLS, "rrdset_set_name() old: %s, new: %s", st->name, name); + + if(st->name) rrdset_index_del_name(st); + + char b[CONFIG_MAX_VALUE + 1]; + char n[RRD_ID_LENGTH_MAX + 1]; + + snprintf(n, RRD_ID_LENGTH_MAX, "%s.%s", st->type, name); + rrdset_strncpy_name(b, n, CONFIG_MAX_VALUE); + st->name = config_get(st->id, "name", b); + st->hash_name = simple_hash(st->name); + + rrdset_index_add_name(st); +} + +// ---------------------------------------------------------------------------- +// cache directory + +char *rrdset_cache_dir(const char *id) +{ + char *ret = NULL; + + static char *cache_dir = NULL; + if(!cache_dir) cache_dir = config_get("global", "cache directory", CACHE_DIR); + + char b[FILENAME_MAX + 1]; + char n[FILENAME_MAX + 1]; + rrdset_strncpy_name(b, id, FILENAME_MAX); + + snprintf(n, FILENAME_MAX, "%s/%s", cache_dir, b); + ret = config_get(id, "cache directory", n); + + if(rrd_memory_mode == RRD_MEMORY_MODE_MAP || rrd_memory_mode == RRD_MEMORY_MODE_SAVE) { + int r = mkdir(ret, 0775); + if(r != 0 && errno != EEXIST) + error("Cannot create directory '%s'", ret); + } + + return ret; +} + +// ---------------------------------------------------------------------------- +// core functions + +void rrdset_reset(RRDSET *st) +{ + debug(D_RRD_CALLS, "rrdset_reset() %s", st->name); + + st->last_collected_time.tv_sec = 0; + st->last_collected_time.tv_usec = 0; + st->last_updated.tv_sec = 0; + st->last_updated.tv_usec = 0; + st->current_entry = 0; + st->counter = 0; + st->counter_done = 0; + + RRDDIM *rd; + for(rd = st->dimensions; rd ; rd = rd->next) { + rd->last_collected_time.tv_sec = 0; + rd->last_collected_time.tv_usec = 0; + rd->counter = 0; + bzero(rd->values, rd->entries * sizeof(storage_number)); + } +} + +RRDSET *rrdset_create(const char *type, const char *id, const char *name, const char *family, const char *context, const char *title, const char *units, long priority, int update_every, int chart_type) +{ + if(!type || !type[0]) { + fatal("Cannot create rrd stats without a type."); + return NULL; + } + + if(!id || !id[0]) { + fatal("Cannot create rrd stats without an id."); + return NULL; + } + + char fullid[RRD_ID_LENGTH_MAX + 1]; + char fullfilename[FILENAME_MAX + 1]; + RRDSET *st = NULL; + + snprintf(fullid, RRD_ID_LENGTH_MAX, "%s.%s", type, id); + + st = rrdset_find(fullid); + if(st) { + error("Cannot create rrd stats for '%s', it already exists.", fullid); + return st; + } + + long entries = config_get_number(fullid, "history", rrd_default_history_entries); + if(entries < 5) entries = config_set_number(fullid, "history", 5); + if(entries > RRD_HISTORY_ENTRIES_MAX) entries = config_set_number(fullid, "history", RRD_HISTORY_ENTRIES_MAX); + + int enabled = config_get_boolean(fullid, "enabled", 1); + if(!enabled) entries = 5; + + unsigned long size = sizeof(RRDSET); + char *cache_dir = rrdset_cache_dir(fullid); + + debug(D_RRD_CALLS, "Creating RRD_STATS for '%s.%s'.", type, id); + + snprintf(fullfilename, FILENAME_MAX, "%s/main.db", cache_dir); + if(rrd_memory_mode != RRD_MEMORY_MODE_RAM) st = (RRDSET *)mymmap(fullfilename, size, ((rrd_memory_mode == RRD_MEMORY_MODE_MAP)?MAP_SHARED:MAP_PRIVATE), 0); + if(st) { + if(strcmp(st->magic, RRDSET_MAGIC) != 0) { + errno = 0; + info("Initializing file %s.", fullfilename); + bzero(st, size); + } + else if(strcmp(st->id, fullid) != 0) { + errno = 0; + error("File %s contents are not for chart %s. Clearing it.", fullfilename, fullid); + // munmap(st, size); + // st = NULL; + bzero(st, size); + } + else if(st->memsize != size || st->entries != entries) { + errno = 0; + error("File %s does not have the desired size. Clearing it.", fullfilename); + bzero(st, size); + } + else if(st->update_every != update_every) { + errno = 0; + error("File %s does not have the desired update frequency. Clearing it.", fullfilename); + bzero(st, size); + } + else if((time(NULL) - st->last_updated.tv_sec) > update_every * entries) { + errno = 0; + error("File %s is too old. Clearing it.", fullfilename); + bzero(st, size); + } + } + + if(st) { + st->name = NULL; + st->type = NULL; + st->family = NULL; + st->context = NULL; + st->title = NULL; + st->units = NULL; + st->dimensions = NULL; + st->next = NULL; + st->mapped = rrd_memory_mode; + } + else { + st = calloc(1, size); + if(!st) { + fatal("Cannot allocate memory for RRD_STATS %s.%s", type, id); + return NULL; + } + st->mapped = RRD_MEMORY_MODE_RAM; + } + st->memsize = size; + st->entries = entries; + st->update_every = update_every; + + strcpy(st->cache_filename, fullfilename); + strcpy(st->magic, RRDSET_MAGIC); + + strcpy(st->id, fullid); + st->hash = simple_hash(st->id); + + st->cache_dir = cache_dir; + + st->chart_type = rrdset_type_id(config_get(st->id, "chart type", rrdset_type_name(chart_type))); + st->type = config_get(st->id, "type", type); + st->family = config_get(st->id, "family", family?family:st->type); + st->context = config_get(st->id, "context", context?context:st->id); + st->units = config_get(st->id, "units", units?units:""); + + st->priority = config_get_number(st->id, "priority", priority); + st->enabled = enabled; + + st->isdetail = 0; + st->debug = 0; + + st->last_collected_time.tv_sec = 0; + st->last_collected_time.tv_usec = 0; + st->counter_done = 0; + + st->gap_when_lost_iterations_above = (int) ( + config_get_number(st->id, "gap when lost iterations above", RRD_DEFAULT_GAP_INTERPOLATIONS) + 2); + + avl_init(&st->dimensions_index, rrddim_compare); + + pthread_rwlock_init(&st->rwlock, NULL); + pthread_rwlock_wrlock(&rrdset_root_rwlock); + + if(name && *name) rrdset_set_name(st, name); + else rrdset_set_name(st, id); + + { + char varvalue[CONFIG_MAX_VALUE + 1]; + snprintf(varvalue, CONFIG_MAX_VALUE, "%s (%s)", title?title:"", st->name); + st->title = config_get(st->id, "title", varvalue); + } + + st->next = rrdset_root; + rrdset_root = st; + + rrdset_index_add(st); + + pthread_rwlock_unlock(&rrdset_root_rwlock); + + return(st); +} + +RRDDIM *rrddim_add(RRDSET *st, const char *id, const char *name, long multiplier, long divisor, int algorithm) +{ + char filename[FILENAME_MAX + 1]; + char fullfilename[FILENAME_MAX + 1]; + + char varname[CONFIG_MAX_NAME + 1]; + RRDDIM *rd = NULL; + unsigned long size = sizeof(RRDDIM) + (st->entries * sizeof(storage_number)); + + debug(D_RRD_CALLS, "Adding dimension '%s/%s'.", st->id, id); + + rrdset_strncpy_name(filename, id, FILENAME_MAX); + snprintf(fullfilename, FILENAME_MAX, "%s/%s.db", st->cache_dir, filename); + if(rrd_memory_mode != RRD_MEMORY_MODE_RAM) rd = (RRDDIM *)mymmap(fullfilename, size, ((rrd_memory_mode == RRD_MEMORY_MODE_MAP)?MAP_SHARED:MAP_PRIVATE), 1); + if(rd) { + struct timeval now; + gettimeofday(&now, NULL); + + if(strcmp(rd->magic, RRDDIMENSION_MAGIC) != 0) { + errno = 0; + info("Initializing file %s.", fullfilename); + bzero(rd, size); + } + else if(rd->memsize != size) { + errno = 0; + error("File %s does not have the desired size. Clearing it.", fullfilename); + bzero(rd, size); + } + else if(rd->multiplier != multiplier) { + errno = 0; + error("File %s does not have the same multiplier. Clearing it.", fullfilename); + bzero(rd, size); + } + else if(rd->divisor != divisor) { + errno = 0; + error("File %s does not have the same divisor. Clearing it.", fullfilename); + bzero(rd, size); + } + else if(rd->algorithm != algorithm) { + errno = 0; + error("File %s does not have the same algorithm. Clearing it.", fullfilename); + bzero(rd, size); + } + else if(rd->update_every != st->update_every) { + errno = 0; + error("File %s does not have the same refresh frequency. Clearing it.", fullfilename); + bzero(rd, size); + } + else if(usecdiff(&now, &rd->last_collected_time) > (rd->entries * rd->update_every * 1000000ULL)) { + errno = 0; + error("File %s is too old. Clearing it.", fullfilename); + bzero(rd, size); + } + else if(strcmp(rd->id, id) != 0) { + errno = 0; + error("File %s contents are not for dimension %s. Clearing it.", fullfilename, id); + // munmap(rd, size); + // rd = NULL; + bzero(rd, size); + } + } + + if(rd) { + // we have a file mapped for rd + rd->mapped = rrd_memory_mode; + rd->flags = 0x00000000; + rd->next = NULL; + rd->name = NULL; + } + else { + // if we didn't manage to get a mmap'd dimension, just create one + + rd = calloc(1, size); + if(!rd) { + fatal("Cannot allocate RRD_DIMENSION %s/%s.", st->id, id); + return NULL; + } + + rd->mapped = RRD_MEMORY_MODE_RAM; + } + rd->memsize = size; + + strcpy(rd->magic, RRDDIMENSION_MAGIC); + strcpy(rd->cache_filename, fullfilename); + strncpy(rd->id, id, RRD_ID_LENGTH_MAX); + rd->hash = simple_hash(rd->id); + + snprintf(varname, CONFIG_MAX_NAME, "dim %s name", rd->id); + rd->name = config_get(st->id, varname, (name && *name)?name:rd->id); + + snprintf(varname, CONFIG_MAX_NAME, "dim %s algorithm", rd->id); + rd->algorithm = rrddim_algorithm_id(config_get(st->id, varname, rrddim_algorithm_name(algorithm))); + + snprintf(varname, CONFIG_MAX_NAME, "dim %s multiplier", rd->id); + rd->multiplier = config_get_number(st->id, varname, multiplier); + + snprintf(varname, CONFIG_MAX_NAME, "dim %s divisor", rd->id); + rd->divisor = config_get_number(st->id, varname, divisor); + if(!rd->divisor) rd->divisor = 1; + + rd->entries = st->entries; + rd->update_every = st->update_every; + + // prevent incremental calculation spikes + rd->counter = 0; + + // append this dimension + pthread_rwlock_wrlock(&st->rwlock); + if(!st->dimensions) + st->dimensions = rd; + else { + RRDDIM *td = st->dimensions; + for(; td->next; td = td->next) ; + td->next = rd; + } + pthread_rwlock_unlock(&st->rwlock); + + rrddim_index_add(st, rd); + + return(rd); +} + +void rrddim_set_name(RRDSET *st, RRDDIM *rd, const char *name) +{ + debug(D_RRD_CALLS, "rrddim_set_name() %s.%s", st->name, rd->name); + + char varname[CONFIG_MAX_NAME + 1]; + snprintf(varname, CONFIG_MAX_NAME, "dim %s name", rd->id); + config_set_default(st->id, varname, name); +} + +void rrddim_free(RRDSET *st, RRDDIM *rd) +{ + debug(D_RRD_CALLS, "rrddim_free() %s.%s", st->name, rd->name); + + RRDDIM *i, *last = NULL; + for(i = st->dimensions; i && i != rd ; i = i->next) last = i; + + if(!i) { + error("Request to free dimension %s.%s but it is not linked.", st->id, rd->name); + return; + } + + if(last) last->next = rd->next; + else st->dimensions = rd->next; + rd->next = NULL; + + rrddim_index_del(st, rd); + + // free(rd->annotations); + if(rd->mapped == RRD_MEMORY_MODE_SAVE) { + debug(D_RRD_CALLS, "Saving dimension '%s' to '%s'.", rd->name, rd->cache_filename); + savememory(rd->cache_filename, rd, rd->memsize); + + debug(D_RRD_CALLS, "Unmapping dimension '%s'.", rd->name); + munmap(rd, rd->memsize); + } + else if(rd->mapped == RRD_MEMORY_MODE_MAP) { + debug(D_RRD_CALLS, "Unmapping dimension '%s'.", rd->name); + munmap(rd, rd->memsize); + } + else { + debug(D_RRD_CALLS, "Removing dimension '%s'.", rd->name); + free(rd); + } +} + +void rrdset_free_all(void) +{ + info("Freeing all memory..."); + + RRDSET *st; + for(st = rrdset_root; st ;) { + RRDSET *next = st->next; + + while(st->dimensions) + rrddim_free(st, st->dimensions); + + rrdset_index_del(st); + + if(st->mapped == RRD_MEMORY_MODE_SAVE) { + debug(D_RRD_CALLS, "Saving stats '%s' to '%s'.", st->name, st->cache_filename); + savememory(st->cache_filename, st, st->memsize); + + debug(D_RRD_CALLS, "Unmapping stats '%s'.", st->name); + munmap(st, st->memsize); + } + else if(st->mapped == RRD_MEMORY_MODE_MAP) { + debug(D_RRD_CALLS, "Unmapping stats '%s'.", st->name); + munmap(st, st->memsize); + } + else + free(st); + + st = next; + } + rrdset_root = NULL; + + info("Memory cleanup completed..."); +} + +void rrdset_save_all(void) +{ + debug(D_RRD_CALLS, "rrdset_save_all()"); + + RRDSET *st; + RRDDIM *rd; + + pthread_rwlock_wrlock(&rrdset_root_rwlock); + for(st = rrdset_root; st ; st = st->next) { + pthread_rwlock_wrlock(&st->rwlock); + + if(st->mapped == RRD_MEMORY_MODE_SAVE) { + debug(D_RRD_CALLS, "Saving stats '%s' to '%s'.", st->name, st->cache_filename); + savememory(st->cache_filename, st, st->memsize); + } + + for(rd = st->dimensions; rd ; rd = rd->next) { + if(likely(rd->mapped == RRD_MEMORY_MODE_SAVE)) { + debug(D_RRD_CALLS, "Saving dimension '%s' to '%s'.", rd->name, rd->cache_filename); + savememory(rd->cache_filename, rd, rd->memsize); + } + } + + pthread_rwlock_unlock(&st->rwlock); + } + pthread_rwlock_unlock(&rrdset_root_rwlock); +} + + +RRDSET *rrdset_find(const char *id) +{ + debug(D_RRD_CALLS, "rrdset_find() for chart %s", id); + + RRDSET *st = rrdset_index_find(id, 0); + return(st); +} + +RRDSET *rrdset_find_bytype(const char *type, const char *id) +{ + debug(D_RRD_CALLS, "rrdset_find_bytype() for chart %s.%s", type, id); + + char buf[RRD_ID_LENGTH_MAX + 1]; + + strncpy(buf, type, RRD_ID_LENGTH_MAX - 1); + buf[RRD_ID_LENGTH_MAX - 1] = '\0'; + strcat(buf, "."); + int len = (int) strlen(buf); + strncpy(&buf[len], id, (size_t) (RRD_ID_LENGTH_MAX - len)); + buf[RRD_ID_LENGTH_MAX] = '\0'; + + return(rrdset_find(buf)); +} + +RRDSET *rrdset_find_byname(const char *name) +{ + debug(D_RRD_CALLS, "rrdset_find_byname() for chart %s", name); + + RRDSET *st = rrdset_index_find_name(name, 0); + return(st); +} + +RRDDIM *rrddim_find(RRDSET *st, const char *id) +{ + debug(D_RRD_CALLS, "rrddim_find() for chart %s, dimension %s", st->name, id); + + return rrddim_index_find(st, id, 0); +} + +int rrddim_hide(RRDSET *st, const char *id) +{ + debug(D_RRD_CALLS, "rrddim_hide() for chart %s, dimension %s", st->name, id); + + RRDDIM *rd = rrddim_find(st, id); + if(unlikely(!rd)) { + error("Cannot find dimension with id '%s' on stats '%s' (%s).", id, st->name, st->id); + return 1; + } + + rd->flags |= RRDDIM_FLAG_HIDDEN; + return 0; +} + +int rrddim_unhide(RRDSET *st, const char *id) +{ + debug(D_RRD_CALLS, "rrddim_unhide() for chart %s, dimension %s", st->name, id); + + RRDDIM *rd = rrddim_find(st, id); + if(unlikely(!rd)) { + error("Cannot find dimension with id '%s' on stats '%s' (%s).", id, st->name, st->id); + return 1; + } + + if(rd->flags & RRDDIM_FLAG_HIDDEN) rd->flags ^= RRDDIM_FLAG_HIDDEN; + return 0; +} + +collected_number rrddim_set_by_pointer(RRDSET *st, RRDDIM *rd, collected_number value) +{ + debug(D_RRD_CALLS, "rrddim_set_by_pointer() for chart %s, dimension %s, value " COLLECTED_NUMBER_FORMAT, st->name, rd->name, value); + + gettimeofday(&rd->last_collected_time, NULL); + rd->collected_value = value; + rd->updated = 1; + rd->counter++; + + return rd->last_collected_value; +} + +collected_number rrddim_set(RRDSET *st, const char *id, collected_number value) +{ + RRDDIM *rd = rrddim_find(st, id); + if(unlikely(!rd)) { + error("Cannot find dimension with id '%s' on stats '%s' (%s).", id, st->name, st->id); + return 0; + } + + return rrddim_set_by_pointer(st, rd, value); +} + +void rrdset_next_usec(RRDSET *st, unsigned long long microseconds) +{ + if(!microseconds) rrdset_next(st); + else { + debug(D_RRD_CALLS, "rrdset_next_usec() for chart %s with microseconds %llu", st->name, microseconds); + + if(unlikely(st->debug)) debug(D_RRD_STATS, "%s: NEXT: %llu microseconds", st->name, microseconds); + st->usec_since_last_update = microseconds; + } +} + +void rrdset_next(RRDSET *st) +{ + unsigned long long microseconds = 0; + + if(likely(st->last_collected_time.tv_sec)) { + struct timeval now; + gettimeofday(&now, NULL); + microseconds = usecdiff(&now, &st->last_collected_time); + } + // prevent infinite loop + else microseconds = st->update_every * 1000000ULL; + + rrdset_next_usec(st, microseconds); +} + +void rrdset_next_plugins(RRDSET *st) +{ + rrdset_next(st); +} + +unsigned long long rrdset_done(RRDSET *st) +{ + debug(D_RRD_CALLS, "rrdset_done() for chart %s", st->name); + + RRDDIM *rd, *last; + int oldstate, store_this_entry = 1, first_entry = 0; + unsigned long long last_ut, now_ut, next_ut, stored_entries = 0; + + if(unlikely(pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &oldstate) != 0)) + error("Cannot set pthread cancel state to DISABLE."); + + // a read lock is OK here + pthread_rwlock_rdlock(&st->rwlock); + + // enable the chart, if it was disabled + st->enabled = 1; + + // check if the chart has a long time to be updated + if(unlikely(st->usec_since_last_update > st->entries * st->update_every * 1000000ULL)) { + info("%s: took too long to be updated (%0.3Lf secs). Reseting it.", st->name, (long double)(st->usec_since_last_update / 1000000.0)); + rrdset_reset(st); + st->usec_since_last_update = st->update_every * 1000000ULL; + first_entry = 1; + } + if(unlikely(st->debug)) debug(D_RRD_STATS, "%s: microseconds since last update: %llu", st->name, st->usec_since_last_update); + + // set last_collected_time + if(unlikely(!st->last_collected_time.tv_sec)) { + // it is the first entry + // set the last_collected_time to now + gettimeofday(&st->last_collected_time, NULL); + + // the first entry should not be stored + store_this_entry = 0; + first_entry = 1; + + if(unlikely(st->debug)) debug(D_RRD_STATS, "%s: has not set last_collected_time. Setting it now. Will not store the next entry.", st->name); + } + else { + // it is not the first entry + // calculate the proper last_collected_time, using usec_since_last_update + unsigned long long ut = st->last_collected_time.tv_sec * 1000000ULL + st->last_collected_time.tv_usec + st->usec_since_last_update; + st->last_collected_time.tv_sec = (time_t) (ut / 1000000ULL); + st->last_collected_time.tv_usec = (useconds_t) (ut % 1000000ULL); + } + + // if this set has not been updated in the past + // we fake the last_update time to be = now - usec_since_last_update + if(unlikely(!st->last_updated.tv_sec)) { + // it has never been updated before + // set a fake last_updated, in the past using usec_since_last_update + unsigned long long ut = st->last_collected_time.tv_sec * 1000000ULL + st->last_collected_time.tv_usec - st->usec_since_last_update; + st->last_updated.tv_sec = (time_t) (ut / 1000000ULL); + st->last_updated.tv_usec = (useconds_t) (ut % 1000000ULL); + + // the first entry should not be stored + store_this_entry = 0; + first_entry = 1; + + if(unlikely(st->debug)) debug(D_RRD_STATS, "%s: initializing last_updated to now - %llu microseconds (%0.3Lf). Will not store the next entry.", st->name, st->usec_since_last_update, (long double)ut/1000000.0); + } + + // check if we will re-write the entire data set + if(unlikely(usecdiff(&st->last_collected_time, &st->last_updated) > st->update_every * st->entries * 1000000ULL)) { + info("%s: too old data (last updated at %u.%u, last collected at %u.%u). Reseting it. Will not store the next entry.", st->name, st->last_updated.tv_sec, st->last_updated.tv_usec, st->last_collected_time.tv_sec, st->last_collected_time.tv_usec); + rrdset_reset(st); + + st->usec_since_last_update = st->update_every * 1000000ULL; + + gettimeofday(&st->last_collected_time, NULL); + + unsigned long long ut = st->last_collected_time.tv_sec * 1000000ULL + st->last_collected_time.tv_usec - st->usec_since_last_update; + st->last_updated.tv_sec = (time_t) (ut / 1000000ULL); + st->last_updated.tv_usec = (useconds_t) (ut % 1000000ULL); + + // the first entry should not be stored + store_this_entry = 0; + first_entry = 1; + } + + // these are the 3 variables that will help us in interpolation + // last_ut = the last time we added a value to the storage + // now_ut = the time the current value is taken at + // next_ut = the time of the next interpolation point + last_ut = st->last_updated.tv_sec * 1000000ULL + st->last_updated.tv_usec; + now_ut = st->last_collected_time.tv_sec * 1000000ULL + st->last_collected_time.tv_usec; + next_ut = (st->last_updated.tv_sec + st->update_every) * 1000000ULL; + + if(unlikely(!first_entry && now_ut < next_ut)) { + if(unlikely(st->debug)) debug(D_RRD_STATS, "%s: THIS IS IN THE SAME INTERPOLATION POINT", st->name); + } + + if(unlikely(st->debug)) { + debug(D_RRD_STATS, "%s: last ut = %0.3Lf (last updated time)", st->name, (long double)last_ut/1000000.0); + debug(D_RRD_STATS, "%s: now ut = %0.3Lf (current update time)", st->name, (long double)now_ut/1000000.0); + debug(D_RRD_STATS, "%s: next ut = %0.3Lf (next interpolation point)", st->name, (long double)next_ut/1000000.0); + } + + if(unlikely(!st->counter_done)) { + store_this_entry = 0; + if(unlikely(st->debug)) debug(D_RRD_STATS, "%s: Will not store the next entry.", st->name); + } + st->counter_done++; + + // calculate totals and count the dimensions + int dimensions; + st->collected_total = 0; + for( rd = st->dimensions, dimensions = 0 ; likely(rd) ; rd = rd->next, dimensions++ ) + st->collected_total += rd->collected_value; + + uint32_t storage_flags = SN_EXISTS; + + // process all dimensions to calculate their values + // based on the collected figures only + // at this stage we do not interpolate anything + for( rd = st->dimensions ; likely(rd) ; rd = rd->next ) { + + if(unlikely(st->debug)) debug(D_RRD_STATS, "%s/%s: START " + " last_collected_value = " COLLECTED_NUMBER_FORMAT + " collected_value = " COLLECTED_NUMBER_FORMAT + " last_calculated_value = " CALCULATED_NUMBER_FORMAT + " calculated_value = " CALCULATED_NUMBER_FORMAT + , st->id, rd->name + , rd->last_collected_value + , rd->collected_value + , rd->last_calculated_value + , rd->calculated_value + ); + + switch(rd->algorithm) { + case RRDDIM_ABSOLUTE: + rd->calculated_value = (calculated_number)rd->collected_value + * (calculated_number)rd->multiplier + / (calculated_number)rd->divisor; + + if(unlikely(st->debug)) + debug(D_RRD_STATS, "%s/%s: CALC ABS/ABS-NO-IN " + CALCULATED_NUMBER_FORMAT " = " + COLLECTED_NUMBER_FORMAT + " * " CALCULATED_NUMBER_FORMAT + " / " CALCULATED_NUMBER_FORMAT + , st->id, rd->name + , rd->calculated_value + , rd->collected_value + , (calculated_number)rd->multiplier + , (calculated_number)rd->divisor + ); + break; + + case RRDDIM_PCENT_OVER_ROW_TOTAL: + if(unlikely(!st->collected_total)) rd->calculated_value = 0; + else + // the percentage of the current value + // over the total of all dimensions + rd->calculated_value = + (calculated_number)100 + * (calculated_number)rd->collected_value + / (calculated_number)st->collected_total; + + if(unlikely(st->debug)) + debug(D_RRD_STATS, "%s/%s: CALC PCENT-ROW " + CALCULATED_NUMBER_FORMAT " = 100" + " * " COLLECTED_NUMBER_FORMAT + " / " COLLECTED_NUMBER_FORMAT + , st->id, rd->name + , rd->calculated_value + , rd->collected_value + , st->collected_total + ); + break; + + case RRDDIM_INCREMENTAL: + if(unlikely(!rd->updated || rd->counter <= 1)) { + rd->calculated_value = 0; + continue; + } + + // if the new is smaller than the old (an overflow, or reset), set the old equal to the new + // to reset the calculation (it will give zero as the calculation for this second) + if(unlikely(rd->last_collected_value > rd->collected_value)) { + debug(D_RRD_STATS, "%s.%s: RESET or OVERFLOW. Last collected value = " COLLECTED_NUMBER_FORMAT ", current = " COLLECTED_NUMBER_FORMAT + , st->name, rd->name + , rd->last_collected_value + , rd->collected_value); + if(!(rd->flags & RRDDIM_FLAG_DONT_DETECT_RESETS_OR_OVERFLOWS)) storage_flags = SN_EXISTS_RESET; + rd->last_collected_value = rd->collected_value; + } + + rd->calculated_value = (calculated_number)(rd->collected_value - rd->last_collected_value) + * (calculated_number)rd->multiplier + / (calculated_number)rd->divisor; + + if(unlikely(st->debug)) + debug(D_RRD_STATS, "%s/%s: CALC INC PRE " + CALCULATED_NUMBER_FORMAT " = (" + COLLECTED_NUMBER_FORMAT " - " COLLECTED_NUMBER_FORMAT + ")" + " * " CALCULATED_NUMBER_FORMAT + " / " CALCULATED_NUMBER_FORMAT + , st->id, rd->name + , rd->calculated_value + , rd->collected_value, rd->last_collected_value + , (calculated_number)rd->multiplier + , (calculated_number)rd->divisor + ); + break; + + case RRDDIM_PCENT_OVER_DIFF_TOTAL: + if(unlikely(!rd->updated || rd->counter <= 1)) { + rd->calculated_value = 0; + continue; + } + + // the percentage of the current increment + // over the increment of all dimensions together + if(unlikely(st->collected_total == st->last_collected_total)) rd->calculated_value = rd->last_calculated_value; + else rd->calculated_value = + (calculated_number)100 + * (calculated_number)(rd->collected_value - rd->last_collected_value) + / (calculated_number)(st->collected_total - st->last_collected_total); + + if(unlikely(st->debug)) + debug(D_RRD_STATS, "%s/%s: CALC PCENT-DIFF " + CALCULATED_NUMBER_FORMAT " = 100" + " * (" COLLECTED_NUMBER_FORMAT " - " COLLECTED_NUMBER_FORMAT ")" + " / (" COLLECTED_NUMBER_FORMAT " - " COLLECTED_NUMBER_FORMAT ")" + , st->id, rd->name + , rd->calculated_value + , rd->collected_value, rd->last_collected_value + , st->collected_total, st->last_collected_total + ); + break; + + default: + // make the default zero, to make sure + // it gets noticed when we add new types + rd->calculated_value = 0; + + if(unlikely(st->debug)) + debug(D_RRD_STATS, "%s/%s: CALC " + CALCULATED_NUMBER_FORMAT " = 0" + , st->id, rd->name + , rd->calculated_value + ); + break; + } + + if(unlikely(st->debug)) debug(D_RRD_STATS, "%s/%s: PHASE2 " + " last_collected_value = " COLLECTED_NUMBER_FORMAT + " collected_value = " COLLECTED_NUMBER_FORMAT + " last_calculated_value = " CALCULATED_NUMBER_FORMAT + " calculated_value = " CALCULATED_NUMBER_FORMAT + , st->id, rd->name + , rd->last_collected_value + , rd->collected_value + , rd->last_calculated_value + , rd->calculated_value + ); + + } + + // at this point we have all the calculated values ready + // it is now time to interpolate values on a second boundary + + unsigned long long first_ut = last_ut; + long long iterations = (now_ut - last_ut) / (st->update_every * 1000000ULL); + if((now_ut % (st->update_every * 1000000ULL)) == 0) iterations++; + + for( ; likely(next_ut <= now_ut) ; next_ut += st->update_every * 1000000ULL, iterations-- ) { +#ifdef NETDATA_INTERNAL_CHECKS + if(iterations < 0) { error("%s: iterations calculation wrapped! first_ut = %llu, last_ut = %llu, next_ut = %llu, now_ut = %llu", st->name, first_ut, last_ut, next_ut, now_ut); } +#endif + + if(unlikely(st->debug)) { + debug(D_RRD_STATS, "%s: last ut = %0.3Lf (last updated time)", st->name, (long double)last_ut/1000000.0); + debug(D_RRD_STATS, "%s: next ut = %0.3Lf (next interpolation point)", st->name, (long double)next_ut/1000000.0); + } + + st->last_updated.tv_sec = (time_t) (next_ut / 1000000ULL); + st->last_updated.tv_usec = 0; + + for( rd = st->dimensions ; likely(rd) ; rd = rd->next ) { + calculated_number new_value; + + switch(rd->algorithm) { + case RRDDIM_INCREMENTAL: + new_value = (calculated_number) + ( rd->calculated_value + * (calculated_number)(next_ut - last_ut) + / (calculated_number)(now_ut - last_ut) + ); + + if(unlikely(st->debug)) + debug(D_RRD_STATS, "%s/%s: CALC2 INC " + CALCULATED_NUMBER_FORMAT " = " + CALCULATED_NUMBER_FORMAT + " * %llu" + " / %llu" + , st->id, rd->name + , new_value + , rd->calculated_value + , (next_ut - last_ut) + , (now_ut - last_ut) + ); + + rd->calculated_value -= new_value; + new_value += rd->last_calculated_value; + rd->last_calculated_value = 0; + new_value /= (calculated_number)st->update_every; + break; + + case RRDDIM_ABSOLUTE: + case RRDDIM_PCENT_OVER_ROW_TOTAL: + case RRDDIM_PCENT_OVER_DIFF_TOTAL: + default: + if(iterations == 1) { + // this is the last iteration + // do not interpolate + // just show the calculated value + + new_value = rd->calculated_value; + } + else { + // we have missed an update + // interpolate in the middle values + + new_value = (calculated_number) + ( ( (rd->calculated_value - rd->last_calculated_value) + * (calculated_number)(next_ut - first_ut) + / (calculated_number)(now_ut - first_ut) + ) + + rd->last_calculated_value + ); + + if(unlikely(st->debug)) + debug(D_RRD_STATS, "%s/%s: CALC2 DEF " + CALCULATED_NUMBER_FORMAT " = (((" + "(" CALCULATED_NUMBER_FORMAT " - " CALCULATED_NUMBER_FORMAT ")" + " * %llu" + " / %llu) + " CALCULATED_NUMBER_FORMAT + , st->id, rd->name + , new_value + , rd->calculated_value, rd->last_calculated_value + , (next_ut - first_ut) + , (now_ut - first_ut), rd->last_calculated_value + ); + + // this is wrong + // it fades the value towards the target + // while we know the calculated value is different + // if(likely(next_ut + st->update_every * 1000000ULL > now_ut)) rd->calculated_value = new_value; + } + break; + } + + if(unlikely(!store_this_entry)) { + store_this_entry = 1; + continue; + } + + if(likely(rd->updated && rd->counter > 1 && iterations < st->gap_when_lost_iterations_above)) { + rd->values[st->current_entry] = pack_storage_number(new_value, storage_flags ); + + if(unlikely(st->debug)) + debug(D_RRD_STATS, "%s/%s: STORE[%ld] " + CALCULATED_NUMBER_FORMAT " = " CALCULATED_NUMBER_FORMAT + , st->id, rd->name + , st->current_entry + , unpack_storage_number(rd->values[st->current_entry]), new_value + ); + } + else { + if(unlikely(st->debug)) debug(D_RRD_STATS, "%s/%s: STORE[%ld] = NON EXISTING " + , st->id, rd->name + , st->current_entry + ); + rd->values[st->current_entry] = pack_storage_number(0, SN_NOT_EXISTS); + } + + stored_entries++; + + if(unlikely(st->debug)) { + calculated_number t1 = new_value * (calculated_number)rd->multiplier / (calculated_number)rd->divisor; + calculated_number t2 = unpack_storage_number(rd->values[st->current_entry]); + calculated_number accuracy = accuracy_loss(t1, t2); + debug(D_RRD_STATS, "%s/%s: UNPACK[%ld] = " CALCULATED_NUMBER_FORMAT " FLAGS=0x%08x (original = " CALCULATED_NUMBER_FORMAT ", accuracy loss = " CALCULATED_NUMBER_FORMAT "%%%s)" + , st->id, rd->name + , st->current_entry + , t2 + , get_storage_number_flags(rd->values[st->current_entry]) + , t1 + , accuracy + , (accuracy > ACCURACY_LOSS) ? " **TOO BIG** " : "" + ); + + rd->collected_volume += t1; + rd->stored_volume += t2; + accuracy = accuracy_loss(rd->collected_volume, rd->stored_volume); + debug(D_RRD_STATS, "%s/%s: VOLUME[%ld] = " CALCULATED_NUMBER_FORMAT ", calculated = " CALCULATED_NUMBER_FORMAT ", accuracy loss = " CALCULATED_NUMBER_FORMAT "%%%s" + , st->id, rd->name + , st->current_entry + , rd->stored_volume + , rd->collected_volume + , accuracy + , (accuracy > ACCURACY_LOSS) ? " **TOO BIG** " : "" + ); + + } + } + // reset the storage flags for the next point, if any; + storage_flags = SN_EXISTS; + + st->counter++; + st->current_entry = ((st->current_entry + 1) >= st->entries) ? 0 : st->current_entry + 1; + last_ut = next_ut; + } + + // align next interpolation to last collection point + if(likely(stored_entries || !store_this_entry)) { + st->last_updated.tv_sec = st->last_collected_time.tv_sec; + st->last_updated.tv_usec = st->last_collected_time.tv_usec; + } + + for( rd = st->dimensions; likely(rd) ; rd = rd->next ) { + if(unlikely(!rd->updated)) continue; + + if(likely(stored_entries || !store_this_entry)) { + if(unlikely(st->debug)) debug(D_RRD_STATS, "%s/%s: setting last_collected_value (old: " COLLECTED_NUMBER_FORMAT ") to last_collected_value (new: " COLLECTED_NUMBER_FORMAT ")", st->id, rd->name, rd->last_collected_value, rd->collected_value); + rd->last_collected_value = rd->collected_value; + + if(unlikely(st->debug)) debug(D_RRD_STATS, "%s/%s: setting last_calculated_value (old: " CALCULATED_NUMBER_FORMAT ") to last_calculated_value (new: " CALCULATED_NUMBER_FORMAT ")", st->id, rd->name, rd->last_calculated_value, rd->calculated_value); + rd->last_calculated_value = rd->calculated_value; + } + + rd->calculated_value = 0; + rd->collected_value = 0; + rd->updated = 0; + + if(unlikely(st->debug)) debug(D_RRD_STATS, "%s/%s: END " + " last_collected_value = " COLLECTED_NUMBER_FORMAT + " collected_value = " COLLECTED_NUMBER_FORMAT + " last_calculated_value = " CALCULATED_NUMBER_FORMAT + " calculated_value = " CALCULATED_NUMBER_FORMAT + , st->id, rd->name + , rd->last_collected_value + , rd->collected_value + , rd->last_calculated_value + , rd->calculated_value + ); + } + st->last_collected_total = st->collected_total; + + // ALL DONE ABOUT THE DATA UPDATE + // -------------------------------------------------------------------- + + // find if there are any obsolete dimensions (not updated recently) + if(unlikely(rrd_delete_unupdated_dimensions)) { + + for( rd = st->dimensions; likely(rd) ; rd = rd->next ) + if((rd->last_collected_time.tv_sec + (rrd_delete_unupdated_dimensions * st->update_every)) < st->last_collected_time.tv_sec) + break; + + if(unlikely(rd)) { + // there is dimension to free + // upgrade our read lock to a write lock + pthread_rwlock_unlock(&st->rwlock); + pthread_rwlock_wrlock(&st->rwlock); + + for( rd = st->dimensions, last = NULL ; likely(rd) ; ) { + // remove it only it is not updated in rrd_delete_unupdated_dimensions seconds + + if(unlikely((rd->last_collected_time.tv_sec + (rrd_delete_unupdated_dimensions * st->update_every)) < st->last_collected_time.tv_sec)) { + info("Removing obsolete dimension '%s' (%s) of '%s' (%s).", rd->name, rd->id, st->name, st->id); + + if(unlikely(!last)) { + st->dimensions = rd->next; + rd->next = NULL; + rrddim_free(st, rd); + rd = st->dimensions; + continue; + } + else { + last->next = rd->next; + rd->next = NULL; + rrddim_free(st, rd); + rd = last->next; + continue; + } + } + + last = rd; + rd = rd->next; + } + + if(unlikely(!st->dimensions)) { + info("Disabling chart %s (%s) since it does not have any dimensions", st->name, st->id); + st->enabled = 0; + } + } + } + + pthread_rwlock_unlock(&st->rwlock); + + if(unlikely(pthread_setcancelstate(oldstate, NULL) != 0)) + error("Cannot set pthread cancel state to RESTORE (%d).", oldstate); + + return(st->usec_since_last_update); +} diff --git a/src/rrd.h b/src/rrd.h new file mode 100755 index 000000000..4211b7d56 --- /dev/null +++ b/src/rrd.h @@ -0,0 +1,340 @@ +#include <sys/time.h> +#include <pthread.h> +#include <stdint.h> + +#include "avl.h" +#include "storage_number.h" + +#ifndef NETDATA_RRD_H +#define NETDATA_RRD_H 1 + +#define UPDATE_EVERY 1 +#define UPDATE_EVERY_MAX 3600 +extern int rrd_update_every; + +#define RRD_DEFAULT_HISTORY_ENTRIES 3600 +#define RRD_HISTORY_ENTRIES_MAX (86400*10) +extern int rrd_default_history_entries; + +// time in seconds to delete unupdated dimensions +// set to zero to disable this feature +extern int rrd_delete_unupdated_dimensions; + +#define RRD_ID_LENGTH_MAX 1024 + +#define RRDSET_MAGIC "NETDATA RRD SET FILE V017" +#define RRDDIMENSION_MAGIC "NETDATA RRD DIMENSION FILE V017" + +typedef long long total_number; +#define TOTAL_NUMBER_FORMAT "%lld" + +// ---------------------------------------------------------------------------- +// chart types + +#define RRDSET_TYPE_LINE_NAME "line" +#define RRDSET_TYPE_AREA_NAME "area" +#define RRDSET_TYPE_STACKED_NAME "stacked" + +#define RRDSET_TYPE_LINE 0 +#define RRDSET_TYPE_AREA 1 +#define RRDSET_TYPE_STACKED 2 + +int rrdset_type_id(const char *name); +const char *rrdset_type_name(int chart_type); + + +// ---------------------------------------------------------------------------- +// memory mode + +#define RRD_MEMORY_MODE_RAM_NAME "ram" +#define RRD_MEMORY_MODE_MAP_NAME "map" +#define RRD_MEMORY_MODE_SAVE_NAME "save" + +#define RRD_MEMORY_MODE_RAM 0 +#define RRD_MEMORY_MODE_MAP 1 +#define RRD_MEMORY_MODE_SAVE 2 + +extern int rrd_memory_mode; + +extern const char *rrd_memory_mode_name(int id); +extern int rrd_memory_mode_id(const char *name); + + +// ---------------------------------------------------------------------------- +// algorithms types + +#define RRDDIM_ABSOLUTE_NAME "absolute" +#define RRDDIM_INCREMENTAL_NAME "incremental" +#define RRDDIM_PCENT_OVER_DIFF_TOTAL_NAME "percentage-of-incremental-row" +#define RRDDIM_PCENT_OVER_ROW_TOTAL_NAME "percentage-of-absolute-row" + +#define RRDDIM_ABSOLUTE 0 +#define RRDDIM_INCREMENTAL 1 +#define RRDDIM_PCENT_OVER_DIFF_TOTAL 2 +#define RRDDIM_PCENT_OVER_ROW_TOTAL 3 + +extern int rrddim_algorithm_id(const char *name); +extern const char *rrddim_algorithm_name(int chart_type); + +// ---------------------------------------------------------------------------- +// flags + +#define RRDDIM_FLAG_HIDDEN 0x00000001 // this dimension will not be offered to callers +#define RRDDIM_FLAG_DONT_DETECT_RESETS_OR_OVERFLOWS 0x00000002 // do not offer RESET or OVERFLOW info to callers + +// ---------------------------------------------------------------------------- +// RRD DIMENSION + +struct rrddim { + // ------------------------------------------------------------------------ + // binary indexing structures + + avl avl; // the binary index - this has to be first member! + + // ------------------------------------------------------------------------ + // the dimension definition + + char id[RRD_ID_LENGTH_MAX + 1]; // the id of this dimension (for internal identification) + + const char *name; // the name of this dimension (as presented to user) + // this is a pointer to the config structure + // since the config always has a higher priority + // (the user overwrites the name of the charts) + + int algorithm; // the algorithm that is applied to add new collected values + long multiplier; // the multiplier of the collected values + long divisor; // the divider of the collected values + + int mapped; // if set to non zero, this dimension is mapped to a file + + // ------------------------------------------------------------------------ + // members for temporary data we need for calculations + + uint32_t hash; // a simple hash of the id, to speed up searching / indexing + // instead of strcmp() every item in the binary index + // we first compare the hashes + + uint32_t flags; + + char cache_filename[FILENAME_MAX+1]; // the filename we load/save from/to this set + + unsigned long counter; // the number of times we added values to this rrdim + + int updated; // set to 0 after each calculation, to 1 after each collected value + // we use this to detect that a dimension is not updated + + struct timeval last_collected_time; // when was this dimension last updated + // this is actual date time we updated the last_collected_value + // THIS IS DIFFERENT FROM THE SAME MEMBER OF RRDSET + + calculated_number calculated_value; // the current calculated value, after applying the algorithm + calculated_number last_calculated_value; // the last calculated value + + collected_number collected_value; // the current value, as collected + collected_number last_collected_value; // the last value that was collected + + // the *_volume members are used to calculate the accuracy of the rounding done by the + // storage number - they are printed to debug.log when debug is enabled for a set. + calculated_number collected_volume; // the sum of all collected values so far + calculated_number stored_volume; // the sum of all stored values so far + + struct rrddim *next; // linking of dimensions within the same data set + + // ------------------------------------------------------------------------ + // members for checking the data when loading from disk + + long entries; // how many entries this dimension has in ram + // this is the same to the entries of the data set + // we set it here, to check the data when we load it from disk. + + int update_every; // every how many seconds is this updated + + unsigned long memsize; // the memory allocated for this dimension + + char magic[sizeof(RRDDIMENSION_MAGIC) + 1]; // a string to be saved, used to identify our data file + + // ------------------------------------------------------------------------ + // the values stored in this dimension, using our floating point numbers + + storage_number values[]; // the array of values - THIS HAS TO BE THE LAST MEMBER +}; +typedef struct rrddim RRDDIM; + + +// ---------------------------------------------------------------------------- +// RRDSET + +struct rrdset { + // ------------------------------------------------------------------------ + // binary indexing structures + + avl avl; // the index, with key the id - this has to be first! + avl avlname; // the index, with key the name + + // ------------------------------------------------------------------------ + // the set configuration + + char id[RRD_ID_LENGTH_MAX + 1]; // id of the data set + + const char *name; // the name of this dimension (as presented to user) + // this is a pointer to the config structure + // since the config always has a higher priority + // (the user overwrites the name of the charts) + + char *type; // the type of graph RRD_TYPE_* (a category, for determining graphing options) + char *family; // grouping sets under the same family + char *context; // the template of this data set + char *title; // title shown to user + char *units; // units of measurement + + int chart_type; + + int update_every; // every how many seconds is this updated? + + long entries; // total number of entries in the data set + + long current_entry; // the entry that is currently being updated + // it goes around in a round-robin fashion + + int enabled; + + int gap_when_lost_iterations_above; // after how many lost iterations a gap should be stored + // netdata will interpolate values for gaps lower than this + + long priority; + + int isdetail; // if set, the data set should be considered as a detail of another + // (the master data set should be the one that has the same family and is not detail) + + // ------------------------------------------------------------------------ + // members for temporary data we need for calculations + + int mapped; // if set to 1, this is memory mapped + + int debug; + + char *cache_dir; // the directory to store dimensions + char cache_filename[FILENAME_MAX+1]; // the filename to store this set + + pthread_rwlock_t rwlock; + + unsigned long counter; // the number of times we added values to this rrd + unsigned long counter_done; // the number of times we added values to this rrd + + uint32_t hash; // a simple hash on the id, to speed up searching + // we first compare hashes, and only if the hashes are equal we do string comparisons + + uint32_t hash_name; // a simple hash on the name + + unsigned long long usec_since_last_update; // the time in microseconds since the last collection of data + + struct timeval last_updated; // when this data set was last updated (updated every time the rrd_stats_done() function) + struct timeval last_collected_time; // when did this data set last collected values + + total_number collected_total; // used internally to calculate percentages + total_number last_collected_total; // used internally to calculate percentages + + struct rrdset *next; // linking of rrdsets + + // ------------------------------------------------------------------------ + // members for checking the data when loading from disk + + unsigned long memsize; // how much mem we have allocated for this (without dimensions) + + char magic[sizeof(RRDSET_MAGIC) + 1]; // our magic + + // ------------------------------------------------------------------------ + // the dimensions + + avl_tree dimensions_index; // the root of the dimensions index + RRDDIM *dimensions; // the actual data for every dimension +}; +typedef struct rrdset RRDSET; + +extern RRDSET *rrdset_root; +extern pthread_rwlock_t rrdset_root_rwlock; + +// ---------------------------------------------------------------------------- +// RRD SET functions + +extern char *rrdset_strncpy_name(char *to, const char *from, int length); +extern void rrdset_set_name(RRDSET *st, const char *name); + +extern char *rrdset_cache_dir(const char *id); + +extern void rrdset_reset(RRDSET *st); + +extern RRDSET *rrdset_create(const char *type + , const char *id + , const char *name + , const char *family + , const char *context + , const char *title + , const char *units + , long priority + , int update_every + , int chart_type); + +extern void rrdset_free_all(void); +extern void rrdset_save_all(void); + +extern RRDSET *rrdset_find(const char *id); +extern RRDSET *rrdset_find_bytype(const char *type, const char *id); +extern RRDSET *rrdset_find_byname(const char *name); + +extern void rrdset_next_usec(RRDSET *st, unsigned long long microseconds); +extern void rrdset_next(RRDSET *st); +extern void rrdset_next_plugins(RRDSET *st); + +extern unsigned long long rrdset_done(RRDSET *st); + +// get the total duration in seconds of the round robin database +#define rrdset_duration(st) ((time_t)( (((st)->counter >= ((unsigned long)(st)->entries))?(unsigned long)(st)->entries:(st)->counter) * (st)->update_every )) + +// get the timestamp of the last entry in the round robin database +#define rrdset_last_entry_t(st) ((time_t)(((st)->last_updated.tv_sec))) + +// get the timestamp of first entry in the round robin database +#define rrdset_first_entry_t(st) ((time_t)(rrdset_last_entry_t(st) - rrdset_duration(st))) + +// get the last slot updated in the round robin database +#define rrdset_last_slot(st) ((unsigned long)(((st)->current_entry == 0) ? (st)->entries - 1 : (st)->current_entry - 1)) + +// get the first / oldest slot updated in the round robin database +#define rrdset_first_slot(st) ((unsigned long)( (((st)->counter >= ((unsigned long)(st)->entries)) ? (unsigned long)( ((unsigned long)(st)->current_entry > 0) ? ((unsigned long)(st)->current_entry) : ((unsigned long)(st)->entries) ) - 1 : 0) )) + +// get the slot of the round robin database, for the given timestamp (t) +// it always returns a valid slot, although may not be for the time requested if the time is outside the round robin database +#define rrdset_time2slot(st, t) ( \ + ( (time_t)(t) >= rrdset_last_entry_t(st)) ? ( rrdset_last_slot(st) ) : \ + ( ((time_t)(t) <= rrdset_first_entry_t(st)) ? rrdset_first_slot(st) : \ + ( (rrdset_last_slot(st) >= (unsigned long)((rrdset_last_entry_t(st) - (time_t)(t)) / (unsigned long)((st)->update_every)) ) ? \ + (rrdset_last_slot(st) - (unsigned long)((rrdset_last_entry_t(st) - (time_t)(t)) / (unsigned long)((st)->update_every)) ) : \ + (rrdset_last_slot(st) - (unsigned long)((rrdset_last_entry_t(st) - (time_t)(t)) / (unsigned long)((st)->update_every)) + (unsigned long)(st)->entries ) \ + ))) + +// get the timestamp of a specific slot in the round robin database +#define rrdset_slot2time(st, slot) ( rrdset_last_entry_t(st) - \ + ((unsigned long)(st)->update_every * ( \ + ( (unsigned long)(slot) > rrdset_last_slot(st)) ? \ + ( (rrdset_last_slot(st) - (unsigned long)(slot) + (unsigned long)(st)->entries) ) : \ + ( (rrdset_last_slot(st) - (unsigned long)(slot)) )) \ + )) + +// ---------------------------------------------------------------------------- +// RRD DIMENSION functions + +extern RRDDIM *rrddim_add(RRDSET *st, const char *id, const char *name, long multiplier, long divisor, int algorithm); + +extern void rrddim_set_name(RRDSET *st, RRDDIM *rd, const char *name); +extern void rrddim_free(RRDSET *st, RRDDIM *rd); + +extern RRDDIM *rrddim_find(RRDSET *st, const char *id); + +extern int rrddim_hide(RRDSET *st, const char *id); +extern int rrddim_unhide(RRDSET *st, const char *id); + +extern collected_number rrddim_set_by_pointer(RRDSET *st, RRDDIM *rd, collected_number value); +extern collected_number rrddim_set(RRDSET *st, const char *id, collected_number value); + +#endif /* NETDATA_RRD_H */ diff --git a/src/rrd2json.c b/src/rrd2json.c new file mode 100755 index 000000000..ad453cb68 --- /dev/null +++ b/src/rrd2json.c @@ -0,0 +1,2009 @@ +#ifdef HAVE_CONFIG_H +#include <config.h> +#endif +#include <pthread.h> +#include <stdlib.h> +#include <string.h> +#include <stdint.h> + +#include "log.h" +#include "common.h" +#include "rrd2json.h" + +#define HOSTNAME_MAX 1024 +char *hostname = "unknown"; + +void rrd_stats_api_v1_chart(RRDSET *st, BUFFER *wb) +{ + pthread_rwlock_rdlock(&st->rwlock); + + buffer_sprintf(wb, + "\t\t{\n" + "\t\t\t\"id\": \"%s\",\n" + "\t\t\t\"name\": \"%s\",\n" + "\t\t\t\"type\": \"%s\",\n" + "\t\t\t\"family\": \"%s\",\n" + "\t\t\t\"context\": \"%s\",\n" + "\t\t\t\"title\": \"%s\",\n" + "\t\t\t\"priority\": %ld,\n" + "\t\t\t\"enabled\": %s,\n" + "\t\t\t\"units\": \"%s\",\n" + "\t\t\t\"data_url\": \"/api/v1/data?chart=%s\",\n" + "\t\t\t\"chart_type\": \"%s\",\n" + "\t\t\t\"duration\": %ld,\n" + "\t\t\t\"first_entry\": %lu,\n" + "\t\t\t\"last_entry\": %lu,\n" + "\t\t\t\"update_every\": %d,\n" + "\t\t\t\"dimensions\": {\n" + , st->id + , st->name + , st->type + , st->family + , st->context + , st->title + , st->priority + , st->enabled?"true":"false" + , st->units + , st->name + , rrdset_type_name(st->chart_type) + , st->entries * st->update_every + , rrdset_first_entry_t(st) + , rrdset_last_entry_t(st) + , st->update_every + ); + + unsigned long memory = st->memsize; + + int c = 0; + RRDDIM *rd; + for(rd = st->dimensions; rd ; rd = rd->next) { + if(rd->flags & RRDDIM_FLAG_HIDDEN) continue; + + memory += rd->memsize; + + buffer_sprintf(wb, + "%s" + "\t\t\t\t\"%s\": { \"name\": \"%s\" }" + , c?",\n":"" + , rd->id + , rd->name + ); + + c++; + } + + buffer_sprintf(wb, + "\n\t\t\t}\n" + "\t\t}" + ); + + pthread_rwlock_unlock(&st->rwlock); +} + +void rrd_stats_api_v1_charts(BUFFER *wb) +{ + long c; + RRDSET *st; + + buffer_sprintf(wb, "{\n" + "\t\"hostname\": \"%s\"" + ",\n\t\"update_every\": %d" + ",\n\t\"history\": %d" + ",\n\t\"charts\": {" + , hostname + , rrd_update_every + , rrd_default_history_entries + ); + + pthread_rwlock_rdlock(&rrdset_root_rwlock); + for(st = rrdset_root, c = 0; st ; st = st->next) { + if(st->enabled) { + if(c) buffer_strcat(wb, ","); + buffer_strcat(wb, "\n\t\t\""); + buffer_strcat(wb, st->id); + buffer_strcat(wb, "\": "); + rrd_stats_api_v1_chart(st, wb); + c++; + } + } + pthread_rwlock_unlock(&rrdset_root_rwlock); + + buffer_strcat(wb, "\n\t}\n}\n"); +} + + +unsigned long rrd_stats_one_json(RRDSET *st, char *options, BUFFER *wb) +{ + time_t now = time(NULL); + + pthread_rwlock_rdlock(&st->rwlock); + + buffer_sprintf(wb, + "\t\t{\n" + "\t\t\t\"id\": \"%s\",\n" + "\t\t\t\"name\": \"%s\",\n" + "\t\t\t\"type\": \"%s\",\n" + "\t\t\t\"family\": \"%s\",\n" + "\t\t\t\"context\": \"%s\",\n" + "\t\t\t\"title\": \"%s\",\n" + "\t\t\t\"priority\": %ld,\n" + "\t\t\t\"enabled\": %d,\n" + "\t\t\t\"units\": \"%s\",\n" + "\t\t\t\"url\": \"/data/%s/%s\",\n" + "\t\t\t\"chart_type\": \"%s\",\n" + "\t\t\t\"counter\": %ld,\n" + "\t\t\t\"entries\": %ld,\n" + "\t\t\t\"first_entry_t\": %lu,\n" + "\t\t\t\"last_entry\": %ld,\n" + "\t\t\t\"last_entry_t\": %lu,\n" + "\t\t\t\"last_entry_secs_ago\": %lu,\n" + "\t\t\t\"update_every\": %d,\n" + "\t\t\t\"isdetail\": %d,\n" + "\t\t\t\"usec_since_last_update\": %llu,\n" + "\t\t\t\"collected_total\": " TOTAL_NUMBER_FORMAT ",\n" + "\t\t\t\"last_collected_total\": " TOTAL_NUMBER_FORMAT ",\n" + "\t\t\t\"dimensions\": [\n" + , st->id + , st->name + , st->type + , st->family + , st->context + , st->title + , st->priority + , st->enabled + , st->units + , st->name, options?options:"" + , rrdset_type_name(st->chart_type) + , st->counter + , st->entries + , rrdset_first_entry_t(st) + , rrdset_last_slot(st) + , rrdset_last_entry_t(st) + , (now < rrdset_last_entry_t(st)) ? (time_t)0 : now - rrdset_last_entry_t(st) + , st->update_every + , st->isdetail + , st->usec_since_last_update + , st->collected_total + , st->last_collected_total + ); + + unsigned long memory = st->memsize; + + RRDDIM *rd; + for(rd = st->dimensions; rd ; rd = rd->next) { + + memory += rd->memsize; + + buffer_sprintf(wb, + "\t\t\t\t{\n" + "\t\t\t\t\t\"id\": \"%s\",\n" + "\t\t\t\t\t\"name\": \"%s\",\n" + "\t\t\t\t\t\"entries\": %ld,\n" + "\t\t\t\t\t\"isHidden\": %d,\n" + "\t\t\t\t\t\"algorithm\": \"%s\",\n" + "\t\t\t\t\t\"multiplier\": %ld,\n" + "\t\t\t\t\t\"divisor\": %ld,\n" + "\t\t\t\t\t\"last_entry_t\": %lu,\n" + "\t\t\t\t\t\"collected_value\": " COLLECTED_NUMBER_FORMAT ",\n" + "\t\t\t\t\t\"calculated_value\": " CALCULATED_NUMBER_FORMAT ",\n" + "\t\t\t\t\t\"last_collected_value\": " COLLECTED_NUMBER_FORMAT ",\n" + "\t\t\t\t\t\"last_calculated_value\": " CALCULATED_NUMBER_FORMAT ",\n" + "\t\t\t\t\t\"memory\": %lu\n" + "\t\t\t\t}%s\n" + , rd->id + , rd->name + , rd->entries + , (rd->flags & RRDDIM_FLAG_HIDDEN)?1:0 + , rrddim_algorithm_name(rd->algorithm) + , rd->multiplier + , rd->divisor + , rd->last_collected_time.tv_sec + , rd->collected_value + , rd->calculated_value + , rd->last_collected_value + , rd->last_calculated_value + , rd->memsize + , rd->next?",":"" + ); + } + + buffer_sprintf(wb, + "\t\t\t],\n" + "\t\t\t\"memory\" : %lu\n" + "\t\t}" + , memory + ); + + pthread_rwlock_unlock(&st->rwlock); + return memory; +} + +#define RRD_GRAPH_JSON_HEADER "{\n\t\"charts\": [\n" +#define RRD_GRAPH_JSON_FOOTER "\n\t]\n}\n" + +void rrd_stats_graph_json(RRDSET *st, char *options, BUFFER *wb) +{ + buffer_strcat(wb, RRD_GRAPH_JSON_HEADER); + rrd_stats_one_json(st, options, wb); + buffer_strcat(wb, RRD_GRAPH_JSON_FOOTER); +} + +void rrd_stats_all_json(BUFFER *wb) +{ + unsigned long memory = 0; + long c; + RRDSET *st; + + buffer_strcat(wb, RRD_GRAPH_JSON_HEADER); + + pthread_rwlock_rdlock(&rrdset_root_rwlock); + for(st = rrdset_root, c = 0; st ; st = st->next) { + if(st->enabled) { + if(c) buffer_strcat(wb, ",\n"); + memory += rrd_stats_one_json(st, NULL, wb); + c++; + } + } + pthread_rwlock_unlock(&rrdset_root_rwlock); + + buffer_sprintf(wb, "\n\t],\n" + "\t\"hostname\": \"%s\",\n" + "\t\"update_every\": %d,\n" + "\t\"history\": %d,\n" + "\t\"memory\": %lu\n" + "}\n" + , hostname + , rrd_update_every + , rrd_default_history_entries + , memory + ); +} + + + +// ---------------------------------------------------------------------------- + +// RRDR dimension options +#define RRDR_EMPTY 0x01 // the dimension contains / the value is empty (null) +#define RRDR_RESET 0x02 // the dimension contains / the value is reset +#define RRDR_HIDDEN 0x04 // the dimension contains / the value is hidden +#define RRDR_NONZERO 0x08 // the dimension contains / the value is non-zero + +// RRDR result options +#define RRDR_RESULT_OPTION_ABSOLUTE 0x00000001 +#define RRDR_RESULT_OPTION_RELATIVE 0x00000002 + +typedef struct rrdresult { + RRDSET *st; // the chart this result refers to + + uint32_t result_options; // RRDR_RESULT_OPTION_* + + int d; // the number of dimensions + long n; // the number of values in the arrays + long rows; // the number of rows used + + uint8_t *od; // the options for the dimensions + + time_t *t; // array of n timestamps + calculated_number *v; // array n x d values + uint8_t *o; // array n x d options + + long c; // current line ( -1 ~ n ), ( -1 = none, use rrdr_rows() to get number of rows ) + + long group; // how many collected values were grouped for each row + long update_every; // what is the suggested update frequency in seconds + + calculated_number min; + calculated_number max; + + time_t before; + time_t after; + + int has_st_lock; // if st is read locked by us +} RRDR; + +#define rrdr_rows(r) ((r)->rows) + +/* +static void rrdr_dump(RRDR *r) +{ + long c, i; + RRDDIM *d; + + fprintf(stderr, "\nCHART %s (%s)\n", r->st->id, r->st->name); + + for(c = 0, d = r->st->dimensions; d ;c++, d = d->next) { + fprintf(stderr, "DIMENSION %s (%s), %s%s%s%s\n" + , d->id + , d->name + , (r->od[c] & RRDR_EMPTY)?"EMPTY ":"" + , (r->od[c] & RRDR_RESET)?"RESET ":"" + , (r->od[c] & RRDR_HIDDEN)?"HIDDEN ":"" + , (r->od[c] & RRDR_NONZERO)?"NONZERO ":"" + ); + } + + if(r->rows <= 0) { + fprintf(stderr, "RRDR does not have any values in it.\n"); + return; + } + + fprintf(stderr, "RRDR includes %d values in it:\n", r->rows); + + // for each line in the array + for(i = 0; i < r->rows ;i++) { + calculated_number *cn = &r->v[ i * r->d ]; + uint8_t *co = &r->o[ i * r->d ]; + + // print the id and the timestamp of the line + fprintf(stderr, "%ld %ld ", i + 1, r->t[i]); + + // for each dimension + for(c = 0, d = r->st->dimensions; d ;c++, d = d->next) { + if(unlikely(r->od[c] & RRDR_HIDDEN)) continue; + if(unlikely(!(r->od[c] & RRDR_NONZERO))) continue; + + if(co[c] & RRDR_EMPTY) + fprintf(stderr, "null "); + else + fprintf(stderr, CALCULATED_NUMBER_FORMAT " %s%s%s%s " + , cn[c] + , (co[c] & RRDR_EMPTY)?"E":" " + , (co[c] & RRDR_RESET)?"R":" " + , (co[c] & RRDR_HIDDEN)?"H":" " + , (co[c] & RRDR_NONZERO)?"N":" " + ); + } + + fprintf(stderr, "\n"); + } +} +*/ + +void rrdr_disable_not_selected_dimensions(RRDR *r, const char *dims) +{ + char b[strlen(dims) + 1]; + char *o = b, *tok; + strcpy(o, dims); + + long c; + RRDDIM *d; + + // disable all of them + for(c = 0, d = r->st->dimensions; d ;c++, d = d->next) + r->od[c] |= RRDR_HIDDEN; + + while(o && *o && (tok = mystrsep(&o, ", |"))) { + if(!*tok) continue; + + // find it and enable it + for(c = 0, d = r->st->dimensions; d ;c++, d = d->next) { + if(!strcmp(d->name, tok)) { + r->od[c] &= ~RRDR_HIDDEN; + + // since the user needs this dimension + // make it appear as NONZERO, to return it + // even if the dimension has only zeros + r->od[c] |= RRDR_NONZERO; + } + } + } +} + +void rrdr_buffer_print_format(BUFFER *wb, uint32_t format) +{ + switch(format) { + case DATASOURCE_JSON: + buffer_strcat(wb, DATASOURCE_FORMAT_JSON); + break; + + case DATASOURCE_DATATABLE_JSON: + buffer_strcat(wb, DATASOURCE_FORMAT_DATATABLE_JSON); + break; + + case DATASOURCE_DATATABLE_JSONP: + buffer_strcat(wb, DATASOURCE_FORMAT_DATATABLE_JSONP); + break; + + case DATASOURCE_JSONP: + buffer_strcat(wb, DATASOURCE_FORMAT_JSONP); + break; + + case DATASOURCE_SSV: + buffer_strcat(wb, DATASOURCE_FORMAT_SSV); + break; + + case DATASOURCE_CSV: + buffer_strcat(wb, DATASOURCE_FORMAT_CSV); + break; + + case DATASOURCE_TSV: + buffer_strcat(wb, DATASOURCE_FORMAT_TSV); + break; + + case DATASOURCE_HTML: + buffer_strcat(wb, DATASOURCE_FORMAT_HTML); + break; + + case DATASOURCE_JS_ARRAY: + buffer_strcat(wb, DATASOURCE_FORMAT_JS_ARRAY); + break; + + case DATASOURCE_SSV_COMMA: + buffer_strcat(wb, DATASOURCE_FORMAT_SSV_COMMA); + break; + + default: + buffer_strcat(wb, "unknown"); + break; + } +} + +uint32_t rrdr_check_options(RRDR *r, uint32_t options, const char *dims) +{ + if(options & RRDR_OPTION_NONZERO) { + long i; + + if(dims && *dims) { + // the caller wants specific dimensions + // disable NONZERO option + // to make sure we don't accidentally prevent + // the specific dimensions from being returned + i = 0; + } + else { + // find how many dimensions are not zero + long c; + RRDDIM *rd; + for(c = 0, i = 0, rd = r->st->dimensions; rd && c < r->d ; c++, rd = rd->next) { + if(unlikely(r->od[c] & RRDR_HIDDEN)) continue; + if(unlikely(!(r->od[c] & RRDR_NONZERO))) continue; + i++; + } + } + + // if with nonzero we get i = 0 (no dimensions will be returned) + // disable nonzero to show all dimensions + if(!i) options &= ~RRDR_OPTION_NONZERO; + } + + return options; +} + +void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, uint32_t options, int string_value) +{ + long rows = rrdr_rows(r); + long c, i; + RRDDIM *rd; + + //info("JSONWRAPPER(): %s: BEGIN", r->st->id); + char kq[2] = "", // key quote + sq[2] = ""; // string quote + + if( options & RRDR_OPTION_GOOGLE_JSON ) { + kq[0] = '\0'; + sq[0] = '\''; + } + else { + kq[0] = '"'; + sq[0] = '"'; + } + + buffer_sprintf(wb, "{\n" + " %sapi%s: 1,\n" + " %sid%s: %s%s%s,\n" + " %sname%s: %s%s%s,\n" + " %sview_update_every%s: %d,\n" + " %supdate_every%s: %d,\n" + " %sfirst_entry%s: %u,\n" + " %slast_entry%s: %u,\n" + " %sbefore%s: %u,\n" + " %safter%s: %u,\n" + " %sdimension_names%s: [" + , kq, kq + , kq, kq, sq, r->st->id, sq + , kq, kq, sq, r->st->name, sq + , kq, kq, r->update_every + , kq, kq, r->st->update_every + , kq, kq, rrdset_first_entry_t(r->st) + , kq, kq, rrdset_last_entry_t(r->st) + , kq, kq, r->before + , kq, kq, r->after + , kq, kq); + + for(c = 0, i = 0, rd = r->st->dimensions; rd && c < r->d ;c++, rd = rd->next) { + if(unlikely(r->od[c] & RRDR_HIDDEN)) continue; + if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_NONZERO))) continue; + + if(i) buffer_strcat(wb, ", "); + buffer_strcat(wb, sq); + buffer_strcat(wb, rd->name); + buffer_strcat(wb, sq); + i++; + } + if(!i) { +#ifdef NETDATA_INTERNAL_CHECKS + error("RRDR is empty for %s (RRDR has %d dimensions, options is 0x%08x)", r->st->id, r->d, options); +#endif + rows = 0; + buffer_strcat(wb, sq); + buffer_strcat(wb, "no data"); + buffer_strcat(wb, sq); + } + + buffer_sprintf(wb, "],\n" + " %sdimension_ids%s: [" + , kq, kq); + + for(c = 0, i = 0, rd = r->st->dimensions; rd && c < r->d ;c++, rd = rd->next) { + if(unlikely(r->od[c] & RRDR_HIDDEN)) continue; + if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_NONZERO))) continue; + + if(i) buffer_strcat(wb, ", "); + buffer_strcat(wb, sq); + buffer_strcat(wb, rd->id); + buffer_strcat(wb, sq); + i++; + } + if(!i) { + rows = 0; + buffer_strcat(wb, sq); + buffer_strcat(wb, "no data"); + buffer_strcat(wb, sq); + } + + buffer_sprintf(wb, "],\n" + " %slatest_values%s: [" + , kq, kq); + + for(c = 0, i = 0, rd = r->st->dimensions; rd && c < r->d ;c++, rd = rd->next) { + if(unlikely(r->od[c] & RRDR_HIDDEN)) continue; + if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_NONZERO))) continue; + + if(i) buffer_strcat(wb, ", "); + i++; + + storage_number n = rd->values[rrdset_last_slot(r->st)]; + + if(!does_storage_number_exist(n)) + buffer_strcat(wb, "null"); + else + buffer_rrd_value(wb, unpack_storage_number(n)); + } + if(!i) { + rows = 0; + buffer_strcat(wb, "null"); + } + + buffer_sprintf(wb, "],\n" + " %sview_latest_values%s: [" + , kq, kq); + + i = 0; + if(rows) { + for(c = 0, i = 0, rd = r->st->dimensions; rd && c < r->d ;c++, rd = rd->next) { + if(unlikely(r->od[c] & RRDR_HIDDEN)) continue; + if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_NONZERO))) continue; + + if(i) buffer_strcat(wb, ", "); + i++; + + calculated_number *cn = &r->v[ (0) * r->d ]; + uint8_t *co = &r->o[ (0) * r->d ]; + + if(co[c] & RRDR_EMPTY) + buffer_strcat(wb, "null"); + else + buffer_rrd_value(wb, cn[c]); + } + } + if(!i) { + rows = 0; + buffer_strcat(wb, "null"); + } + + buffer_sprintf(wb, "],\n" + " %sdimensions%s: %d,\n" + " %spoints%s: %d,\n" + " %sformat%s: %s" + , kq, kq, i + , kq, kq, rows + , kq, kq, sq + ); + + rrdr_buffer_print_format(wb, format); + + buffer_sprintf(wb, "%s,\n" + " %sresult%s: " + , sq + , kq, kq + ); + + if(string_value) buffer_strcat(wb, sq); + //info("JSONWRAPPER(): %s: END", r->st->id); +} + +void rrdr_json_wrapper_end(RRDR *r, BUFFER *wb, uint32_t format, uint32_t options, int string_value) +{ + if(r) {;} + if(format) {;} + + char kq[2] = "", // key quote + sq[2] = ""; // string quote + + if( options & RRDR_OPTION_GOOGLE_JSON ) { + kq[0] = '\0'; + sq[0] = '\''; + } + else { + kq[0] = '"'; + sq[0] = '"'; + } + + if(string_value) buffer_strcat(wb, sq); + + buffer_sprintf(wb, ",\n %smin%s: ", kq, kq); + buffer_rrd_value(wb, r->min); + buffer_sprintf(wb, ",\n %smax%s: ", kq, kq); + buffer_rrd_value(wb, r->max); + buffer_strcat(wb, "\n}\n"); +} + +#define JSON_DATES_JS 1 +#define JSON_DATES_TIMESTAMP 2 + +static void rrdr2json(RRDR *r, BUFFER *wb, uint32_t options, int datatable) +{ + //info("RRD2JSON(): %s: BEGIN", r->st->id); + int row_annotations = 0, dates, dates_with_new = 0; + char kq[2] = "", // key quote + sq[2] = "", // string quote + pre_label[101] = "", // before each label + post_label[101] = "", // after each label + pre_date[101] = "", // the beginning of line, to the date + post_date[101] = "", // closing the date + pre_value[101] = "", // before each value + post_value[101] = "", // after each value + post_line[101] = "", // at the end of each row + normal_annotation[201] = "", // default row annotation + overflow_annotation[201] = "", // overflow row annotation + data_begin[101] = "", // between labels and values + finish[101] = ""; // at the end of everything + + if(datatable) { + dates = JSON_DATES_JS; + if( options & RRDR_OPTION_GOOGLE_JSON ) { + kq[0] = '\0'; + sq[0] = '\''; + } + else { + kq[0] = '"'; + sq[0] = '"'; + } + row_annotations = 1; + snprintf(pre_date, 100, " {%sc%s:[{%sv%s:%s", kq, kq, kq, kq, sq); + snprintf(post_date, 100, "%s}", sq); + snprintf(pre_label, 100, ",\n {%sid%s:%s%s,%slabel%s:%s", kq, kq, sq, sq, kq, kq, sq); + snprintf(post_label, 100, "%s,%spattern%s:%s%s,%stype%s:%snumber%s}", sq, kq, kq, sq, sq, kq, kq, sq, sq); + snprintf(pre_value, 100, ",{%sv%s:", kq, kq); + snprintf(post_value, 100, "}"); + snprintf(post_line, 100, "]}"); + snprintf(data_begin, 100, "\n ],\n %srows%s:\n [\n", kq, kq); + snprintf(finish, 100, "\n ]\n}"); + + snprintf(overflow_annotation, 200, ",{%sv%s:%sRESET OR OVERFLOW%s},{%sv%s:%sThe counters have been wrapped.%s}", kq, kq, sq, sq, kq, kq, sq, sq); + snprintf(normal_annotation, 200, ",{%sv%s:null},{%sv%s:null}", kq, kq, kq, kq); + + buffer_sprintf(wb, "{\n %scols%s:\n [\n", kq, kq, kq, kq); + buffer_sprintf(wb, " {%sid%s:%s%s,%slabel%s:%stime%s,%spattern%s:%s%s,%stype%s:%sdatetime%s},\n", kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq); + buffer_sprintf(wb, " {%sid%s:%s%s,%slabel%s:%s%s,%spattern%s:%s%s,%stype%s:%sstring%s,%sp%s:{%srole%s:%sannotation%s}},\n", kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, kq, kq, sq, sq); + buffer_sprintf(wb, " {%sid%s:%s%s,%slabel%s:%s%s,%spattern%s:%s%s,%stype%s:%sstring%s,%sp%s:{%srole%s:%sannotationText%s}}", kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, kq, kq, sq, sq); + + // remove the valueobjects flag + // google wants its own keys + if(options & RRDR_OPTION_OBJECTSROWS) + options &= ~RRDR_OPTION_OBJECTSROWS; + } + else { + kq[0] = '"'; + sq[0] = '"'; + if((options & RRDR_OPTION_SECONDS) || (options & RRDR_OPTION_MILLISECONDS)) { + dates = JSON_DATES_TIMESTAMP; + dates_with_new = 0; + } + else { + dates = JSON_DATES_JS; + dates_with_new = 1; + } + if( options & RRDR_OPTION_OBJECTSROWS ) + snprintf(pre_date, 100, " { "); + else + snprintf(pre_date, 100, " [ "); + snprintf(pre_label, 100, ", \""); + snprintf(post_label, 100, "\""); + snprintf(pre_value, 100, ", "); + if( options & RRDR_OPTION_OBJECTSROWS ) + snprintf(post_line, 100, "}"); + else + snprintf(post_line, 100, "]"); + snprintf(data_begin, 100, "],\n %sdata%s:\n [\n", kq, kq); + snprintf(finish, 100, "\n ]\n}"); + + buffer_sprintf(wb, "{\n %slabels%s: [", kq, kq); + buffer_sprintf(wb, "%stime%s", sq, sq); + } + + // ------------------------------------------------------------------------- + // print the JSON header + + long c, i; + RRDDIM *rd; + + // print the header lines + for(c = 0, i = 0, rd = r->st->dimensions; rd && c < r->d ;c++, rd = rd->next) { + if(unlikely(r->od[c] & RRDR_HIDDEN)) continue; + if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_NONZERO))) continue; + + buffer_strcat(wb, pre_label); + buffer_strcat(wb, rd->name); + buffer_strcat(wb, post_label); + i++; + } + if(!i) { + buffer_strcat(wb, pre_label); + buffer_strcat(wb, "no data"); + buffer_strcat(wb, post_label); + } + + // print the begin of row data + buffer_strcat(wb, data_begin); + + // if all dimensions are hidden, print a null + if(!i) { + buffer_strcat(wb, finish); + return; + } + + long start = 0, end = rrdr_rows(r), step = 1; + if((options & RRDR_OPTION_REVERSED)) { + start = rrdr_rows(r) - 1; + end = -1; + step = -1; + } + + // for each line in the array + calculated_number total = 1; + for(i = start; i != end ;i += step) { + calculated_number *cn = &r->v[ i * r->d ]; + uint8_t *co = &r->o[ i * r->d ]; + + time_t now = r->t[i]; + + if(dates == JSON_DATES_JS) { + // generate the local date time + struct tm tmbuf, *tm = localtime_r(&now, &tmbuf); + if(!tm) { error("localtime_r() failed."); continue; } + + if(likely(i != start)) buffer_strcat(wb, ",\n"); + buffer_strcat(wb, pre_date); + + if( options & RRDR_OPTION_OBJECTSROWS ) + buffer_sprintf(wb, "%stime%s: ", kq, kq); + + if(dates_with_new) + buffer_strcat(wb, "new "); + + buffer_jsdate(wb, tm->tm_year + 1900, tm->tm_mon, tm->tm_mday, tm->tm_hour, tm->tm_min, tm->tm_sec); + + buffer_strcat(wb, post_date); + + if(row_annotations) { + // google supports one annotation per row + int annotation_found = 0; + for(c = 0, rd = r->st->dimensions; rd ;c++, rd = rd->next) { + if(co[c] & RRDR_RESET) { + buffer_strcat(wb, overflow_annotation); + annotation_found = 1; + break; + } + } + if(!annotation_found) + buffer_strcat(wb, normal_annotation); + } + } + else { + // print the timestamp of the line + if(likely(i != start)) buffer_strcat(wb, ",\n"); + buffer_strcat(wb, pre_date); + + if( options & RRDR_OPTION_OBJECTSROWS ) + buffer_sprintf(wb, "%stime%s: ", kq, kq); + + buffer_rrd_value(wb, (calculated_number)r->t[i]); + // in ms + if(options & RRDR_OPTION_MILLISECONDS) buffer_strcat(wb, "000"); + + buffer_strcat(wb, post_date); + } + + if(unlikely(options & RRDR_OPTION_PERCENTAGE)) { + total = 0; + for(c = 0, rd = r->st->dimensions; rd && c < r->d ;c++, rd = rd->next) { + calculated_number n = cn[c]; + + if(likely((options & RRDR_OPTION_ABSOLUTE) && n < 0)) + n = -n; + + total += n; + } + // prevent a division by zero + if(total == 0) total = 1; + } + + // for each dimension + for(c = 0, rd = r->st->dimensions; rd && c < r->d ;c++, rd = rd->next) { + if(unlikely(r->od[c] & RRDR_HIDDEN)) continue; + if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_NONZERO))) continue; + + calculated_number n = cn[c]; + + buffer_strcat(wb, pre_value); + + if( options & RRDR_OPTION_OBJECTSROWS ) + buffer_sprintf(wb, "%s%s%s: ", kq, rd->name, kq); + + if(co[c] & RRDR_EMPTY) { + if(options & RRDR_OPTION_NULL2ZERO) + buffer_strcat(wb, "0"); + else + buffer_strcat(wb, "null"); + } + else { + if(unlikely((options & RRDR_OPTION_ABSOLUTE) && n < 0)) + n = -n; + + if(unlikely(options & RRDR_OPTION_PERCENTAGE)) + n = n * 100 / total; + + buffer_rrd_value(wb, n); + } + + buffer_strcat(wb, post_value); + } + + buffer_strcat(wb, post_line); + } + + buffer_strcat(wb, finish); + //info("RRD2JSON(): %s: END", r->st->id); +} + +static void rrdr2csv(RRDR *r, BUFFER *wb, uint32_t options, const char *startline, const char *separator, const char *endline, const char *betweenlines) +{ + //info("RRD2CSV(): %s: BEGIN", r->st->id); + long c, i; + RRDDIM *d; + + // print the csv header + for(c = 0, i = 0, d = r->st->dimensions; d && c < r->d ;c++, d = d->next) { + if(unlikely(r->od[c] & RRDR_HIDDEN)) continue; + if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_NONZERO))) continue; + + if(!i) { + buffer_strcat(wb, startline); + if(options & RRDR_OPTION_LABEL_QUOTES) buffer_strcat(wb, "\""); + buffer_strcat(wb, "time"); + if(options & RRDR_OPTION_LABEL_QUOTES) buffer_strcat(wb, "\""); + } + buffer_strcat(wb, separator); + if(options & RRDR_OPTION_LABEL_QUOTES) buffer_strcat(wb, "\""); + buffer_strcat(wb, d->name); + if(options & RRDR_OPTION_LABEL_QUOTES) buffer_strcat(wb, "\""); + i++; + } + buffer_strcat(wb, endline); + + if(!i) { + // no dimensions present + return; + } + + long start = 0, end = rrdr_rows(r), step = 1; + if((options & RRDR_OPTION_REVERSED)) { + start = rrdr_rows(r) - 1; + end = -1; + step = -1; + } + + // for each line in the array + calculated_number total = 1; + for(i = start; i != end ;i += step) { + calculated_number *cn = &r->v[ i * r->d ]; + uint8_t *co = &r->o[ i * r->d ]; + + buffer_strcat(wb, betweenlines); + buffer_strcat(wb, startline); + + time_t now = r->t[i]; + + if((options & RRDR_OPTION_SECONDS) || (options & RRDR_OPTION_MILLISECONDS)) { + // print the timestamp of the line + buffer_rrd_value(wb, (calculated_number)now); + // in ms + if(options & RRDR_OPTION_MILLISECONDS) buffer_strcat(wb, "000"); + } + else { + // generate the local date time + struct tm tmbuf, *tm = localtime_r(&now, &tmbuf); + if(!tm) { error("localtime() failed."); continue; } + buffer_date(wb, tm->tm_year + 1900, tm->tm_mon, tm->tm_mday, tm->tm_hour, tm->tm_min, tm->tm_sec); + } + + if(unlikely(options & RRDR_OPTION_PERCENTAGE)) { + total = 0; + for(c = 0, d = r->st->dimensions; d && c < r->d ;c++, d = d->next) { + calculated_number n = cn[c]; + + if(likely((options & RRDR_OPTION_ABSOLUTE) && n < 0)) + n = -n; + + total += n; + } + // prevent a division by zero + if(total == 0) total = 1; + } + + // for each dimension + for(c = 0, d = r->st->dimensions; d && c < r->d ;c++, d = d->next) { + if(unlikely(r->od[c] & RRDR_HIDDEN)) continue; + if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_NONZERO))) continue; + + buffer_strcat(wb, separator); + + calculated_number n = cn[c]; + + if(co[c] & RRDR_EMPTY) { + if(options & RRDR_OPTION_NULL2ZERO) + buffer_strcat(wb, "0"); + else + buffer_strcat(wb, "null"); + } + else { + if(unlikely((options & RRDR_OPTION_ABSOLUTE) && n < 0)) + n = -n; + + if(unlikely(options & RRDR_OPTION_PERCENTAGE)) + n = n * 100 / total; + + buffer_rrd_value(wb, n); + } + } + + buffer_strcat(wb, endline); + } + //info("RRD2CSV(): %s: END", r->st->id); +} + +static void rrdr2ssv(RRDR *r, BUFFER *wb, uint32_t options, const char *prefix, const char *separator, const char *suffix) +{ + //info("RRD2SSV(): %s: BEGIN", r->st->id); + long c, i; + RRDDIM *d; + + buffer_strcat(wb, prefix); + long start = 0, end = rrdr_rows(r), step = 1; + if((options & RRDR_OPTION_REVERSED)) { + start = rrdr_rows(r) - 1; + end = -1; + step = -1; + } + + // for each line in the array + calculated_number total = 1; + for(i = start; i != end ;i += step) { + + calculated_number *cn = &r->v[ i * r->d ]; + uint8_t *co = &r->o[ i * r->d ]; + + calculated_number sum = 0, min = 0, max = 0, v; + int all_null = 1, init = 1; + + if(unlikely(options & RRDR_OPTION_PERCENTAGE)) { + total = 0; + for(c = 0, d = r->st->dimensions; d && c < r->d ;c++, d = d->next) { + calculated_number n = cn[c]; + + if(likely((options & RRDR_OPTION_ABSOLUTE) && n < 0)) + n = -n; + + total += n; + } + // prevent a division by zero + if(total == 0) total = 1; + } + + // for each dimension + for(c = 0, d = r->st->dimensions; d && c < r->d ;c++, d = d->next) { + if(unlikely(r->od[c] & RRDR_HIDDEN)) continue; + if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_NONZERO))) continue; + + calculated_number n = cn[c]; + + if(likely((options & RRDR_OPTION_ABSOLUTE) && n < 0)) + n = -n; + + if(unlikely(options & RRDR_OPTION_PERCENTAGE)) + n = n * 100 / total; + + if(unlikely(init)) { + if(n > 0) { + min = 0; + max = n; + } + else { + min = n; + max = 0; + } + init = 0; + } + + if(likely(!(co[c] & RRDR_EMPTY))) { + all_null = 0; + sum += n; + } + + if(n < min) min = n; + if(n > max) max = n; + } + + if(likely(i != start)) + buffer_strcat(wb, separator); + + if(all_null) { + if(options & RRDR_OPTION_NULL2ZERO) + buffer_strcat(wb, "0"); + else + buffer_strcat(wb, "null"); + } + else { + if(options & RRDR_OPTION_MIN2MAX) + v = max - min; + else + v = sum; + + if(likely(i != start)) { + if(r->min > v) r->min = v; + if(r->max < v) r->max = v; + } + else { + r->min = v; + r->max = v; + } + + buffer_rrd_value(wb, v); + } + } + buffer_strcat(wb, suffix); + //info("RRD2SSV(): %s: END", r->st->id); +} + +inline static calculated_number *rrdr_line_values(RRDR *r) +{ + return &r->v[ r->c * r->d ]; +} + +inline static uint8_t *rrdr_line_options(RRDR *r) +{ + return &r->o[ r->c * r->d ]; +} + +inline static int rrdr_line_init(RRDR *r, time_t t) +{ + r->c++; + + if(unlikely(r->c >= r->n)) { + error("requested to step above RRDR size for chart %s", r->st->name); + r->c = r->n - 1; + } + + // save the time + r->t[r->c] = t; + + return 1; +} + +inline static void rrdr_lock_rrdset(RRDR *r) { + if(unlikely(!r)) { + error("NULL value given!"); + return; + } + + pthread_rwlock_rdlock(&r->st->rwlock); + r->has_st_lock = 1; +} + +inline static void rrdr_unlock_rrdset(RRDR *r) { + if(unlikely(!r)) { + error("NULL value given!"); + return; + } + + if(likely(r->has_st_lock)) { + pthread_rwlock_unlock(&r->st->rwlock); + r->has_st_lock = 0; + } +} + +inline static void rrdr_free(RRDR *r) +{ + if(unlikely(!r)) { + error("NULL value given!"); + return; + } + + rrdr_unlock_rrdset(r); + if(likely(r->t)) free(r->t); + if(likely(r->v)) free(r->v); + if(likely(r->o)) free(r->o); + if(likely(r->od)) free(r->od); + free(r); +} + +inline void rrdr_done(RRDR *r) +{ + r->rows = r->c + 1; + r->c = 0; +} + +static RRDR *rrdr_create(RRDSET *st, long n) +{ + if(unlikely(!st)) { + error("NULL value given!"); + return NULL; + } + + RRDR *r = calloc(1, sizeof(RRDR)); + if(unlikely(!r)) goto cleanup; + + r->st = st; + + rrdr_lock_rrdset(r); + + RRDDIM *rd; + for(rd = st->dimensions ; rd ; rd = rd->next) r->d++; + + r->n = n; + + r->t = malloc(n * sizeof(time_t)); + if(unlikely(!r->t)) goto cleanup; + + r->v = malloc(n * r->d * sizeof(calculated_number)); + if(unlikely(!r->v)) goto cleanup; + + r->o = malloc(n * r->d * sizeof(uint8_t)); + if(unlikely(!r->o)) goto cleanup; + + r->od = malloc(r->d * sizeof(uint8_t)); + if(unlikely(!r->od)) goto cleanup; + + // set the hidden flag on hidden dimensions + int c; + for(c = 0, rd = st->dimensions ; rd ; c++, rd = rd->next) { + if(unlikely(rd->flags & RRDDIM_FLAG_HIDDEN)) r->od[c] = RRDR_HIDDEN; + else r->od[c] = 0; + } + + r->c = -1; + + r->group = 1; + r->update_every = 1; + + return r; + +cleanup: + error("Cannot allocate RRDR memory for %d entries", n); + if(likely(r)) rrdr_free(r); + return NULL; +} + +RRDR *rrd2rrdr(RRDSET *st, long points, long long after, long long before, int group_method) +{ + int debug = st->debug; + int absolute_period_requested = -1; + + time_t first_entry_t = rrdset_first_entry_t(st); + time_t last_entry_t = rrdset_last_entry_t(st); + + if(before == 0 && after == 0) { + before = last_entry_t; + after = first_entry_t; + absolute_period_requested = 0; + } + + // allow relative for before and after + if(before <= st->update_every * st->entries) { + before = last_entry_t + before; + absolute_period_requested = 0; + } + + if(after <= st->update_every * st->entries) { + after = last_entry_t + after; + absolute_period_requested = 0; + } + + if(absolute_period_requested == -1) + absolute_period_requested = 1; + + // make sure they are within our timeframe + if(before > last_entry_t) before = last_entry_t; + if(before < first_entry_t) before = first_entry_t; + + if(after > last_entry_t) after = last_entry_t; + if(after < first_entry_t) after = first_entry_t; + + // check if they are upside down + if(after > before) { + time_t tmp = before; + before = after; + after = tmp; + } + + // the duration of the chart + time_t duration = before - after; + long available_points = duration / st->update_every; + + if(duration <= 0 || available_points <= 0) + return rrdr_create(st, 1); + + // check the wanted points + if(points < 0) points = -points; + if(points > available_points) points = available_points; + if(points == 0) points = available_points; + + // calculate proper grouping of source data + long group = available_points / points; + if(group <= 0) group = 1; + + // round group to the closest integer + if(available_points % points > points / 2) group++; + + time_t after_new = after - (after % (group * st->update_every)); + time_t before_new = before - (before % (group * st->update_every)); + long points_new = (before_new - after_new) / st->update_every / group; + + // find the starting and ending slots in our round robin db + long start_at_slot = rrdset_time2slot(st, before_new), + stop_at_slot = rrdset_time2slot(st, after_new); + +#ifdef NETDATA_INTERNAL_CHECKS + if(after_new < first_entry_t) { + error("after_new %u is too small, minimum %u", after_new, first_entry_t); + } + if(after_new > last_entry_t) { + error("after_new %u is too big, maximum %u", after_new, last_entry_t); + } + if(before_new < first_entry_t) { + error("before_new %u is too small, minimum %u", before_new, first_entry_t); + } + if(before_new > last_entry_t) { + error("before_new %u is too big, maximum %u", before_new, last_entry_t); + } + if(start_at_slot < 0 || start_at_slot >= st->entries) { + error("start_at_slot is invalid %ld, expected %ld to %ld", start_at_slot, 0, st->entries - 1); + } + if(stop_at_slot < 0 || stop_at_slot >= st->entries) { + error("stop_at_slot is invalid %ld, expected %ld to %ld", stop_at_slot, 0, st->entries - 1); + } + if(points_new > (before_new - after_new) / group / st->update_every + 1) { + error("points_new %ld is more than points %ld", points_new, (before_new - after_new) / group / st->update_every + 1); + } +#endif + + //info("RRD2RRDR(): %s: wanted %ld points, got %ld - group=%ld, wanted duration=%u, got %u - wanted %ld - %ld, got %ld - %ld", st->id, points, points_new, group, before - after, before_new - after_new, after, before, after_new, before_new); + + after = after_new; + before = before_new; + duration = before - after; + points = points_new; + + // Now we have: + // before = the end time of the calculation + // after = the start time of the calculation + // duration = the duration of the calculation + // group = the number of source points to aggregate / group together + // method = the method of grouping source points + // points = the number of points to generate + + + // ------------------------------------------------------------------------- + // initialize our result set + + RRDR *r = rrdr_create(st, points); + if(!r) { +#ifdef NETDATA_INTERNAL_CHECKS + error("Cannot create RRDR for %s, after=%u, before=%u, duration=%u, points=%d", st->id, after, before, duration, points); +#endif + return NULL; + } + if(!r->d) { +#ifdef NETDATA_INTERNAL_CHECKS + error("Returning empty RRDR (no dimensions in RRDSET) for %s, after=%u, before=%u, duration=%u, points=%d", st->id, after, before, duration, points); +#endif + return r; + } + + if(absolute_period_requested == 1) + r->result_options |= RRDR_RESULT_OPTION_ABSOLUTE; + else + r->result_options |= RRDR_RESULT_OPTION_RELATIVE; + + // find how many dimensions we have + long dimensions = r->d; + + + // ------------------------------------------------------------------------- + // checks for debugging + + if(debug) debug(D_RRD_STATS, "INFO %s first_t: %lu, last_t: %lu, all_duration: %lu, after: %lu, before: %lu, duration: %lu, points: %ld, group: %ld" + , st->id + , first_entry_t + , last_entry_t + , last_entry_t - first_entry_t + , after + , before + , duration + , points + , group + ); + + + // ------------------------------------------------------------------------- + // temp arrays for keeping values per dimension + + calculated_number group_values[dimensions]; // keep sums when grouping + long group_counts[dimensions]; // keep the number of values added to group_values + uint8_t group_options[dimensions]; + uint8_t found_non_zero[dimensions]; + + + // initialize them + RRDDIM *rd; + long c; + for( rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) { + group_values[c] = 0; + group_counts[c] = 0; + group_options[c] = 0; + found_non_zero[c] = 0; + } + + + // ------------------------------------------------------------------------- + // the main loop + + time_t now = rrdset_slot2time(st, start_at_slot), + dt = st->update_every, + group_start_t = 0; + + if(unlikely(debug)) debug(D_RRD_STATS, "BEGIN %s after_t: %lu (stop_at_t: %ld), before_t: %lu (start_at_t: %ld), start_t(now): %lu, current_entry: %ld, entries: %ld" + , st->id + , after + , stop_at_slot + , before + , start_at_slot + , now + , st->current_entry + , st->entries + ); + + r->group = group; + r->update_every = group * st->update_every; + r->before = now; + r->after = now; + + //info("RRD2RRDR(): %s: STARTING", st->id); + + long slot = start_at_slot, counter = 0, stop_now = 0, added = 0, group_count = 0, add_this = 0; + for(; !stop_now ; now -= dt, slot--, counter++) { + if(unlikely(slot < 0)) slot = st->entries - 1; + if(unlikely(slot == stop_at_slot)) stop_now = counter; + + if(unlikely(debug)) debug(D_RRD_STATS, "ROW %s slot: %ld, entries_counter: %ld, group_count: %ld, added: %ld, now: %lu, %s %s" + , st->id + , slot + , counter + , group_count + 1 + , added + , now + , (group_count + 1 == group)?"PRINT":" - " + , (now >= after && now <= before)?"RANGE":" - " + ); + + // make sure we return data in the proper time range + if(unlikely(now > before)) continue; + if(unlikely(now < after)) break; + + if(unlikely(group_count == 0)) { + group_start_t = now; + } + group_count++; + + if(unlikely(group_count == group)) { + if(unlikely(added >= points)) break; + add_this = 1; + } + + // do the calculations + for(rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) { + storage_number n = rd->values[slot]; + if(unlikely(!does_storage_number_exist(n))) continue; + + group_counts[c]++; + + calculated_number value = unpack_storage_number(n); + if(likely(value != 0.0)) { + group_options[c] |= RRDR_NONZERO; + found_non_zero[c] = 1; + } + + if(unlikely(did_storage_number_reset(n))) + group_options[c] |= RRDR_RESET; + + switch(group_method) { + case GROUP_MAX: + if(unlikely(abs(value) > abs(group_values[c]))) + group_values[c] = value; + break; + + default: + case GROUP_SUM: + case GROUP_AVERAGE: + group_values[c] += value; + break; + } + } + + // added it + if(unlikely(add_this)) { + if(unlikely(!rrdr_line_init(r, group_start_t))) break; + + r->after = now; + + calculated_number *cn = rrdr_line_values(r); + uint8_t *co = rrdr_line_options(r); + + for(rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) { + + // update the dimension options + if(likely(found_non_zero[c])) r->od[c] |= RRDR_NONZERO; + + // store the specific point options + co[c] = group_options[c]; + + // store the value + if(unlikely(group_counts[c] == 0)) { + cn[c] = 0.0; + co[c] |= RRDR_EMPTY; + } + else if(unlikely(group_method == GROUP_AVERAGE)) { + cn[c] = group_values[c] / group_counts[c]; + } + else { + cn[c] = group_values[c]; + } + + if(cn[c] < r->min) r->min = cn[c]; + if(cn[c] > r->max) r->max = cn[c]; + + // reset them for the next loop + group_values[c] = 0; + group_counts[c] = 0; + group_options[c] = 0; + } + + added++; + group_count = 0; + add_this = 0; + } + } + + rrdr_done(r); + //info("RRD2RRDR(): %s: END %ld loops made, %ld points generated", st->id, counter, rrdr_rows(r)); + //error("SHIFT: %s: wanted %ld points, got %ld", st->id, points, rrdr_rows(r)); + return r; +} + +int rrd2format(RRDSET *st, BUFFER *wb, BUFFER *dimensions, uint32_t format, long points, long long after, long long before, int group_method, uint32_t options, time_t *latest_timestamp) +{ + RRDR *r = rrd2rrdr(st, points, after, before, group_method); + if(!r) { + buffer_strcat(wb, "Cannot generate output with these parameters on this chart."); + return 500; + } + + if(r->result_options & RRDR_RESULT_OPTION_RELATIVE) + wb->options |= WB_CONTENT_NO_CACHEABLE; + else if(r->result_options & RRDR_RESULT_OPTION_ABSOLUTE) + wb->options |= WB_CONTENT_CACHEABLE; + + options = rrdr_check_options(r, options, (dimensions)?buffer_tostring(dimensions):NULL); + + if(dimensions) + rrdr_disable_not_selected_dimensions(r, buffer_tostring(dimensions)); + + if(latest_timestamp && rrdr_rows(r) > 0) + *latest_timestamp = r->before; + + switch(format) { + case DATASOURCE_SSV: + if(options & RRDR_OPTION_JSON_WRAP) { + wb->contenttype = CT_APPLICATION_JSON; + rrdr_json_wrapper_begin(r, wb, format, options, 1); + rrdr2ssv(r, wb, options, "", " ", ""); + rrdr_json_wrapper_end(r, wb, format, options, 1); + } + else { + wb->contenttype = CT_TEXT_PLAIN; + rrdr2ssv(r, wb, options, "", " ", ""); + } + break; + + case DATASOURCE_SSV_COMMA: + if(options & RRDR_OPTION_JSON_WRAP) { + wb->contenttype = CT_APPLICATION_JSON; + rrdr_json_wrapper_begin(r, wb, format, options, 1); + rrdr2ssv(r, wb, options, "", ",", ""); + rrdr_json_wrapper_end(r, wb, format, options, 1); + } + else { + wb->contenttype = CT_TEXT_PLAIN; + rrdr2ssv(r, wb, options, "", ",", ""); + } + break; + + case DATASOURCE_JS_ARRAY: + if(options & RRDR_OPTION_JSON_WRAP) { + wb->contenttype = CT_APPLICATION_JSON; + rrdr_json_wrapper_begin(r, wb, format, options, 0); + rrdr2ssv(r, wb, options, "[", ",", "]"); + rrdr_json_wrapper_end(r, wb, format, options, 0); + } + else { + wb->contenttype = CT_APPLICATION_JSON; + rrdr2ssv(r, wb, options, "[", ",", "]"); + } + break; + + case DATASOURCE_CSV: + if(options & RRDR_OPTION_JSON_WRAP) { + wb->contenttype = CT_APPLICATION_JSON; + rrdr_json_wrapper_begin(r, wb, format, options, 1); + rrdr2csv(r, wb, options, "", ",", "\\n", ""); + rrdr_json_wrapper_end(r, wb, format, options, 1); + } + else { + wb->contenttype = CT_TEXT_PLAIN; + rrdr2csv(r, wb, options, "", ",", "\r\n", ""); + } + break; + + case DATASOURCE_CSV_JSON_ARRAY: + wb->contenttype = CT_APPLICATION_JSON; + if(options & RRDR_OPTION_JSON_WRAP) { + rrdr_json_wrapper_begin(r, wb, format, options, 0); + buffer_strcat(wb, "[\n"); + rrdr2csv(r, wb, options + RRDR_OPTION_LABEL_QUOTES, "[", ",", "]", ",\n"); + buffer_strcat(wb, "\n]"); + rrdr_json_wrapper_end(r, wb, format, options, 0); + } + else { + wb->contenttype = CT_TEXT_PLAIN; + buffer_strcat(wb, "[\n"); + rrdr2csv(r, wb, options + RRDR_OPTION_LABEL_QUOTES, "[", ",", "]", ",\n"); + buffer_strcat(wb, "\n]"); + } + break; + + case DATASOURCE_TSV: + if(options & RRDR_OPTION_JSON_WRAP) { + wb->contenttype = CT_APPLICATION_JSON; + rrdr_json_wrapper_begin(r, wb, format, options, 1); + rrdr2csv(r, wb, options, "", "\t", "\\n", ""); + rrdr_json_wrapper_end(r, wb, format, options, 1); + } + else { + wb->contenttype = CT_TEXT_PLAIN; + rrdr2csv(r, wb, options, "", "\t", "\r\n", ""); + } + break; + + case DATASOURCE_HTML: + if(options & RRDR_OPTION_JSON_WRAP) { + wb->contenttype = CT_APPLICATION_JSON; + rrdr_json_wrapper_begin(r, wb, format, options, 1); + buffer_strcat(wb, "<html>\\n<center>\\n<table border=\\\"0\\\" cellpadding=\\\"5\\\" cellspacing=\\\"5\\\">\\n"); + rrdr2csv(r, wb, options, "<tr><td>", "</td><td>", "</td></tr>\\n", ""); + buffer_strcat(wb, "</table>\\n</center>\\n</html>\\n"); + rrdr_json_wrapper_end(r, wb, format, options, 1); + } + else { + wb->contenttype = CT_TEXT_HTML; + buffer_strcat(wb, "<html>\n<center>\n<table border=\"0\" cellpadding=\"5\" cellspacing=\"5\">\n"); + rrdr2csv(r, wb, options, "<tr><td>", "</td><td>", "</td></tr>\n", ""); + buffer_strcat(wb, "</table>\n</center>\n</html>\n"); + } + break; + + case DATASOURCE_DATATABLE_JSONP: + wb->contenttype = CT_APPLICATION_X_JAVASCRIPT; + + if(options & RRDR_OPTION_JSON_WRAP) + rrdr_json_wrapper_begin(r, wb, format, options, 0); + + rrdr2json(r, wb, options, 1); + + if(options & RRDR_OPTION_JSON_WRAP) + rrdr_json_wrapper_end(r, wb, format, options, 0); + break; + + case DATASOURCE_DATATABLE_JSON: + wb->contenttype = CT_APPLICATION_JSON; + + if(options & RRDR_OPTION_JSON_WRAP) + rrdr_json_wrapper_begin(r, wb, format, options, 0); + + rrdr2json(r, wb, options, 1); + + if(options & RRDR_OPTION_JSON_WRAP) + rrdr_json_wrapper_end(r, wb, format, options, 0); + break; + + case DATASOURCE_JSONP: + wb->contenttype = CT_APPLICATION_X_JAVASCRIPT; + if(options & RRDR_OPTION_JSON_WRAP) + rrdr_json_wrapper_begin(r, wb, format, options, 0); + + rrdr2json(r, wb, options, 0); + + if(options & RRDR_OPTION_JSON_WRAP) + rrdr_json_wrapper_end(r, wb, format, options, 0); + break; + + case DATASOURCE_JSON: + default: + wb->contenttype = CT_APPLICATION_JSON; + + if(options & RRDR_OPTION_JSON_WRAP) + rrdr_json_wrapper_begin(r, wb, format, options, 0); + + rrdr2json(r, wb, options, 0); + + if(options & RRDR_OPTION_JSON_WRAP) + rrdr_json_wrapper_end(r, wb, format, options, 0); + break; + } + + rrdr_free(r); + return 200; +} + +time_t rrd_stats_json(int type, RRDSET *st, BUFFER *wb, long points, long group, int group_method, time_t after, time_t before, int only_non_zero) +{ + int c; + pthread_rwlock_rdlock(&st->rwlock); + + + // ------------------------------------------------------------------------- + // switch from JSON to google JSON + + char kq[2] = "\""; + char sq[2] = "\""; + switch(type) { + case DATASOURCE_DATATABLE_JSON: + case DATASOURCE_DATATABLE_JSONP: + kq[0] = '\0'; + sq[0] = '\''; + break; + + case DATASOURCE_JSON: + default: + break; + } + + + // ------------------------------------------------------------------------- + // validate the parameters + + if(points < 1) points = 1; + if(group < 1) group = 1; + + if(before == 0 || before > rrdset_last_entry_t(st)) before = rrdset_last_entry_t(st); + if(after == 0 || after < rrdset_first_entry_t(st)) after = rrdset_first_entry_t(st); + + // --- + + // our return value (the last timestamp printed) + // this is required to detect re-transmit in google JSONP + time_t last_timestamp = 0; + + + // ------------------------------------------------------------------------- + // find how many dimensions we have + + int dimensions = 0; + RRDDIM *rd; + for( rd = st->dimensions ; rd ; rd = rd->next) dimensions++; + if(!dimensions) { + pthread_rwlock_unlock(&st->rwlock); + buffer_strcat(wb, "No dimensions yet."); + return 0; + } + + + // ------------------------------------------------------------------------- + // prepare various strings, to speed up the loop + + char overflow_annotation[201]; snprintf(overflow_annotation, 200, ",{%sv%s:%sRESET OR OVERFLOW%s},{%sv%s:%sThe counters have been wrapped.%s}", kq, kq, sq, sq, kq, kq, sq, sq); + char normal_annotation[201]; snprintf(normal_annotation, 200, ",{%sv%s:null},{%sv%s:null}", kq, kq, kq, kq); + char pre_date[51]; snprintf(pre_date, 50, " {%sc%s:[{%sv%s:%s", kq, kq, kq, kq, sq); + char post_date[21]; snprintf(post_date, 20, "%s}", sq); + char pre_value[21]; snprintf(pre_value, 20, ",{%sv%s:", kq, kq); + char post_value[21]; snprintf(post_value, 20, "}"); + + + // ------------------------------------------------------------------------- + // checks for debugging + + if(st->debug) { + debug(D_RRD_STATS, "%s first_entry_t = %lu, last_entry_t = %lu, duration = %lu, after = %lu, before = %lu, duration = %lu, entries_to_show = %lu, group = %lu" + , st->id + , rrdset_first_entry_t(st) + , rrdset_last_entry_t(st) + , rrdset_last_entry_t(st) - rrdset_first_entry_t(st) + , after + , before + , before - after + , points + , group + ); + + if(before < after) + debug(D_RRD_STATS, "WARNING: %s The newest value in the database (%lu) is earlier than the oldest (%lu)", st->name, before, after); + + if((before - after) > st->entries * st->update_every) + debug(D_RRD_STATS, "WARNING: %s The time difference between the oldest and the newest entries (%lu) is higher than the capacity of the database (%lu)", st->name, before - after, st->entries * st->update_every); + } + + + // ------------------------------------------------------------------------- + // temp arrays for keeping values per dimension + + calculated_number group_values[dimensions]; // keep sums when grouping + int print_hidden[dimensions]; // keep hidden flags + int found_non_zero[dimensions]; + int found_non_existing[dimensions]; + + // initialize them + for( rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) { + group_values[c] = 0; + print_hidden[c] = (rd->flags & RRDDIM_FLAG_HIDDEN)?1:0; + found_non_zero[c] = 0; + found_non_existing[c] = 0; + } + + + // error("OLD: points=%d after=%d before=%d group=%d, duration=%d", entries_to_show, before - (st->update_every * group * entries_to_show), before, group, before - after + 1); + // rrd2array(st, entries_to_show, before - (st->update_every * group * entries_to_show), before, group_method, only_non_zero); + // rrd2rrdr(st, entries_to_show, before - (st->update_every * group * entries_to_show), before, group_method); + + // ------------------------------------------------------------------------- + // remove dimensions that contain only zeros + + int max_loop = 1; + if(only_non_zero) max_loop = 2; + + for(; max_loop ; max_loop--) { + + // ------------------------------------------------------------------------- + // print the JSON header + + buffer_sprintf(wb, "{\n %scols%s:\n [\n", kq, kq); + buffer_sprintf(wb, " {%sid%s:%s%s,%slabel%s:%stime%s,%spattern%s:%s%s,%stype%s:%sdatetime%s},\n", kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq); + buffer_sprintf(wb, " {%sid%s:%s%s,%slabel%s:%s%s,%spattern%s:%s%s,%stype%s:%sstring%s,%sp%s:{%srole%s:%sannotation%s}},\n", kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, kq, kq, sq, sq); + buffer_sprintf(wb, " {%sid%s:%s%s,%slabel%s:%s%s,%spattern%s:%s%s,%stype%s:%sstring%s,%sp%s:{%srole%s:%sannotationText%s}}", kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, kq, kq, sq, sq); + + // print the header for each dimension + // and update the print_hidden array for the dimensions that should be hidden + int pc = 0; + for( rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) { + if(!print_hidden[c]) { + pc++; + buffer_sprintf(wb, ",\n {%sid%s:%s%s,%slabel%s:%s%s%s,%spattern%s:%s%s,%stype%s:%snumber%s}", kq, kq, sq, sq, kq, kq, sq, rd->name, sq, kq, kq, sq, sq, kq, kq, sq, sq); + } + } + if(!pc) { + buffer_sprintf(wb, ",\n {%sid%s:%s%s,%slabel%s:%s%s%s,%spattern%s:%s%s,%stype%s:%snumber%s}", kq, kq, sq, sq, kq, kq, sq, "no data", sq, kq, kq, sq, sq, kq, kq, sq, sq); + } + + // print the begin of row data + buffer_sprintf(wb, "\n ],\n %srows%s:\n [\n", kq, kq); + + + // ------------------------------------------------------------------------- + // the main loop + + int annotate_reset = 0; + int annotation_count = 0; + + long t = rrdset_time2slot(st, before), + stop_at_t = rrdset_time2slot(st, after), + stop_now = 0; + + t -= t % group; + + time_t now = rrdset_slot2time(st, t), + dt = st->update_every; + + long count = 0, printed = 0, group_count = 0; + last_timestamp = 0; + + if(st->debug) debug(D_RRD_STATS, "%s: REQUEST after:%lu before:%lu, points:%d, group:%d, CHART cur:%ld first: %lu last:%lu, CALC start_t:%ld, stop_t:%ld" + , st->id + , after + , before + , points + , group + , st->current_entry + , rrdset_first_entry_t(st) + , rrdset_last_entry_t(st) + , t + , stop_at_t + ); + + long counter = 0; + for(; !stop_now ; now -= dt, t--, counter++) { + if(t < 0) t = st->entries - 1; + if(t == stop_at_t) stop_now = counter; + + int print_this = 0; + + if(st->debug) debug(D_RRD_STATS, "%s t = %ld, count = %ld, group_count = %ld, printed = %ld, now = %lu, %s %s" + , st->id + , t + , count + 1 + , group_count + 1 + , printed + , now + , (group_count + 1 == group)?"PRINT":" - " + , (now >= after && now <= before)?"RANGE":" - " + ); + + + // make sure we return data in the proper time range + if(now > before) continue; + if(now < after) break; + + //if(rrdset_slot2time(st, t) != now) + // error("%s: slot=%ld, now=%ld, slot2time=%ld, diff=%ld, last_entry_t=%ld, rrdset_last_slot=%ld", st->id, t, now, rrdset_slot2time(st,t), now - rrdset_slot2time(st,t), rrdset_last_entry_t(st), rrdset_last_slot(st)); + + count++; + group_count++; + + // check if we have to print this now + if(group_count == group) { + if(printed >= points) { + // debug(D_RRD_STATS, "Already printed all rows. Stopping."); + break; + } + + // generate the local date time + struct tm tmbuf, *tm = localtime_r(&now, &tmbuf); + if(!tm) { error("localtime() failed."); continue; } + if(now > last_timestamp) last_timestamp = now; + + if(printed) buffer_strcat(wb, "]},\n"); + buffer_strcat(wb, pre_date); + buffer_jsdate(wb, tm->tm_year + 1900, tm->tm_mon, tm->tm_mday, tm->tm_hour, tm->tm_min, tm->tm_sec); + buffer_strcat(wb, post_date); + + print_this = 1; + } + + // do the calculations + for(rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) { + storage_number n = rd->values[t]; + calculated_number value = unpack_storage_number(n); + + if(!does_storage_number_exist(n)) { + value = 0.0; + found_non_existing[c]++; + } + if(did_storage_number_reset(n)) annotate_reset = 1; + + switch(group_method) { + case GROUP_MAX: + if(abs(value) > abs(group_values[c])) group_values[c] = value; + break; + + case GROUP_SUM: + group_values[c] += value; + break; + + default: + case GROUP_AVERAGE: + group_values[c] += value; + if(print_this) group_values[c] /= ( group_count - found_non_existing[c] ); + break; + } + } + + if(print_this) { + if(annotate_reset) { + annotation_count++; + buffer_strcat(wb, overflow_annotation); + annotate_reset = 0; + } + else + buffer_strcat(wb, normal_annotation); + + pc = 0; + for(c = 0 ; c < dimensions ; c++) { + if(found_non_existing[c] == group_count) { + // all entries are non-existing + pc++; + buffer_strcat(wb, pre_value); + buffer_strcat(wb, "null"); + buffer_strcat(wb, post_value); + } + else if(!print_hidden[c]) { + pc++; + buffer_strcat(wb, pre_value); + buffer_rrd_value(wb, group_values[c]); + buffer_strcat(wb, post_value); + + if(group_values[c]) found_non_zero[c]++; + } + + // reset them for the next loop + group_values[c] = 0; + found_non_existing[c] = 0; + } + + // if all dimensions are hidden, print a null + if(!pc) { + buffer_strcat(wb, pre_value); + buffer_strcat(wb, "null"); + buffer_strcat(wb, post_value); + } + + printed++; + group_count = 0; + } + } + + if(printed) buffer_strcat(wb, "]}"); + buffer_strcat(wb, "\n ]\n}\n"); + + if(only_non_zero && max_loop > 1) { + int changed = 0; + for(rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) { + group_values[c] = 0; + found_non_existing[c] = 0; + + if(!print_hidden[c] && !found_non_zero[c]) { + changed = 1; + print_hidden[c] = 1; + } + } + + if(changed) buffer_flush(wb); + else break; + } + else break; + + } // max_loop + + debug(D_RRD_STATS, "RRD_STATS_JSON: %s total %ld bytes", st->name, wb->len); + + pthread_rwlock_unlock(&st->rwlock); + return last_timestamp; +} diff --git a/src/rrd2json.h b/src/rrd2json.h new file mode 100755 index 000000000..d1f850d4a --- /dev/null +++ b/src/rrd2json.h @@ -0,0 +1,69 @@ +#include <time.h> + +#include "web_buffer.h" +#include "rrd.h" + +#ifndef NETDATA_RRD2JSON_H +#define NETDATA_RRD2JSON_H 1 + +#define HOSTNAME_MAX 1024 +extern char *hostname; + +// type of JSON generations +#define DATASOURCE_INVALID -1 +#define DATASOURCE_JSON 0 +#define DATASOURCE_DATATABLE_JSON 1 +#define DATASOURCE_DATATABLE_JSONP 2 +#define DATASOURCE_SSV 3 +#define DATASOURCE_CSV 4 +#define DATASOURCE_JSONP 5 +#define DATASOURCE_TSV 6 +#define DATASOURCE_HTML 7 +#define DATASOURCE_JS_ARRAY 8 +#define DATASOURCE_SSV_COMMA 9 +#define DATASOURCE_CSV_JSON_ARRAY 10 + +#define DATASOURCE_FORMAT_JSON "json" +#define DATASOURCE_FORMAT_DATATABLE_JSON "datatable" +#define DATASOURCE_FORMAT_DATATABLE_JSONP "datasource" +#define DATASOURCE_FORMAT_JSONP "jsonp" +#define DATASOURCE_FORMAT_SSV "ssv" +#define DATASOURCE_FORMAT_CSV "csv" +#define DATASOURCE_FORMAT_TSV "tsv" +#define DATASOURCE_FORMAT_HTML "html" +#define DATASOURCE_FORMAT_JS_ARRAY "array" +#define DATASOURCE_FORMAT_SSV_COMMA "ssvcomma" +#define DATASOURCE_FORMAT_CSV_JSON_ARRAY "csvjsonarray" + +#define GROUP_AVERAGE 0 +#define GROUP_MAX 1 +#define GROUP_SUM 2 + +#define RRDR_OPTION_NONZERO 0x00000001 // don't output dimensions will just zero values +#define RRDR_OPTION_REVERSED 0x00000002 // output the rows in reverse order (oldest to newest) +#define RRDR_OPTION_ABSOLUTE 0x00000004 // values positive, for DATASOURCE_SSV before summing +#define RRDR_OPTION_MIN2MAX 0x00000008 // for DATASOURCE_SSV, out max - min, instead of sum +#define RRDR_OPTION_SECONDS 0x00000010 // output seconds, instead of dates +#define RRDR_OPTION_MILLISECONDS 0x00000020 // output milliseconds, instead of dates +#define RRDR_OPTION_NULL2ZERO 0x00000040 // do not show nulls, convert them to zeros +#define RRDR_OPTION_OBJECTSROWS 0x00000080 // each row of values should be an object, not an array +#define RRDR_OPTION_GOOGLE_JSON 0x00000100 // comply with google JSON/JSONP specs +#define RRDR_OPTION_JSON_WRAP 0x00000200 // wrap the response in a JSON header with info about the result +#define RRDR_OPTION_LABEL_QUOTES 0x00000400 // in CSV output, wrap header labels in double quotes +#define RRDR_OPTION_PERCENTAGE 0x00000800 // give values as percentage of total + +extern void rrd_stats_api_v1_chart(RRDSET *st, BUFFER *wb); +extern void rrd_stats_api_v1_charts(BUFFER *wb); + +extern unsigned long rrd_stats_one_json(RRDSET *st, char *options, BUFFER *wb); + +extern void rrd_stats_graph_json(RRDSET *st, char *options, BUFFER *wb); + +extern void rrd_stats_all_json(BUFFER *wb); + +extern time_t rrd_stats_json(int type, RRDSET *st, BUFFER *wb, long entries_to_show, long group, int group_method, time_t after, time_t before, int only_non_zero); + +extern int rrd2format(RRDSET *st, BUFFER *out, BUFFER *dimensions, uint32_t format, long points, long long after, long long before, int group_method, uint32_t options, time_t *latest_timestamp); + + +#endif /* NETDATA_RRD2JSON_H */ diff --git a/src/storage_number.c b/src/storage_number.c new file mode 100755 index 000000000..225cf0348 --- /dev/null +++ b/src/storage_number.c @@ -0,0 +1,206 @@ +#ifdef HAVE_CONFIG_H +#include <config.h> +#endif +#ifdef STORAGE_WITH_MATH +#include <math.h> +#endif + +#include "common.h" +#include "log.h" +#include "storage_number.h" + +#if __GNUC__ +#if __x86_64__ || __ppc64__ +#define ENVIRONMENT64 +#else +#define ENVIRONMENT32 +#endif +#endif + +storage_number pack_storage_number(calculated_number value, uint32_t flags) +{ + // bit 32 = sign 0:positive, 1:negative + // bit 31 = 0:divide, 1:multiply + // bit 30, 29, 28 = (multiplier or divider) 0-6 (7 total) + // bit 27, 26, 25 flags + // bit 24 to bit 1 = the value + + storage_number r = get_storage_number_flags(flags); + if(!value) return r; + + int m = 0; + calculated_number n = value; + + // if the value is negative + // add the sign bit and make it positive + if(n < 0) { + r += (1 << 31); // the sign bit 32 + n = -n; + } + + // make its integer part fit in 0x00ffffff + // by dividing it by 10 up to 7 times + // and increasing the multiplier + while(m < 7 && n > (calculated_number)0x00ffffff) { + n /= 10; + m++; + } + + if(m) { + // the value was too big and we divided it + // so we add a multiplier to unpack it + r += (1 << 30) + (m << 27); // the multiplier m + + if(n > (calculated_number)0x00ffffff) { + error("Number " CALCULATED_NUMBER_FORMAT " is too big.", value); + r += 0x00ffffff; + return r; + } + } + else { + // 0x0019999e is the number that can be multiplied + // by 10 to give 0x00ffffff + // while the value is below 0x0019999e we can + // multiply it by 10, up to 7 times, increasing + // the multiplier + while(m < 7 && n < (calculated_number)0x0019999e) { + n *= 10; + m++; + } + + // the value was small enough and we multiplied it + // so we add a divider to unpack it + r += (0 << 30) + (m << 27); // the divider m + } + +#ifdef STORAGE_WITH_MATH + // without this there are rounding problems + // example: 0.9 becomes 0.89 + r += lrint((double) n); +#else + r += (storage_number)n; +#endif + + return r; +} + +calculated_number unpack_storage_number(storage_number value) +{ + if(!value) return 0; + + int sign = 0, exp = 0; + + value ^= get_storage_number_flags(value); + + if(value & (1 << 31)) { + sign = 1; + value ^= 1 << 31; + } + + if(value & (1 << 30)) { + exp = 1; + value ^= 1 << 30; + } + + int mul = value >> 27; + value ^= mul << 27; + + calculated_number n = value; + + // fprintf(stderr, "UNPACK: %08X, sign = %d, exp = %d, mul = %d, n = " CALCULATED_NUMBER_FORMAT "\n", value, sign, exp, mul, n); + + while(mul > 0) { + if(exp) n *= 10; + else n /= 10; + mul--; + } + + if(sign) n = -n; + return n; +} + +#ifdef ENVIRONMENT32 +// This trick seems to give an 80% speed increase in 32bit systems +// print_calculated_number_llu_r() will just print the digits up to the +// point the remaining value fits in 32 bits, and then calls +// print_calculated_number_lu_r() to print the rest with 32 bit arithmetic. + +static char *print_calculated_number_lu_r(char *str, unsigned long uvalue) { + char *wstr = str; + + // print each digit + do *wstr++ = (char)(48 + (uvalue % 10)); while(uvalue /= 10); + return wstr; +} + +static char *print_calculated_number_llu_r(char *str, unsigned long long uvalue) { + char *wstr = str; + + // print each digit + do *wstr++ = (char)(48 + (uvalue % 10)); while((uvalue /= 10) && uvalue > (unsigned long long)0xffffffff); + if(uvalue) return print_calculated_number_lu_r(wstr, uvalue); + return wstr; +} +#endif + +int print_calculated_number(char *str, calculated_number value) +{ + char *wstr = str; + + int sign = (value < 0) ? 1 : 0; + if(sign) value = -value; + +#ifdef STORAGE_WITH_MATH + // without llrint() there are rounding problems + // for example 0.9 becomes 0.89 + unsigned long long uvalue = (unsigned long long int) llrint(value * (calculated_number)100000); +#else + unsigned long long uvalue = value * (calculated_number)100000; +#endif + +#ifdef ENVIRONMENT32 + if(uvalue > (unsigned long long)0xffffffff) + wstr = print_calculated_number_llu_r(str, uvalue); + else + wstr = print_calculated_number_lu_r(str, uvalue); +#else + do *wstr++ = (char)(48 + (uvalue % 10)); while(uvalue /= 10); +#endif + + // make sure we have 6 bytes at least + while((wstr - str) < 6) *wstr++ = '0'; + + // put the sign back + if(sign) *wstr++ = '-'; + + // reverse it + char *begin = str, *end = --wstr, aux; + while (end > begin) aux = *end, *end-- = *begin, *begin++ = aux; + // wstr--; + // strreverse(str, wstr); + + // remove trailing zeros + int decimal = 5; + while(decimal > 0 && *wstr == '0') { + *wstr-- = '\0'; + decimal--; + } + + // terminate it, one position to the right + // to let space for a dot + wstr[2] = '\0'; + + // make space for the dot + int i; + for(i = 0; i < decimal ;i++) { + wstr[1] = wstr[0]; + wstr--; + } + + // put the dot + if(wstr[2] == '\0') { wstr[1] = '\0'; decimal--; } + else wstr[1] = '.'; + + // return the buffer length + return (int) ((wstr - str) + 2 + decimal ); +} diff --git a/src/storage_number.h b/src/storage_number.h new file mode 100755 index 000000000..f35e99ddd --- /dev/null +++ b/src/storage_number.h @@ -0,0 +1,52 @@ +#ifndef NETDATA_STORAGE_NUMBER_H +#define NETDATA_STORAGE_NUMBER_H + +typedef long double calculated_number; +#define CALCULATED_NUMBER_FORMAT "%0.7Lf" +//typedef long long calculated_number; +//#define CALCULATED_NUMBER_FORMAT "%lld" + +typedef long long collected_number; +#define COLLECTED_NUMBER_FORMAT "%lld" + +/* +typedef long double collected_number; +#define COLLECTED_NUMBER_FORMAT "%0.7Lf" +*/ + +typedef int32_t storage_number; +typedef uint32_t ustorage_number; +#define STORAGE_NUMBER_FORMAT "%d" + +#define SN_NOT_EXISTS (0x0 << 24) +#define SN_EXISTS (0x1 << 24) +#define SN_EXISTS_RESET (0x2 << 24) +#define SN_EXISTS_UNDEF1 (0x3 << 24) +#define SN_EXISTS_UNDEF2 (0x4 << 24) +#define SN_EXISTS_UNDEF3 (0x5 << 24) +#define SN_EXISTS_UNDEF4 (0x6 << 24) + +#define SN_FLAGS_MASK (~(0x6 << 24)) + +// extract the flags +#define get_storage_number_flags(value) ((((storage_number)value) & (1 << 24)) | (((storage_number)value) & (2 << 24)) | (((storage_number)value) & (4 << 24))) + +// checks +#define does_storage_number_exist(value) ((get_storage_number_flags(value) != 0)?1:0) +#define did_storage_number_reset(value) ((get_storage_number_flags(value) == SN_EXISTS_RESET)?1:0) + +storage_number pack_storage_number(calculated_number value, uint32_t flags); +calculated_number unpack_storage_number(storage_number value); + +int print_calculated_number(char *str, calculated_number value); + +#define STORAGE_NUMBER_POSITIVE_MAX 167772150000000.0 +#define STORAGE_NUMBER_POSITIVE_MIN 0.00001 +#define STORAGE_NUMBER_NEGATIVE_MAX -0.00001 +#define STORAGE_NUMBER_NEGATIVE_MIN -167772150000000.0 + +// accepted accuracy loss +#define ACCURACY_LOSS 0.0001 +#define accuracy_loss(t1, t2) ((t1 == t2 || t1 == 0.0 || t2 == 0.0) ? 0.0 : (100.0 - ((t1 > t2) ? (t2 * 100.0 / t1 ) : (t1 * 100.0 / t2)))) + +#endif /* NETDATA_STORAGE_NUMBER_H */ diff --git a/src/sys_kernel_mm_ksm.c b/src/sys_kernel_mm_ksm.c new file mode 100755 index 000000000..822e0d41a --- /dev/null +++ b/src/sys_kernel_mm_ksm.c @@ -0,0 +1,150 @@ +#ifdef HAVE_CONFIG_H +#include <config.h> +#endif +#include <stdio.h> +#include <stdlib.h> +#include <string.h> + +#include "common.h" +#include "log.h" +#include "appconfig.h" +#include "procfile.h" +#include "rrd.h" +#include "plugin_proc.h" + +typedef struct name_value { + char filename[FILENAME_MAX + 1]; + unsigned long long value; +} NAME_VALUE; + +#define PAGES_SHARED 0 +#define PAGES_SHARING 1 +#define PAGES_UNSHARED 2 +#define PAGES_VOLATILE 3 +#define PAGES_TO_SCAN 4 + +NAME_VALUE values[] = { + [PAGES_SHARED] = { "/sys/kernel/mm/ksm/pages_shared", 0ULL }, + [PAGES_SHARING] = { "/sys/kernel/mm/ksm/pages_sharing", 0ULL }, + [PAGES_UNSHARED] = { "/sys/kernel/mm/ksm/pages_unshared", 0ULL }, + [PAGES_VOLATILE] = { "/sys/kernel/mm/ksm/pages_volatile", 0ULL }, + [PAGES_TO_SCAN] = { "/sys/kernel/mm/ksm/pages_to_scan", 0ULL }, +}; + +int do_sys_kernel_mm_ksm(int update_every, unsigned long long dt) { + static procfile *ff_pages_shared = NULL, *ff_pages_sharing = NULL, *ff_pages_unshared = NULL, *ff_pages_volatile = NULL, *ff_pages_to_scan = NULL; + static long page_size = -1; + + if(dt) {}; + + if(page_size == -1) + page_size = sysconf(_SC_PAGESIZE); + + if(!ff_pages_shared) { + snprintf(values[PAGES_SHARED].filename, FILENAME_MAX, "%s%s", global_host_prefix, "/sys/kernel/mm/ksm/pages_shared"); + snprintf(values[PAGES_SHARED].filename, FILENAME_MAX, "%s", config_get("plugin:proc:/sys/kernel/mm/ksm", "/sys/kernel/mm/ksm/pages_shared", values[PAGES_SHARED].filename)); + ff_pages_shared = procfile_open(values[PAGES_SHARED].filename, " \t:", PROCFILE_FLAG_DEFAULT); + } + + if(!ff_pages_sharing) { + snprintf(values[PAGES_SHARING].filename, FILENAME_MAX, "%s%s", global_host_prefix, "/sys/kernel/mm/ksm/pages_sharing"); + snprintf(values[PAGES_SHARING].filename, FILENAME_MAX, "%s", config_get("plugin:proc:/sys/kernel/mm/ksm", "/sys/kernel/mm/ksm/pages_sharing", values[PAGES_SHARING].filename)); + ff_pages_sharing = procfile_open(values[PAGES_SHARING].filename, " \t:", PROCFILE_FLAG_DEFAULT); + } + + if(!ff_pages_unshared) { + snprintf(values[PAGES_UNSHARED].filename, FILENAME_MAX, "%s%s", global_host_prefix, "/sys/kernel/mm/ksm/pages_unshared"); + snprintf(values[PAGES_UNSHARED].filename, FILENAME_MAX, "%s", config_get("plugin:proc:/sys/kernel/mm/ksm", "/sys/kernel/mm/ksm/pages_unshared", values[PAGES_UNSHARED].filename)); + ff_pages_unshared = procfile_open(values[PAGES_UNSHARED].filename, " \t:", PROCFILE_FLAG_DEFAULT); + } + + if(!ff_pages_volatile) { + snprintf(values[PAGES_VOLATILE].filename, FILENAME_MAX, "%s%s", global_host_prefix, "/sys/kernel/mm/ksm/pages_volatile"); + snprintf(values[PAGES_VOLATILE].filename, FILENAME_MAX, "%s", config_get("plugin:proc:/sys/kernel/mm/ksm", "/sys/kernel/mm/ksm/pages_volatile", values[PAGES_VOLATILE].filename)); + ff_pages_volatile = procfile_open(values[PAGES_VOLATILE].filename, " \t:", PROCFILE_FLAG_DEFAULT); + } + + if(!ff_pages_to_scan) { + snprintf(values[PAGES_TO_SCAN].filename, FILENAME_MAX, "%s%s", global_host_prefix, "/sys/kernel/mm/ksm/pages_to_scan"); + snprintf(values[PAGES_TO_SCAN].filename, FILENAME_MAX, "%s", config_get("plugin:proc:/sys/kernel/mm/ksm", "/sys/kernel/mm/ksm/pages_to_scan", values[PAGES_TO_SCAN].filename)); + ff_pages_to_scan = procfile_open(values[PAGES_TO_SCAN].filename, " \t:", PROCFILE_FLAG_DEFAULT); + } + + if(!ff_pages_shared || !ff_pages_sharing || !ff_pages_unshared || !ff_pages_volatile || !ff_pages_to_scan) return 1; + + unsigned long long pages_shared = 0, pages_sharing = 0, pages_unshared = 0, pages_volatile = 0, pages_to_scan = 0, offered = 0, saved = 0; + + ff_pages_shared = procfile_readall(ff_pages_shared); + if(!ff_pages_shared) return 0; // we return 0, so that we will retry to open it next time + pages_shared = strtoull(procfile_lineword(ff_pages_shared, 0, 0), NULL, 10); + + ff_pages_sharing = procfile_readall(ff_pages_sharing); + if(!ff_pages_sharing) return 0; // we return 0, so that we will retry to open it next time + pages_sharing = strtoull(procfile_lineword(ff_pages_sharing, 0, 0), NULL, 10); + + ff_pages_unshared = procfile_readall(ff_pages_unshared); + if(!ff_pages_unshared) return 0; // we return 0, so that we will retry to open it next time + pages_unshared = strtoull(procfile_lineword(ff_pages_unshared, 0, 0), NULL, 10); + + ff_pages_volatile = procfile_readall(ff_pages_volatile); + if(!ff_pages_volatile) return 0; // we return 0, so that we will retry to open it next time + pages_volatile = strtoull(procfile_lineword(ff_pages_volatile, 0, 0), NULL, 10); + + ff_pages_to_scan = procfile_readall(ff_pages_to_scan); + if(!ff_pages_to_scan) return 0; // we return 0, so that we will retry to open it next time + pages_to_scan = strtoull(procfile_lineword(ff_pages_to_scan, 0, 0), NULL, 10); + + offered = pages_sharing + pages_unshared + pages_volatile; + saved = pages_sharing - pages_shared; + + if(!offered || !pages_to_scan) return 0; + + RRDSET *st; + + // -------------------------------------------------------------------- + + st = rrdset_find("mem.ksm"); + if(!st) { + st = rrdset_create("mem", "ksm", NULL, "ksm", NULL, "Kernel Same Page Merging", "MB", 5000, update_every, RRDSET_TYPE_AREA); + + rrddim_add(st, "shared", NULL, 1, 1024 * 1024, RRDDIM_ABSOLUTE); + rrddim_add(st, "unshared", NULL, -1, 1024 * 1024, RRDDIM_ABSOLUTE); + rrddim_add(st, "sharing", NULL, 1, 1024 * 1024, RRDDIM_ABSOLUTE); + rrddim_add(st, "volatile", NULL, -1, 1024 * 1024, RRDDIM_ABSOLUTE); + rrddim_add(st, "to_scan", "to scan", -1, 1024 * 1024, RRDDIM_ABSOLUTE); + } + else rrdset_next(st); + + rrddim_set(st, "shared", pages_shared * page_size); + rrddim_set(st, "unshared", pages_unshared * page_size); + rrddim_set(st, "sharing", pages_sharing * page_size); + rrddim_set(st, "volatile", pages_volatile * page_size); + rrddim_set(st, "to_scan", pages_to_scan * page_size); + rrdset_done(st); + + st = rrdset_find("mem.ksm_savings"); + if(!st) { + st = rrdset_create("mem", "ksm_savings", NULL, "ksm", NULL, "Kernel Same Page Merging Savings", "MB", 5001, update_every, RRDSET_TYPE_AREA); + + rrddim_add(st, "savings", NULL, -1, 1024 * 1024, RRDDIM_ABSOLUTE); + rrddim_add(st, "offered", NULL, 1, 1024 * 1024, RRDDIM_ABSOLUTE); + } + else rrdset_next(st); + + rrddim_set(st, "savings", saved * page_size); + rrddim_set(st, "offered", offered * page_size); + rrdset_done(st); + + st = rrdset_find("mem.ksm_ratios"); + if(!st) { + st = rrdset_create("mem", "ksm_ratios", NULL, "ksm", NULL, "Kernel Same Page Merging Effectiveness", "percentage", 5002, update_every, RRDSET_TYPE_LINE); + + rrddim_add(st, "savings", NULL, 1, 10000, RRDDIM_ABSOLUTE); + } + else rrdset_next(st); + + rrddim_set(st, "savings", (saved * 1000000) / offered); + rrdset_done(st); + + return 0; +} diff --git a/src/unit_test.c b/src/unit_test.c new file mode 100755 index 000000000..47aa5396c --- /dev/null +++ b/src/unit_test.c @@ -0,0 +1,799 @@ +#ifdef HAVE_CONFIG_H +#include <config.h> +#endif +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <sys/resource.h> + +#include "common.h" +#include "storage_number.h" +#include "rrd.h" +#include "log.h" +#include "web_buffer.h" + +int check_storage_number(calculated_number n, int debug) { + char buffer[100]; + uint32_t flags = SN_EXISTS; + + storage_number s = pack_storage_number(n, flags); + calculated_number d = unpack_storage_number(s); + + if(!does_storage_number_exist(s)) { + fprintf(stderr, "Exists flags missing for number " CALCULATED_NUMBER_FORMAT "!\n", n); + return 5; + } + + calculated_number ddiff = d - n; + calculated_number dcdiff = ddiff * 100.0 / n; + + if(dcdiff < 0) dcdiff = -dcdiff; + + size_t len = print_calculated_number(buffer, d); + calculated_number p = strtold(buffer, NULL); + calculated_number pdiff = n - p; + calculated_number pcdiff = pdiff * 100.0 / n; + if(pcdiff < 0) pcdiff = -pcdiff; + + if(debug) { + fprintf(stderr, + CALCULATED_NUMBER_FORMAT " original\n" + CALCULATED_NUMBER_FORMAT " packed and unpacked, (stored as 0x%08X, diff " CALCULATED_NUMBER_FORMAT ", " CALCULATED_NUMBER_FORMAT "%%)\n" + "%s printed after unpacked (%zu bytes)\n" + CALCULATED_NUMBER_FORMAT " re-parsed from printed (diff " CALCULATED_NUMBER_FORMAT ", " CALCULATED_NUMBER_FORMAT "%%)\n\n", + n, + d, s, ddiff, dcdiff, + buffer, + len, p, pdiff, pcdiff + ); + if(len != strlen(buffer)) fprintf(stderr, "ERROR: printed number %s is reported to have length %zu but it has %zu\n", buffer, len, strlen(buffer)); + if(dcdiff > ACCURACY_LOSS) fprintf(stderr, "WARNING: packing number " CALCULATED_NUMBER_FORMAT " has accuracy loss %0.7Lf %%\n", n, dcdiff); + if(pcdiff > ACCURACY_LOSS) fprintf(stderr, "WARNING: re-parsing the packed, unpacked and printed number " CALCULATED_NUMBER_FORMAT " has accuracy loss %0.7Lf %%\n", n, pcdiff); + } + + if(len != strlen(buffer)) return 1; + if(dcdiff > ACCURACY_LOSS) return 3; + if(pcdiff > ACCURACY_LOSS) return 4; + return 0; +} + +void benchmark_storage_number(int loop, int multiplier) { + int i, j; + calculated_number n, d; + storage_number s; + unsigned long long user, system, total, mine, their; + + char buffer[100]; + + struct rusage now, last; + + fprintf(stderr, "\n\nBenchmarking %d numbers, please wait...\n\n", loop); + + // ------------------------------------------------------------------------ + + fprintf(stderr, "SYSTEM LONG DOUBLE SIZE: %zu bytes\n", sizeof(calculated_number)); + fprintf(stderr, "NETDATA FLOATING POINT SIZE: %zu bytes\n", sizeof(storage_number)); + + mine = (calculated_number)sizeof(storage_number) * (calculated_number)loop; + their = (calculated_number)sizeof(calculated_number) * (calculated_number)loop; + + if(mine > their) { + fprintf(stderr, "\nNETDATA NEEDS %0.2Lf TIMES MORE MEMORY. Sorry!\n", (long double)(mine / their)); + } + else { + fprintf(stderr, "\nNETDATA INTERNAL FLOATING POINT ARITHMETICS NEEDS %0.2Lf TIMES LESS MEMORY.\n", (long double)(their / mine)); + } + + fprintf(stderr, "\nNETDATA FLOATING POINT\n"); + fprintf(stderr, "MIN POSITIVE VALUE " CALCULATED_NUMBER_FORMAT "\n", (calculated_number)STORAGE_NUMBER_POSITIVE_MIN); + fprintf(stderr, "MAX POSITIVE VALUE " CALCULATED_NUMBER_FORMAT "\n", (calculated_number)STORAGE_NUMBER_POSITIVE_MAX); + fprintf(stderr, "MIN NEGATIVE VALUE " CALCULATED_NUMBER_FORMAT "\n", (calculated_number)STORAGE_NUMBER_NEGATIVE_MIN); + fprintf(stderr, "MAX NEGATIVE VALUE " CALCULATED_NUMBER_FORMAT "\n", (calculated_number)STORAGE_NUMBER_NEGATIVE_MAX); + fprintf(stderr, "Maximum accuracy loss: " CALCULATED_NUMBER_FORMAT "%%\n\n\n", (calculated_number)ACCURACY_LOSS); + + // ------------------------------------------------------------------------ + + fprintf(stderr, "INTERNAL LONG DOUBLE PRINTING: "); + getrusage(RUSAGE_SELF, &last); + + // do the job + for(j = 1; j < 11 ;j++) { + n = STORAGE_NUMBER_POSITIVE_MIN * j; + + for(i = 0; i < loop ;i++) { + n *= multiplier; + if(n > STORAGE_NUMBER_POSITIVE_MAX) n = STORAGE_NUMBER_POSITIVE_MIN; + + print_calculated_number(buffer, n); + } + } + + getrusage(RUSAGE_SELF, &now); + user = now.ru_utime.tv_sec * 1000000ULL + now.ru_utime.tv_usec - last.ru_utime.tv_sec * 1000000ULL + last.ru_utime.tv_usec; + system = now.ru_stime.tv_sec * 1000000ULL + now.ru_stime.tv_usec - last.ru_stime.tv_sec * 1000000ULL + last.ru_stime.tv_usec; + total = user + system; + mine = total; + + fprintf(stderr, "user %0.5Lf, system %0.5Lf, total %0.5Lf\n", (long double)(user / 1000000.0), (long double)(system / 1000000.0), (long double)(total / 1000000.0)); + + // ------------------------------------------------------------------------ + + fprintf(stderr, "SYSTEM LONG DOUBLE PRINTING: "); + getrusage(RUSAGE_SELF, &last); + + // do the job + for(j = 1; j < 11 ;j++) { + n = STORAGE_NUMBER_POSITIVE_MIN * j; + + for(i = 0; i < loop ;i++) { + n *= multiplier; + if(n > STORAGE_NUMBER_POSITIVE_MAX) n = STORAGE_NUMBER_POSITIVE_MIN; + snprintf(buffer, 100, CALCULATED_NUMBER_FORMAT, n); + } + } + + getrusage(RUSAGE_SELF, &now); + user = now.ru_utime.tv_sec * 1000000ULL + now.ru_utime.tv_usec - last.ru_utime.tv_sec * 1000000ULL + last.ru_utime.tv_usec; + system = now.ru_stime.tv_sec * 1000000ULL + now.ru_stime.tv_usec - last.ru_stime.tv_sec * 1000000ULL + last.ru_stime.tv_usec; + total = user + system; + their = total; + + fprintf(stderr, "user %0.5Lf, system %0.5Lf, total %0.5Lf\n", (long double)(user / 1000000.0), (long double)(system / 1000000.0), (long double)(total / 1000000.0)); + + if(mine > total) { + fprintf(stderr, "NETDATA CODE IS SLOWER %0.2Lf %%\n", (long double)(mine * 100.0 / their - 100.0)); + } + else { + fprintf(stderr, "NETDATA CODE IS F A S T E R %0.2Lf %%\n", (long double)(their * 100.0 / mine - 100.0)); + } + + // ------------------------------------------------------------------------ + + fprintf(stderr, "\nINTERNAL LONG DOUBLE PRINTING WITH PACK / UNPACK: "); + getrusage(RUSAGE_SELF, &last); + + // do the job + for(j = 1; j < 11 ;j++) { + n = STORAGE_NUMBER_POSITIVE_MIN * j; + + for(i = 0; i < loop ;i++) { + n *= multiplier; + if(n > STORAGE_NUMBER_POSITIVE_MAX) n = STORAGE_NUMBER_POSITIVE_MIN; + + s = pack_storage_number(n, 1); + d = unpack_storage_number(s); + print_calculated_number(buffer, d); + } + } + + getrusage(RUSAGE_SELF, &now); + user = now.ru_utime.tv_sec * 1000000ULL + now.ru_utime.tv_usec - last.ru_utime.tv_sec * 1000000ULL + last.ru_utime.tv_usec; + system = now.ru_stime.tv_sec * 1000000ULL + now.ru_stime.tv_usec - last.ru_stime.tv_sec * 1000000ULL + last.ru_stime.tv_usec; + total = user + system; + mine = total; + + fprintf(stderr, "user %0.5Lf, system %0.5Lf, total %0.5Lf\n", (long double)(user / 1000000.0), (long double)(system / 1000000.0), (long double)(total / 1000000.0)); + + if(mine > their) { + fprintf(stderr, "WITH PACKING UNPACKING NETDATA CODE IS SLOWER %0.2Lf %%\n", (long double)(mine * 100.0 / their - 100.0)); + } + else { + fprintf(stderr, "EVEN WITH PACKING AND UNPACKING, NETDATA CODE IS F A S T E R %0.2Lf %%\n", (long double)(their * 100.0 / mine - 100.0)); + } + + // ------------------------------------------------------------------------ + +} + +static int check_storage_number_exists() { + uint32_t flags = SN_EXISTS; + + + for(flags = 0; flags < 7 ; flags++) { + if(get_storage_number_flags(flags << 24) != flags << 24) { + fprintf(stderr, "Flag 0x%08x is not checked correctly. It became 0x%08x\n", flags << 24, get_storage_number_flags(flags << 24)); + return 1; + } + } + + flags = SN_EXISTS; + calculated_number n = 0.0; + + storage_number s = pack_storage_number(n, flags); + calculated_number d = unpack_storage_number(s); + if(get_storage_number_flags(s) != flags) { + fprintf(stderr, "Wrong flags. Given %08x, Got %08x!\n", flags, get_storage_number_flags(s)); + return 1; + } + if(n != d) { + fprintf(stderr, "Wrong number returned. Expected " CALCULATED_NUMBER_FORMAT ", returned " CALCULATED_NUMBER_FORMAT "!\n", n, d); + return 1; + } + + return 0; +} + +int unit_test_storage() +{ + if(check_storage_number_exists()) return 0; + + calculated_number c, a = 0; + int i, j, g, r = 0; + + for(g = -1; g <= 1 ; g++) { + a = 0; + + if(!g) continue; + + for(j = 0; j < 9 ;j++) { + a += 0.0000001; + c = a * g; + for(i = 0; i < 21 ;i++, c *= 10) { + if(c > 0 && c < STORAGE_NUMBER_POSITIVE_MIN) continue; + if(c < 0 && c > STORAGE_NUMBER_NEGATIVE_MAX) continue; + + if(check_storage_number(c, 1)) return 1; + } + } + } + + benchmark_storage_number(1000000, 2); + return r; +} + + +// -------------------------------------------------------------------------------------------------------------------- + +struct feed_values { + unsigned long long microseconds; + calculated_number value; +}; + +struct test { + char name[100]; + char description[1024]; + + int update_every; + unsigned long long multiplier; + unsigned long long divisor; + int algorithm; + + unsigned long feed_entries; + unsigned long result_entries; + struct feed_values *feed; + calculated_number *results; +}; + +// -------------------------------------------------------------------------------------------------------------------- +// test1 +// test absolute values stored + +struct feed_values test1_feed[] = { + { 0, 10 }, + { 1000000, 20 }, + { 1000000, 30 }, + { 1000000, 40 }, + { 1000000, 50 }, + { 1000000, 60 }, + { 1000000, 70 }, + { 1000000, 80 }, + { 1000000, 90 }, + { 1000000, 100 }, +}; + +calculated_number test1_results[] = { + 20, 30, 40, 50, 60, 70, 80, 90, 100 +}; + +struct test test1 = { + "test1", // name + "test absolute values stored at exactly second boundaries", + 1, // update_every + 1, // multiplier + 1, // divisor + RRDDIM_ABSOLUTE, // algorithm + 10, // feed entries + 9, // result entries + test1_feed, // feed + test1_results // results +}; + +// -------------------------------------------------------------------------------------------------------------------- +// test2 +// test absolute values stored in the middle of second boundaries + +struct feed_values test2_feed[] = { + { 500000, 10 }, + { 1000000, 20 }, + { 1000000, 30 }, + { 1000000, 40 }, + { 1000000, 50 }, + { 1000000, 60 }, + { 1000000, 70 }, + { 1000000, 80 }, + { 1000000, 90 }, + { 1000000, 100 }, +}; + +calculated_number test2_results[] = { + 20, 30, 40, 50, 60, 70, 80, 90, 100 +}; + +struct test test2 = { + "test2", // name + "test absolute values stored in the middle of second boundaries", + 1, // update_every + 1, // multiplier + 1, // divisor + RRDDIM_ABSOLUTE, // algorithm + 10, // feed entries + 9, // result entries + test2_feed, // feed + test2_results // results +}; + +// -------------------------------------------------------------------------------------------------------------------- +// test3 + +struct feed_values test3_feed[] = { + { 0, 10 }, + { 1000000, 20 }, + { 1000000, 30 }, + { 1000000, 40 }, + { 1000000, 50 }, + { 1000000, 60 }, + { 1000000, 70 }, + { 1000000, 80 }, + { 1000000, 90 }, + { 1000000, 100 }, +}; + +calculated_number test3_results[] = { + 10, 10, 10, 10, 10, 10, 10, 10, 10 +}; + +struct test test3 = { + "test3", // name + "test incremental values stored at exactly second boundaries", + 1, // update_every + 1, // multiplier + 1, // divisor + RRDDIM_INCREMENTAL, // algorithm + 10, // feed entries + 9, // result entries + test3_feed, // feed + test3_results // results +}; + +// -------------------------------------------------------------------------------------------------------------------- +// test4 + +struct feed_values test4_feed[] = { + { 500000, 10 }, + { 1000000, 20 }, + { 1000000, 30 }, + { 1000000, 40 }, + { 1000000, 50 }, + { 1000000, 60 }, + { 1000000, 70 }, + { 1000000, 80 }, + { 1000000, 90 }, + { 1000000, 100 }, +}; + +calculated_number test4_results[] = { + 5, 10, 10, 10, 10, 10, 10, 10, 10 +}; + +struct test test4 = { + "test4", // name + "test incremental values stored in the middle of second boundaries", + 1, // update_every + 1, // multiplier + 1, // divisor + RRDDIM_INCREMENTAL, // algorithm + 10, // feed entries + 9, // result entries + test4_feed, // feed + test4_results // results +}; + +// -------------------------------------------------------------------------------------------------------------------- +// test5 + +struct feed_values test5_feed[] = { + { 500000, 1000 }, + { 1000000, 2000 }, + { 1000000, 2000 }, + { 1000000, 2000 }, + { 1000000, 3000 }, + { 1000000, 2000 }, + { 1000000, 2000 }, + { 1000000, 2000 }, + { 1000000, 2000 }, + { 1000000, 2000 }, +}; + +calculated_number test5_results[] = { + 500, 500, 0, 500, 500, 0, 0, 0, 0 +}; + +struct test test5 = { + "test5", // name + "test incremental values ups and downs", + 1, // update_every + 1, // multiplier + 1, // divisor + RRDDIM_INCREMENTAL, // algorithm + 10, // feed entries + 9, // result entries + test5_feed, // feed + test5_results // results +}; + +// -------------------------------------------------------------------------------------------------------------------- +// test6 + +struct feed_values test6_feed[] = { + { 250000, 1000 }, + { 250000, 2000 }, + { 250000, 3000 }, + { 250000, 4000 }, + { 250000, 5000 }, + { 250000, 6000 }, + { 250000, 7000 }, + { 250000, 8000 }, + { 250000, 9000 }, + { 250000, 10000 }, + { 250000, 11000 }, + { 250000, 12000 }, + { 250000, 13000 }, + { 250000, 14000 }, + { 250000, 15000 }, + { 250000, 16000 }, +}; + +calculated_number test6_results[] = { + 3000, 4000, 4000, 4000 +}; + +struct test test6 = { + "test6", // name + "test incremental values updated within the same second", + 1, // update_every + 1, // multiplier + 1, // divisor + RRDDIM_INCREMENTAL, // algorithm + 16, // feed entries + 4, // result entries + test6_feed, // feed + test6_results // results +}; + +// -------------------------------------------------------------------------------------------------------------------- +// test7 + +struct feed_values test7_feed[] = { + { 500000, 1000 }, + { 2000000, 2000 }, + { 2000000, 3000 }, + { 2000000, 4000 }, + { 2000000, 5000 }, + { 2000000, 6000 }, + { 2000000, 7000 }, + { 2000000, 8000 }, + { 2000000, 9000 }, + { 2000000, 10000 }, +}; + +calculated_number test7_results[] = { + 250, 500, 500, 500, 500, 500, 500, 500, 500, 500, 500, 500, 500, 500, 500, 500, 500, 500, 500 +}; + +struct test test7 = { + "test7", // name + "test incremental values updated in long durations", + 1, // update_every + 1, // multiplier + 1, // divisor + RRDDIM_INCREMENTAL, // algorithm + 10, // feed entries + 18, // result entries + test7_feed, // feed + test7_results // results +}; + +// -------------------------------------------------------------------------------------------------------------------- +// test8 + +struct feed_values test8_feed[] = { + { 500000, 1000 }, + { 2000000, 2000 }, + { 2000000, 3000 }, + { 2000000, 4000 }, + { 2000000, 5000 }, + { 2000000, 6000 }, +}; + +calculated_number test8_results[] = { + 1250, 2000, 2250, 3000, 3250, 4000, 4250, 5000, 5250, 6000 +}; + +struct test test8 = { + "test8", // name + "test absolute values updated in long durations", + 1, // update_every + 1, // multiplier + 1, // divisor + RRDDIM_ABSOLUTE, // algorithm + 6, // feed entries + 10, // result entries + test8_feed, // feed + test8_results // results +}; + +// -------------------------------------------------------------------------------------------------------------------- +// test9 + +struct feed_values test9_feed[] = { + { 250000, 1000 }, + { 250000, 2000 }, + { 250000, 3000 }, + { 250000, 4000 }, + { 250000, 5000 }, + { 250000, 6000 }, + { 250000, 7000 }, + { 250000, 8000 }, + { 250000, 9000 }, + { 250000, 10000 }, + { 250000, 11000 }, + { 250000, 12000 }, + { 250000, 13000 }, + { 250000, 14000 }, + { 250000, 15000 }, + { 250000, 16000 }, +}; + +calculated_number test9_results[] = { + 4000, 8000, 12000, 16000 +}; + +struct test test9 = { + "test9", // name + "test absolute values updated within the same second", + 1, // update_every + 1, // multiplier + 1, // divisor + RRDDIM_ABSOLUTE, // algorithm + 16, // feed entries + 4, // result entries + test9_feed, // feed + test9_results // results +}; + +// -------------------------------------------------------------------------------------------------------------------- +// test10 + +struct feed_values test10_feed[] = { + { 500000, 1000 }, + { 600000, 1000 + 600 }, + { 200000, 1600 + 200 }, + { 1000000, 1800 + 1000 }, + { 200000, 2800 + 200 }, + { 2000000, 3000 + 2000 }, + { 600000, 5000 + 600 }, + { 400000, 5600 + 400 }, + { 900000, 6000 + 900 }, + { 1000000, 6900 + 1000 }, +}; + +calculated_number test10_results[] = { + 500, 1000, 1000, 1000, 1000, 1000, 1000 +}; + +struct test test10 = { + "test10", // name + "test incremental values updated in short and long durations", + 1, // update_every + 1, // multiplier + 1, // divisor + RRDDIM_INCREMENTAL, // algorithm + 10, // feed entries + 7, // result entries + test10_feed, // feed + test10_results // results +}; + +// -------------------------------------------------------------------------------------------------------------------- + +int run_test(struct test *test) +{ + fprintf(stderr, "\nRunning test '%s':\n%s\n", test->name, test->description); + + rrd_memory_mode = RRD_MEMORY_MODE_RAM; + rrd_update_every = test->update_every; + + char name[101]; + snprintf(name, 100, "unittest-%s", test->name); + + // create the chart + RRDSET *st = rrdset_create("netdata", name, name, "netdata", NULL, "Unit Testing", "a value", 1, 1, RRDSET_TYPE_LINE); + RRDDIM *rd = rrddim_add(st, "dimension", NULL, test->multiplier, test->divisor, test->algorithm); + st->debug = 1; + + // feed it with the test data + unsigned long c; + for(c = 0; c < test->feed_entries; c++) { + if(debug_flags) fprintf(stderr, "\n\n"); + + if(c) { + fprintf(stderr, " > %s: feeding position %lu, after %llu microseconds, with value " CALCULATED_NUMBER_FORMAT "\n", test->name, c+1, test->feed[c].microseconds, test->feed[c].value); + rrdset_next_usec(st, test->feed[c].microseconds); + } + else { + fprintf(stderr, " > %s: feeding position %lu with value " CALCULATED_NUMBER_FORMAT "\n", test->name, c+1, test->feed[c].value); + } + + rrddim_set(st, "dimension", test->feed[c].value); + rrdset_done(st); + + // align the first entry to second boundary + if(!c) { + fprintf(stderr, " > %s: fixing first collection time to be %llu microseconds to second boundary\n", test->name, test->feed[c].microseconds); + rd->last_collected_time.tv_usec = st->last_collected_time.tv_usec = st->last_updated.tv_usec = test->feed[c].microseconds; + } + } + + // check the result + int errors = 0; + + if(st->counter != test->result_entries) { + fprintf(stderr, " %s stored %lu entries, but we were expecting %lu, ### E R R O R ###\n", test->name, st->counter, test->result_entries); + errors++; + } + + unsigned long max = (st->counter < test->result_entries)?st->counter:test->result_entries; + for(c = 0 ; c < max ; c++) { + calculated_number v = unpack_storage_number(rd->values[c]), n = test->results[c]; + fprintf(stderr, " %s: checking position %lu, expecting value " CALCULATED_NUMBER_FORMAT ", found " CALCULATED_NUMBER_FORMAT ", %s\n", test->name, c+1, n, v, (v == n)?"OK":"### E R R O R ###"); + if(v != n) errors++; + } + + return errors; +} + +int run_all_mockup_tests(void) +{ + if(run_test(&test1)) + return 1; + + if(run_test(&test2)) + return 1; + + if(run_test(&test3)) + return 1; + + if(run_test(&test4)) + return 1; + + if(run_test(&test5)) + return 1; + + if(run_test(&test6)) + return 1; + + if(run_test(&test7)) + return 1; + + if(run_test(&test8)) + return 1; + + if(run_test(&test9)) + return 1; + + if(run_test(&test10)) + return 1; + + return 0; +} + +int unit_test(long delay, long shift) +{ + static int repeat = 0; + repeat++; + + char name[101]; + snprintf(name, 100, "unittest-%d-%ld-%ld", repeat, delay, shift); + + //debug_flags = 0xffffffff; + rrd_memory_mode = RRD_MEMORY_MODE_RAM; + rrd_update_every = 1; + + int do_abs = 1; + int do_inc = 1; + int do_abst = 0; + int do_absi = 0; + + RRDSET *st = rrdset_create("netdata", name, name, "netdata", NULL, "Unit Testing", "a value", 1, 1, RRDSET_TYPE_LINE); + st->debug = 1; + + RRDDIM *rdabs = NULL; + RRDDIM *rdinc = NULL; + RRDDIM *rdabst = NULL; + RRDDIM *rdabsi = NULL; + + if(do_abs) rdabs = rrddim_add(st, "absolute", "absolute", 1, 1, RRDDIM_ABSOLUTE); + if(do_inc) rdinc = rrddim_add(st, "incremental", "incremental", 1, 1, RRDDIM_INCREMENTAL); + if(do_abst) rdabst = rrddim_add(st, "percentage-of-absolute-row", "percentage-of-absolute-row", 1, 1, RRDDIM_PCENT_OVER_ROW_TOTAL); + if(do_absi) rdabsi = rrddim_add(st, "percentage-of-incremental-row", "percentage-of-incremental-row", 1, 1, RRDDIM_PCENT_OVER_DIFF_TOTAL); + + long increment = 1000; + collected_number i = 0; + + unsigned long c, dimensions = 0; + RRDDIM *rd; + for(rd = st->dimensions ; rd ; rd = rd->next) dimensions++; + + for(c = 0; c < 20 ;c++) { + i += increment; + + fprintf(stderr, "\n\nLOOP = %lu, DELAY = %ld, VALUE = " COLLECTED_NUMBER_FORMAT "\n", c, delay, i); + if(c) { + rrdset_next_usec(st, delay); + } + if(do_abs) rrddim_set(st, "absolute", i); + if(do_inc) rrddim_set(st, "incremental", i); + if(do_abst) rrddim_set(st, "percentage-of-absolute-row", i); + if(do_absi) rrddim_set(st, "percentage-of-incremental-row", i); + + if(!c) { + gettimeofday(&st->last_collected_time, NULL); + st->last_collected_time.tv_usec = shift; + } + + // prevent it from deleting the dimensions + for(rd = st->dimensions ; rd ; rd = rd->next) + rd->last_collected_time.tv_sec = st->last_collected_time.tv_sec; + + rrdset_done(st); + } + + unsigned long oincrement = increment; + increment = increment * st->update_every * 1000000 / delay; + fprintf(stderr, "\n\nORIGINAL INCREMENT: %lu, INCREMENT %lu, DELAY %lu, SHIFT %lu\n", oincrement * 10, increment * 10, delay, shift); + + int ret = 0; + storage_number sn; + calculated_number cn, v; + for(c = 0 ; c < st->counter ; c++) { + fprintf(stderr, "\nPOSITION: c = %lu, EXPECTED VALUE %lu\n", c, (oincrement + c * increment + increment * (1000000 - shift) / 1000000 )* 10); + + for(rd = st->dimensions ; rd ; rd = rd->next) { + sn = rd->values[c]; + cn = unpack_storage_number(sn); + fprintf(stderr, "\t %s " CALCULATED_NUMBER_FORMAT " (PACKED AS " STORAGE_NUMBER_FORMAT ") -> ", rd->id, cn, sn); + + if(rd == rdabs) v = + ( oincrement + // + (increment * (1000000 - shift) / 1000000) + + (c + 1) * increment + ); + + else if(rd == rdinc) v = (c?(increment):(increment * (1000000 - shift) / 1000000)); + else if(rd == rdabst) v = oincrement / dimensions / 10; + else if(rd == rdabsi) v = oincrement / dimensions / 10; + else v = 0; + + if(v == cn) fprintf(stderr, "passed.\n"); + else { + fprintf(stderr, "ERROR! (expected " CALCULATED_NUMBER_FORMAT ")\n", v); + ret = 1; + } + } + } + + if(ret) + fprintf(stderr, "\n\nUNIT TEST(%ld, %ld) FAILED\n\n", delay, shift); + + return ret; +} diff --git a/src/unit_test.h b/src/unit_test.h new file mode 100755 index 000000000..916ad71f2 --- /dev/null +++ b/src/unit_test.h @@ -0,0 +1,8 @@ +#ifndef NETDATA_UNIT_TEST_H +#define NETDATA_UNIT_TEST_H 1 + +extern int unit_test_storage(void); +extern int unit_test(long delay, long shift); +extern int run_all_mockup_tests(void); + +#endif /* NETDATA_UNIT_TEST_H */ diff --git a/src/url.c b/src/url.c new file mode 100755 index 000000000..c4933b205 --- /dev/null +++ b/src/url.c @@ -0,0 +1,81 @@ +#ifdef HAVE_CONFIG_H +#include <config.h> +#endif +#include <stdlib.h> +#include <string.h> +#include <ctype.h> + +#include "common.h" +#include "log.h" +#include "url.h" + +// ---------------------------------------------------------------------------- +// URL encode / decode +// code from: http://www.geekhideout.com/urlcode.shtml + +/* Converts a hex character to its integer value */ +char from_hex(char ch) { + return (char)(isdigit(ch) ? ch - '0' : tolower(ch) - 'a' + 10); +} + +/* Converts an integer value to its hex character*/ +char to_hex(char code) { + static char hex[] = "0123456789abcdef"; + return hex[code & 15]; +} + +/* Returns a url-encoded version of str */ +/* IMPORTANT: be sure to free() the returned string after use */ +char *url_encode(char *str) { + char *pstr = str, + *buf = malloc(strlen(str) * 3 + 1), + *pbuf = buf; + + while (*pstr) { + if (isalnum(*pstr) || *pstr == '-' || *pstr == '_' || *pstr == '.' || *pstr == '~') + *pbuf++ = *pstr; + + else if (*pstr == ' ') + *pbuf++ = '+'; + + else + *pbuf++ = '%', *pbuf++ = to_hex(*pstr >> 4), *pbuf++ = to_hex(*pstr & 15); + + pstr++; + } + + *pbuf = '\0'; + + return buf; +} + +/* Returns a url-decoded version of str */ +/* IMPORTANT: be sure to free() the returned string after use */ +char *url_decode(char *str) { + char *pstr = str, + *buf = malloc(strlen(str) + 1), + *pbuf = buf; + + if(!buf) fatal("Cannot allocate memory."); + + while (*pstr) { + if (*pstr == '%') { + if (pstr[1] && pstr[2]) { + *pbuf++ = from_hex(pstr[1]) << 4 | from_hex(pstr[2]); + pstr += 2; + } + } + else if (*pstr == '+') + *pbuf++ = ' '; + + else + *pbuf++ = *pstr; + + pstr++; + } + + *pbuf = '\0'; + + return buf; +} + diff --git a/src/url.h b/src/url.h new file mode 100755 index 000000000..f79a20ea0 --- /dev/null +++ b/src/url.h @@ -0,0 +1,22 @@ +#ifndef NETDATA_URL_H +#define NETDATA_URL_H 1 + +// ---------------------------------------------------------------------------- +// URL encode / decode +// code from: http://www.geekhideout.com/urlcode.shtml + +/* Converts a hex character to its integer value */ +extern char from_hex(char ch); + +/* Converts an integer value to its hex character*/ +extern char to_hex(char code); + +/* Returns a url-encoded version of str */ +/* IMPORTANT: be sure to free() the returned string after use */ +extern char *url_encode(char *str); + +/* Returns a url-decoded version of str */ +/* IMPORTANT: be sure to free() the returned string after use */ +extern char *url_decode(char *str); + +#endif /* NETDATA_URL_H */ diff --git a/src/web_buffer.c b/src/web_buffer.c new file mode 100755 index 000000000..482eb3900 --- /dev/null +++ b/src/web_buffer.c @@ -0,0 +1,330 @@ +#ifdef HAVE_CONFIG_H +#include <config.h> +#endif +#include <stdlib.h> +#include <string.h> + +#ifdef STORAGE_WITH_MATH +#include <math.h> +#endif + +#include "common.h" +#include "web_buffer.h" +#include "log.h" + +#define BUFFER_OVERFLOW_EOF "EOF" + +static inline void buffer_overflow_init(BUFFER *b) +{ + b->buffer[b->size] = '\0'; + strcpy(&b->buffer[b->size + 1], BUFFER_OVERFLOW_EOF); +} + +#ifdef NETDATA_INTERNAL_CHECKS +#define buffer_overflow_check(b) _buffer_overflow_check(b, __FILE__, __FUNCTION__, __LINE__) +#else +#define buffer_overflow_check(b) +#endif + +static inline void _buffer_overflow_check(BUFFER *b, const char *file, const char *function, const unsigned long line) +{ + if(b->len > b->size) { + error("BUFFER: length %ld is above size %ld, at line %lu, at function %s() of file '%s'.", b->len, b->size, line, function, file); + b->len = b->size; + } + + if(b->buffer[b->size] != '\0' || strcmp(&b->buffer[b->size + 1], BUFFER_OVERFLOW_EOF)) { + error("BUFFER: detected overflow at line %lu, at function %s() of file '%s'.", line, function, file); + buffer_overflow_init(b); + } +} + + +void buffer_reset(BUFFER *wb) +{ + buffer_flush(wb); + + wb->contenttype = CT_TEXT_PLAIN; + wb->options = 0; + wb->date = 0; + + buffer_overflow_check(wb); +} + +const char *buffer_tostring(BUFFER *wb) +{ + buffer_need_bytes(wb, (size_t)1); + wb->buffer[wb->len] = '\0'; + + buffer_overflow_check(wb); + + return(wb->buffer); +} + +void buffer_char_replace(BUFFER *wb, char from, char to) +{ + char *s = wb->buffer, *end = &wb->buffer[wb->len]; + + while(s != end) { + if(*s == from) *s = to; + s++; + } + + buffer_overflow_check(wb); +} + + +void buffer_strcat(BUFFER *wb, const char *txt) +{ + if(unlikely(!txt || !*txt)) return; + + buffer_need_bytes(wb, (size_t)(1)); + + char *s = &wb->buffer[wb->len], *end = &wb->buffer[wb->size]; + long len = wb->len; + + while(*txt && s != end) { + *s++ = *txt++; + len++; + } + + wb->len = len; + buffer_overflow_check(wb); + + if(*txt) { + debug(D_WEB_BUFFER, "strcat(): increasing web_buffer at position %ld, size = %ld\n", wb->len, wb->size); + len = strlen(txt); + buffer_increase(wb, len); + buffer_strcat(wb, txt); + } + else { + // terminate the string + // without increasing the length + buffer_need_bytes(wb, (size_t)1); + wb->buffer[wb->len] = '\0'; + } +} + + +void buffer_snprintf(BUFFER *wb, size_t len, const char *fmt, ...) +{ + if(unlikely(!fmt || !*fmt)) return; + + buffer_need_bytes(wb, len+1); + + va_list args; + va_start(args, fmt); + wb->len += vsnprintf(&wb->buffer[wb->len], len+1, fmt, args); + va_end(args); + + buffer_overflow_check(wb); + + // the buffer is \0 terminated by vsnprintf +} + +void buffer_vsprintf(BUFFER *wb, const char *fmt, va_list args) +{ + if(unlikely(!fmt || !*fmt)) return; + + buffer_need_bytes(wb, 1); + + size_t len = wb->size - wb->len; + + wb->len += vsnprintf(&wb->buffer[wb->len], len, fmt, args); + + buffer_overflow_check(wb); + + // the buffer is \0 terminated by vsnprintf +} + +void buffer_sprintf(BUFFER *wb, const char *fmt, ...) +{ + if(unlikely(!fmt || !*fmt)) return; + + buffer_need_bytes(wb, 1); + + size_t len = wb->size - wb->len, wrote; + + va_list args; + va_start(args, fmt); + wrote = (size_t) vsnprintf(&wb->buffer[wb->len], len, fmt, args); + va_end(args); + + if(unlikely(wrote >= len)) { + // there is bug in vsnprintf() and it returns + // a number higher to len, but it does not + // overflow the buffer. + // our buffer overflow detector will log it + // if it does. + buffer_overflow_check(wb); + + debug(D_WEB_BUFFER, "web_buffer_sprintf(): increasing web_buffer at position %ld, size = %ld\n", wb->len, wb->size); + buffer_need_bytes(wb, len + WEB_DATA_LENGTH_INCREASE_STEP); + + va_start(args, fmt); + buffer_vsprintf(wb, fmt, args); + va_end(args); + } + else + wb->len += wrote; + + // the buffer is \0 terminated by vsnprintf +} + + +void buffer_rrd_value(BUFFER *wb, calculated_number value) +{ + buffer_need_bytes(wb, 50); + wb->len += print_calculated_number(&wb->buffer[wb->len], value); + + // terminate it + buffer_need_bytes(wb, 1); + wb->buffer[wb->len] = '\0'; + + buffer_overflow_check(wb); +} + +// generate a javascript date, the fastest possible way... +void buffer_jsdate(BUFFER *wb, int year, int month, int day, int hours, int minutes, int seconds) +{ + // 10 20 30 = 35 + // 01234567890123456789012345678901234 + // Date(2014,04,01,03,28,20) + + buffer_need_bytes(wb, 30); + + char *b = &wb->buffer[wb->len]; + + int i = 0; + b[i++]='D'; + b[i++]='a'; + b[i++]='t'; + b[i++]='e'; + b[i++]='('; + b[i++]= (char) (48 + year / 1000); year -= (year / 1000) * 1000; + b[i++]= (char) (48 + year / 100); year -= (year / 100) * 100; + b[i++]= (char) (48 + year / 10); + b[i++]= (char) (48 + year % 10); + b[i++]=','; + b[i]= (char) (48 + month / 10); if(b[i] != '0') i++; + b[i++]= (char) (48 + month % 10); + b[i++]=','; + b[i]= (char) (48 + day / 10); if(b[i] != '0') i++; + b[i++]= (char) (48 + day % 10); + b[i++]=','; + b[i]= (char) (48 + hours / 10); if(b[i] != '0') i++; + b[i++]= (char) (48 + hours % 10); + b[i++]=','; + b[i]= (char) (48 + minutes / 10); if(b[i] != '0') i++; + b[i++]= (char) (48 + minutes % 10); + b[i++]=','; + b[i]= (char) (48 + seconds / 10); if(b[i] != '0') i++; + b[i++]= (char) (48 + seconds % 10); + b[i++]=')'; + b[i]='\0'; + + wb->len += i; + + // terminate it + wb->buffer[wb->len] = '\0'; + buffer_overflow_check(wb); +} + +// generate a date, the fastest possible way... +void buffer_date(BUFFER *wb, int year, int month, int day, int hours, int minutes, int seconds) +{ + // 10 20 30 = 35 + // 01234567890123456789012345678901234 + // 2014-04-01 03:28:20 + + buffer_need_bytes(wb, 36); + + char *b = &wb->buffer[wb->len]; + + int i = 0; + b[i++]= (char) (48 + year / 1000); year -= (year / 1000) * 1000; + b[i++]= (char) (48 + year / 100); year -= (year / 100) * 100; + b[i++]= (char) (48 + year / 10); + b[i++]= (char) (48 + year % 10); + b[i++]='-'; + b[i++]= (char) (48 + month / 10); + b[i++]= (char) (48 + month % 10); + b[i++]='-'; + b[i++]= (char) (48 + day / 10); + b[i++]= (char) (48 + day % 10); + b[i++]=' '; + b[i++]= (char) (48 + hours / 10); + b[i++]= (char) (48 + hours % 10); + b[i++]=':'; + b[i++]= (char) (48 + minutes / 10); + b[i++]= (char) (48 + minutes % 10); + b[i++]=':'; + b[i++]= (char) (48 + seconds / 10); + b[i++]= (char) (48 + seconds % 10); + b[i]='\0'; + + wb->len += i; + + // terminate it + wb->buffer[wb->len] = '\0'; + buffer_overflow_check(wb); +} + +BUFFER *buffer_create(long size) +{ + BUFFER *b; + + debug(D_WEB_BUFFER, "Creating new web buffer of size %d.", size); + + b = calloc(1, sizeof(BUFFER)); + if(!b) { + error("Cannot allocate a web_buffer."); + return NULL; + } + + b->buffer = malloc(size + sizeof(BUFFER_OVERFLOW_EOF) + 2); + if(!b->buffer) { + error("Cannot allocate a buffer of size %u.", size + sizeof(BUFFER_OVERFLOW_EOF) + 2); + free(b); + return NULL; + } + b->buffer[0] = '\0'; + b->size = size; + b->contenttype = CT_TEXT_PLAIN; + buffer_overflow_init(b); + buffer_overflow_check(b); + + return(b); +} + +void buffer_free(BUFFER *b) +{ + buffer_overflow_check(b); + + debug(D_WEB_BUFFER, "Freeing web buffer of size %d.", b->size); + + if(b->buffer) free(b->buffer); + free(b); +} + +void buffer_increase(BUFFER *b, size_t free_size_required) +{ + buffer_overflow_check(b); + + size_t left = b->size - b->len; + + if(left >= free_size_required) return; + + size_t increase = free_size_required - left; + if(increase < WEB_DATA_LENGTH_INCREASE_STEP) increase = WEB_DATA_LENGTH_INCREASE_STEP; + + debug(D_WEB_BUFFER, "Increasing data buffer from size %d to %d.", b->size, b->size + increase); + + b->buffer = realloc(b->buffer, b->size + increase + sizeof(BUFFER_OVERFLOW_EOF) + 2); + if(!b->buffer) fatal("Failed to increase data buffer from size %d to %d.", b->size + sizeof(BUFFER_OVERFLOW_EOF) + 2, b->size + increase + sizeof(BUFFER_OVERFLOW_EOF) + 2); + + b->size += increase; + + buffer_overflow_init(b); + buffer_overflow_check(b); +} diff --git a/src/web_buffer.h b/src/web_buffer.h new file mode 100755 index 000000000..58dd9c094 --- /dev/null +++ b/src/web_buffer.h @@ -0,0 +1,70 @@ +#include <stdarg.h> +#include <time.h> +#include "storage_number.h" + +#ifndef NETDATA_WEB_BUFFER_H +#define NETDATA_WEB_BUFFER_H 1 + +#define WEB_DATA_LENGTH_INCREASE_STEP 16384 + +typedef struct web_buffer { + size_t size; // allocation size of buffer + size_t len; // current data length in buffer + char *buffer; // the buffer + uint8_t contenttype; + uint8_t options; + time_t date; // the date this content has been generated +} BUFFER; + +// options +#define WB_CONTENT_CACHEABLE 1 +#define WB_CONTENT_NO_CACHEABLE 2 + +// content-types +#define CT_APPLICATION_JSON 1 +#define CT_TEXT_PLAIN 2 +#define CT_TEXT_HTML 3 +#define CT_APPLICATION_X_JAVASCRIPT 4 +#define CT_TEXT_CSS 5 +#define CT_TEXT_XML 6 +#define CT_APPLICATION_XML 7 +#define CT_TEXT_XSL 8 +#define CT_APPLICATION_OCTET_STREAM 9 +#define CT_APPLICATION_X_FONT_TRUETYPE 10 +#define CT_APPLICATION_X_FONT_OPENTYPE 11 +#define CT_APPLICATION_FONT_WOFF 12 +#define CT_APPLICATION_FONT_WOFF2 13 +#define CT_APPLICATION_VND_MS_FONTOBJ 14 +#define CT_IMAGE_SVG_XML 15 +#define CT_IMAGE_PNG 16 +#define CT_IMAGE_JPG 17 +#define CT_IMAGE_GIF 18 +#define CT_IMAGE_XICON 19 +#define CT_IMAGE_ICNS 20 +#define CT_IMAGE_BMP 21 + +#define buffer_strlen(wb) ((wb)->len) +extern const char *buffer_tostring(BUFFER *wb); + +#define buffer_need_bytes(buffer, needed_free_size) do { if(unlikely((buffer)->size - (buffer)->len < (size_t)(needed_free_size))) buffer_increase((buffer), (size_t)(needed_free_size)); } while(0) + +#define buffer_flush(wb) wb->buffer[wb->len = 0] = '\0' +extern void buffer_reset(BUFFER *wb); + +extern void buffer_strcat(BUFFER *wb, const char *txt); +extern void buffer_rrd_value(BUFFER *wb, calculated_number value); + +extern void buffer_date(BUFFER *wb, int year, int month, int day, int hours, int minutes, int seconds); +extern void buffer_jsdate(BUFFER *wb, int year, int month, int day, int hours, int minutes, int seconds); + +extern BUFFER *buffer_create(long size); +extern void buffer_free(BUFFER *b); +extern void buffer_increase(BUFFER *b, size_t free_size_required); + +extern void buffer_snprintf(BUFFER *wb, size_t len, const char *fmt, ...); +extern void buffer_vsprintf(BUFFER *wb, const char *fmt, va_list args); +extern void buffer_sprintf(BUFFER *wb, const char *fmt, ...); + +extern void buffer_char_replace(BUFFER *wb, char from, char to); + +#endif /* NETDATA_WEB_BUFFER_H */ diff --git a/src/web_client.c b/src/web_client.c new file mode 100755 index 000000000..127333554 --- /dev/null +++ b/src/web_client.c @@ -0,0 +1,1814 @@ +#ifdef HAVE_CONFIG_H +#include <config.h> +#endif +#include <unistd.h> +#include <stdlib.h> +#include <sys/types.h> +#include <sys/socket.h> +#include <netinet/in.h> +#include <arpa/inet.h> +#include <errno.h> +#include <pthread.h> +#include <sys/stat.h> +#include <fcntl.h> +#include <netinet/tcp.h> +#include <malloc.h> +#include <pwd.h> +#include <ctype.h> + +#include "common.h" +#include "log.h" +#include "appconfig.h" +#include "url.h" +#include "web_buffer.h" +#include "web_server.h" +#include "global_statistics.h" +#include "rrd.h" +#include "rrd2json.h" + +#include "web_client.h" + +#define INITIAL_WEB_DATA_LENGTH 16384 +#define WEB_REQUEST_LENGTH 16384 + +int web_client_timeout = DEFAULT_DISCONNECT_IDLE_WEB_CLIENTS_AFTER_SECONDS; +int web_enable_gzip = 1; + +extern int netdata_exit; + +struct web_client *web_clients = NULL; +unsigned long long web_clients_count = 0; + +struct web_client *web_client_create(int listener) +{ + struct web_client *w; + + w = calloc(1, sizeof(struct web_client)); + if(!w) { + error("Cannot allocate new web_client memory."); + return NULL; + } + + w->id = ++web_clients_count; + w->mode = WEB_CLIENT_MODE_NORMAL; + + { + struct sockaddr *sadr; + socklen_t addrlen; + + sadr = (struct sockaddr*) &w->clientaddr; + addrlen = sizeof(w->clientaddr); + + w->ifd = accept(listener, sadr, &addrlen); + if (w->ifd == -1) { + error("%llu: Cannot accept new incoming connection.", w->id); + free(w); + return NULL; + } + w->ofd = w->ifd; + + if(getnameinfo(sadr, addrlen, w->client_ip, NI_MAXHOST, w->client_port, NI_MAXSERV, NI_NUMERICHOST | NI_NUMERICSERV) != 0) { + error("Cannot getnameinfo() on received client connection."); + strncpy(w->client_ip, "UNKNOWN", NI_MAXHOST); + strncpy(w->client_port, "UNKNOWN", NI_MAXSERV); + } + w->client_ip[NI_MAXHOST] = '\0'; + w->client_port[NI_MAXSERV] = '\0'; + + switch(sadr->sa_family) { + + case AF_INET: + debug(D_WEB_CLIENT_ACCESS, "%llu: New IPv4 web client from %s port %s on socket %d.", w->id, w->client_ip, w->client_port, w->ifd); + break; + + case AF_INET6: + if(strncmp(w->client_ip, "::ffff:", 7) == 0) { + strcpy(w->client_ip, &w->client_ip[7]); + debug(D_WEB_CLIENT_ACCESS, "%llu: New IPv4 web client from %s port %s on socket %d.", w->id, w->client_ip, w->client_port, w->ifd); + } + debug(D_WEB_CLIENT_ACCESS, "%llu: New IPv6 web client from %s port %s on socket %d.", w->id, w->client_ip, w->client_port, w->ifd); + break; + + default: + debug(D_WEB_CLIENT_ACCESS, "%llu: New UNKNOWN web client from %s port %s on socket %d.", w->id, w->client_ip, w->client_port, w->ifd); + break; + } + + int flag = 1; + if(setsockopt(w->ifd, SOL_SOCKET, SO_KEEPALIVE, (char *) &flag, sizeof(int)) != 0) error("%llu: Cannot set SO_KEEPALIVE on socket.", w->id); + } + + w->response.data = buffer_create(INITIAL_WEB_DATA_LENGTH); + if(unlikely(!w->response.data)) { + // no need for error log - web_buffer_create already logged the error + close(w->ifd); + free(w); + return NULL; + } + + w->response.header = buffer_create(HTTP_RESPONSE_HEADER_SIZE); + if(unlikely(!w->response.header)) { + // no need for error log - web_buffer_create already logged the error + buffer_free(w->response.data); + close(w->ifd); + free(w); + return NULL; + } + + w->response.header_output = buffer_create(HTTP_RESPONSE_HEADER_SIZE); + if(unlikely(!w->response.header_output)) { + // no need for error log - web_buffer_create already logged the error + buffer_free(w->response.header); + buffer_free(w->response.data); + close(w->ifd); + free(w); + return NULL; + } + + w->wait_receive = 1; + + if(web_clients) web_clients->prev = w; + w->next = web_clients; + web_clients = w; + + global_statistics.connected_clients++; + + return(w); +} + +void web_client_reset(struct web_client *w) +{ + struct timeval tv; + gettimeofday(&tv, NULL); + + long sent = (w->mode == WEB_CLIENT_MODE_FILECOPY)?w->response.rlen:w->response.data->len; + +#ifdef NETDATA_WITH_ZLIB + if(likely(w->response.zoutput)) sent = (long)w->response.zstream.total_out; +#endif + + long size = (w->mode == WEB_CLIENT_MODE_FILECOPY)?w->response.rlen:w->response.data->len; + + if(likely(w->last_url[0])) + log_access("%llu: (sent/all = %ld/%ld bytes %0.0f%%, prep/sent/total = %0.2f/%0.2f/%0.2f ms) %s: %d '%s'", + w->id, + sent, size, -((size>0)?((float)(size-sent)/(float)size * 100.0):0.0), + (float)usecdiff(&w->tv_ready, &w->tv_in) / 1000.0, + (float)usecdiff(&tv, &w->tv_ready) / 1000.0, + (float)usecdiff(&tv, &w->tv_in) / 1000.0, + (w->mode == WEB_CLIENT_MODE_FILECOPY)?"filecopy":((w->mode == WEB_CLIENT_MODE_OPTIONS)?"options":"data"), + w->response.code, + w->last_url + ); + + debug(D_WEB_CLIENT, "%llu: Reseting client.", w->id); + + if(unlikely(w->mode == WEB_CLIENT_MODE_FILECOPY)) { + debug(D_WEB_CLIENT, "%llu: Closing filecopy input file.", w->id); + close(w->ifd); + w->ifd = w->ofd; + } + + w->last_url[0] = '\0'; + + w->mode = WEB_CLIENT_MODE_NORMAL; + + buffer_reset(w->response.header_output); + buffer_reset(w->response.header); + buffer_reset(w->response.data); + w->response.rlen = 0; + w->response.sent = 0; + w->response.code = 0; + + w->wait_receive = 1; + w->wait_send = 0; + + w->response.zoutput = 0; + + // if we had enabled compression, release it +#ifdef NETDATA_WITH_ZLIB + if(w->response.zinitialized) { + debug(D_DEFLATE, "%llu: Reseting compression.", w->id); + deflateEnd(&w->response.zstream); + w->response.zsent = 0; + w->response.zhave = 0; + w->response.zstream.avail_in = 0; + w->response.zstream.avail_out = 0; + w->response.zstream.total_in = 0; + w->response.zstream.total_out = 0; + w->response.zinitialized = 0; + } +#endif // NETDATA_WITH_ZLIB +} + +struct web_client *web_client_free(struct web_client *w) +{ + struct web_client *n = w->next; + + debug(D_WEB_CLIENT_ACCESS, "%llu: Closing web client from %s port %s.", w->id, w->client_ip, w->client_port); + + if(w->prev) w->prev->next = w->next; + if(w->next) w->next->prev = w->prev; + + if(w == web_clients) web_clients = w->next; + + if(w->response.header_output) buffer_free(w->response.header_output); + if(w->response.header) buffer_free(w->response.header); + if(w->response.data) buffer_free(w->response.data); + close(w->ifd); + if(w->ofd != w->ifd) close(w->ofd); + free(w); + + global_statistics.connected_clients--; + + return(n); +} + +uid_t web_files_uid(void) +{ + static char *web_owner = NULL; + static uid_t owner_uid = 0; + + if(unlikely(!web_owner)) { + web_owner = config_get("global", "web files owner", NETDATA_USER); + if(!web_owner || !*web_owner) + owner_uid = geteuid(); + else { + struct passwd *pw = getpwnam(web_owner); + if(!pw) { + error("User %s is not present. Ignoring option.", web_owner); + owner_uid = geteuid(); + } + else { + debug(D_WEB_CLIENT, "Web files owner set to %s.\n", web_owner); + owner_uid = pw->pw_uid; + } + } + } + + return(owner_uid); +} + +int mysendfile(struct web_client *w, char *filename) +{ + static char *web_dir = NULL; + + // initialize our static data + if(unlikely(!web_dir)) web_dir = config_get("global", "web files directory", WEB_DIR); + + debug(D_WEB_CLIENT, "%llu: Looking for file '%s/%s'", w->id, web_dir, filename); + + // skip leading slashes + while (*filename == '/') filename++; + + // if the filename contain known paths, skip them + if(strncmp(filename, WEB_PATH_FILE "/", strlen(WEB_PATH_FILE) + 1) == 0) filename = &filename[strlen(WEB_PATH_FILE) + 1]; + + char *s; + for(s = filename; *s ;s++) { + if( !isalnum(*s) && *s != '/' && *s != '.' && *s != '-' && *s != '_') { + debug(D_WEB_CLIENT_ACCESS, "%llu: File '%s' is not acceptable.", w->id, filename); + buffer_sprintf(w->response.data, "File '%s' cannot be served. Filename contains invalid character '%c'", *s); + return 400; + } + } + + // if the filename contains a .. refuse to serve it + if(strstr(filename, "..") != 0) { + debug(D_WEB_CLIENT_ACCESS, "%llu: File '%s' is not acceptable.", w->id, filename); + buffer_sprintf(w->response.data, "File '%s' cannot be served. Relative filenames with '..' in them are not supported.", filename); + return 400; + } + + // access the file + char webfilename[FILENAME_MAX + 1]; + snprintf(webfilename, FILENAME_MAX, "%s/%s", web_dir, filename); + + // check if the file exists + struct stat stat; + if(lstat(webfilename, &stat) != 0) { + debug(D_WEB_CLIENT_ACCESS, "%llu: File '%s' is not found.", w->id, webfilename); + buffer_sprintf(w->response.data, "File '%s' does not exist, or is not accessible.", filename); + return 404; + } + + // check if the file is owned by us + if(stat.st_uid != web_files_uid()) { + error("%llu: File '%s' is owned by user %d (I run as user %d). Access Denied.", w->id, webfilename, stat.st_uid, getuid()); + buffer_sprintf(w->response.data, "Access to file '%s' is not permitted.", filename); + return 403; + } + + if((stat.st_mode & S_IFMT) == S_IFDIR) { + snprintf(webfilename, FILENAME_MAX+1, "%s/index.html", filename); + return mysendfile(w, webfilename); + } + + if((stat.st_mode & S_IFMT) != S_IFREG) { + error("%llu: File '%s' is not a regular file. Access Denied.", w->id, webfilename); + buffer_sprintf(w->response.data, "Access to file '%s' is not permitted.", filename); + return 403; + } + + // open the file + w->ifd = open(webfilename, O_NONBLOCK, O_RDONLY); + if(w->ifd == -1) { + w->ifd = w->ofd; + + if(errno == EBUSY || errno == EAGAIN) { + error("%llu: File '%s' is busy, sending 307 Moved Temporarily to force retry.", w->id, webfilename); + buffer_sprintf(w->response.header, "Location: /" WEB_PATH_FILE "/%s\r\n", filename); + buffer_sprintf(w->response.data, "The file '%s' is currently busy. Please try again later.", filename); + return 307; + } + else { + error("%llu: Cannot open file '%s'.", w->id, webfilename); + buffer_sprintf(w->response.data, "Cannot open file '%s'.", filename); + return 404; + } + } + + // pick a Content-Type for the file + if(strstr(filename, ".html") != NULL) w->response.data->contenttype = CT_TEXT_HTML; + else if(strstr(filename, ".js") != NULL) w->response.data->contenttype = CT_APPLICATION_X_JAVASCRIPT; + else if(strstr(filename, ".css") != NULL) w->response.data->contenttype = CT_TEXT_CSS; + else if(strstr(filename, ".xml") != NULL) w->response.data->contenttype = CT_TEXT_XML; + else if(strstr(filename, ".xsl") != NULL) w->response.data->contenttype = CT_TEXT_XSL; + else if(strstr(filename, ".txt") != NULL) w->response.data->contenttype = CT_TEXT_PLAIN; + else if(strstr(filename, ".svg") != NULL) w->response.data->contenttype = CT_IMAGE_SVG_XML; + else if(strstr(filename, ".ttf") != NULL) w->response.data->contenttype = CT_APPLICATION_X_FONT_TRUETYPE; + else if(strstr(filename, ".otf") != NULL) w->response.data->contenttype = CT_APPLICATION_X_FONT_OPENTYPE; + else if(strstr(filename, ".woff2")!= NULL) w->response.data->contenttype = CT_APPLICATION_FONT_WOFF2; + else if(strstr(filename, ".woff") != NULL) w->response.data->contenttype = CT_APPLICATION_FONT_WOFF; + else if(strstr(filename, ".eot") != NULL) w->response.data->contenttype = CT_APPLICATION_VND_MS_FONTOBJ; + else if(strstr(filename, ".png") != NULL) w->response.data->contenttype = CT_IMAGE_PNG; + else if(strstr(filename, ".jpg") != NULL) w->response.data->contenttype = CT_IMAGE_JPG; + else if(strstr(filename, ".jpeg") != NULL) w->response.data->contenttype = CT_IMAGE_JPG; + else if(strstr(filename, ".gif") != NULL) w->response.data->contenttype = CT_IMAGE_GIF; + else if(strstr(filename, ".bmp") != NULL) w->response.data->contenttype = CT_IMAGE_BMP; + else if(strstr(filename, ".ico") != NULL) w->response.data->contenttype = CT_IMAGE_XICON; + else if(strstr(filename, ".icns") != NULL) w->response.data->contenttype = CT_IMAGE_ICNS; + else w->response.data->contenttype = CT_APPLICATION_OCTET_STREAM; + + debug(D_WEB_CLIENT_ACCESS, "%llu: Sending file '%s' (%ld bytes, ifd %d, ofd %d).", w->id, webfilename, stat.st_size, w->ifd, w->ofd); + + w->mode = WEB_CLIENT_MODE_FILECOPY; + w->wait_receive = 1; + w->wait_send = 0; + buffer_flush(w->response.data); + w->response.rlen = stat.st_size; + w->response.data->date = stat.st_mtim.tv_sec; + + return 200; +} + + +#ifdef NETDATA_WITH_ZLIB +void web_client_enable_deflate(struct web_client *w) { + if(w->response.zinitialized == 1) { + error("%llu: Compression has already be initialized for this client.", w->id); + return; + } + + if(w->response.sent) { + error("%llu: Cannot enable compression in the middle of a conversation.", w->id); + return; + } + + w->response.zstream.zalloc = Z_NULL; + w->response.zstream.zfree = Z_NULL; + w->response.zstream.opaque = Z_NULL; + + w->response.zstream.next_in = (Bytef *)w->response.data->buffer; + w->response.zstream.avail_in = 0; + w->response.zstream.total_in = 0; + + w->response.zstream.next_out = w->response.zbuffer; + w->response.zstream.avail_out = 0; + w->response.zstream.total_out = 0; + + w->response.zstream.zalloc = Z_NULL; + w->response.zstream.zfree = Z_NULL; + w->response.zstream.opaque = Z_NULL; + +// if(deflateInit(&w->response.zstream, Z_DEFAULT_COMPRESSION) != Z_OK) { +// error("%llu: Failed to initialize zlib. Proceeding without compression.", w->id); +// return; +// } + + // Select GZIP compression: windowbits = 15 + 16 = 31 + if(deflateInit2(&w->response.zstream, Z_DEFAULT_COMPRESSION, Z_DEFLATED, 31, 8, Z_DEFAULT_STRATEGY) != Z_OK) { + error("%llu: Failed to initialize zlib. Proceeding without compression.", w->id); + return; + } + + w->response.zsent = 0; + w->response.zoutput = 1; + w->response.zinitialized = 1; + + debug(D_DEFLATE, "%llu: Initialized compression.", w->id); +} +#endif // NETDATA_WITH_ZLIB + +uint32_t web_client_api_request_v1_data_options(char *o) +{ + uint32_t ret = 0x00000000; + char *tok; + + while(o && *o && (tok = mystrsep(&o, ", |"))) { + if(!*tok) continue; + + if(!strcmp(tok, "nonzero")) + ret |= RRDR_OPTION_NONZERO; + else if(!strcmp(tok, "flip") || !strcmp(tok, "reversed") || !strcmp(tok, "reverse")) + ret |= RRDR_OPTION_REVERSED; + else if(!strcmp(tok, "jsonwrap")) + ret |= RRDR_OPTION_JSON_WRAP; + else if(!strcmp(tok, "min2max")) + ret |= RRDR_OPTION_MIN2MAX; + else if(!strcmp(tok, "ms") || !strcmp(tok, "milliseconds")) + ret |= RRDR_OPTION_MILLISECONDS; + else if(!strcmp(tok, "abs") || !strcmp(tok, "absolute") || !strcmp(tok, "absolute_sum") || !strcmp(tok, "absolute-sum")) + ret |= RRDR_OPTION_ABSOLUTE; + else if(!strcmp(tok, "seconds")) + ret |= RRDR_OPTION_SECONDS; + else if(!strcmp(tok, "null2zero")) + ret |= RRDR_OPTION_NULL2ZERO; + else if(!strcmp(tok, "objectrows")) + ret |= RRDR_OPTION_OBJECTSROWS; + else if(!strcmp(tok, "google_json")) + ret |= RRDR_OPTION_GOOGLE_JSON; + else if(!strcmp(tok, "percentage")) + ret |= RRDR_OPTION_PERCENTAGE; + } + + return ret; +} + +uint32_t web_client_api_request_v1_data_format(char *name) +{ + if(!strcmp(name, DATASOURCE_FORMAT_DATATABLE_JSON)) // datatable + return DATASOURCE_DATATABLE_JSON; + + else if(!strcmp(name, DATASOURCE_FORMAT_DATATABLE_JSONP)) // datasource + return DATASOURCE_DATATABLE_JSONP; + + else if(!strcmp(name, DATASOURCE_FORMAT_JSON)) // json + return DATASOURCE_JSON; + + else if(!strcmp(name, DATASOURCE_FORMAT_JSONP)) // jsonp + return DATASOURCE_JSONP; + + else if(!strcmp(name, DATASOURCE_FORMAT_SSV)) // ssv + return DATASOURCE_SSV; + + else if(!strcmp(name, DATASOURCE_FORMAT_CSV)) // csv + return DATASOURCE_CSV; + + else if(!strcmp(name, DATASOURCE_FORMAT_TSV) || !strcmp(name, "tsv-excel")) // tsv + return DATASOURCE_TSV; + + else if(!strcmp(name, DATASOURCE_FORMAT_HTML)) // html + return DATASOURCE_HTML; + + else if(!strcmp(name, DATASOURCE_FORMAT_JS_ARRAY)) // array + return DATASOURCE_JS_ARRAY; + + else if(!strcmp(name, DATASOURCE_FORMAT_SSV_COMMA)) // ssvcomma + return DATASOURCE_SSV_COMMA; + + else if(!strcmp(name, DATASOURCE_FORMAT_CSV_JSON_ARRAY)) // csvjsonarray + return DATASOURCE_CSV_JSON_ARRAY; + + return DATASOURCE_JSON; +} + +uint32_t web_client_api_request_v1_data_google_format(char *name) +{ + if(!strcmp(name, "json")) + return DATASOURCE_DATATABLE_JSONP; + + else if(!strcmp(name, "html")) + return DATASOURCE_HTML; + + else if(!strcmp(name, "csv")) + return DATASOURCE_CSV; + + else if(!strcmp(name, "tsv-excel")) + return DATASOURCE_TSV; + + return DATASOURCE_JSON; +} + +int web_client_api_request_v1_data_group(char *name) +{ + if(!strcmp(name, "max")) + return GROUP_MAX; + + else if(!strcmp(name, "average")) + return GROUP_AVERAGE; + + return GROUP_MAX; +} + +int web_client_api_request_v1_charts(struct web_client *w, char *url) +{ + if(url) { ; } + + buffer_flush(w->response.data); + w->response.data->contenttype = CT_APPLICATION_JSON; + rrd_stats_api_v1_charts(w->response.data); + return 200; +} + +int web_client_api_request_v1_chart(struct web_client *w, char *url) +{ + int ret = 400; + char *chart = NULL; + + buffer_flush(w->response.data); + + while(url) { + char *value = mystrsep(&url, "?&[]"); + if(!value || !*value) continue; + + char *name = mystrsep(&value, "="); + if(!name || !*name) continue; + if(!value || !*value) continue; + + // name and value are now the parameters + // they are not null and not empty + + if(!strcmp(name, "chart")) chart = value; + //else { + /// buffer_sprintf(w->response.data, "Unknown parameter '%s' in request.", name); + // goto cleanup; + //} + } + + if(!chart || !*chart) { + buffer_sprintf(w->response.data, "No chart id is given at the request."); + goto cleanup; + } + + RRDSET *st = rrdset_find(chart); + if(!st) st = rrdset_find_byname(chart); + if(!st) { + buffer_sprintf(w->response.data, "Chart '%s' is not found.", chart); + ret = 404; + goto cleanup; + } + + w->response.data->contenttype = CT_APPLICATION_JSON; + rrd_stats_api_v1_chart(st, w->response.data); + return 200; + +cleanup: + return ret; +} + +// returns the HTTP code +int web_client_api_request_v1_data(struct web_client *w, char *url) +{ + debug(D_WEB_CLIENT, "%llu: API v1 data with URL '%s'", w->id, url); + + int ret = 400; + BUFFER *dimensions = NULL; + + buffer_flush(w->response.data); + + char *google_version = "0.6", + *google_reqId = "0", + *google_sig = "0", + *google_out = "json", + *responseHandler = NULL, + *outFileName = NULL; + + time_t last_timestamp_in_data = 0, google_timestamp = 0; + + char *chart = NULL + , *before_str = NULL + , *after_str = NULL + , *points_str = NULL; + + int group = GROUP_MAX; + uint32_t format = DATASOURCE_JSON; + uint32_t options = 0x00000000; + + while(url) { + char *value = mystrsep(&url, "?&[]"); + if(!value || !*value) continue; + + char *name = mystrsep(&value, "="); + if(!name || !*name) continue; + if(!value || !*value) continue; + + debug(D_WEB_CLIENT, "%llu: API v1 query param '%s' with value '%s'", w->id, name, value); + + // name and value are now the parameters + // they are not null and not empty + + if(!strcmp(name, "chart")) chart = value; + else if(!strcmp(name, "dimension") || !strcmp(name, "dim") || !strcmp(name, "dimensions") || !strcmp(name, "dims")) { + if(!dimensions) dimensions = buffer_create(strlen(value)); + if(dimensions) { + buffer_strcat(dimensions, "|"); + buffer_strcat(dimensions, value); + } + } + else if(!strcmp(name, "after")) after_str = value; + else if(!strcmp(name, "before")) before_str = value; + else if(!strcmp(name, "points")) points_str = value; + else if(!strcmp(name, "group")) { + group = web_client_api_request_v1_data_group(value); + } + else if(!strcmp(name, "format")) { + format = web_client_api_request_v1_data_format(value); + } + else if(!strcmp(name, "options")) { + options |= web_client_api_request_v1_data_options(value); + } + else if(!strcmp(name, "callback")) { + responseHandler = value; + } + else if(!strcmp(name, "filename")) { + outFileName = value; + } + else if(!strcmp(name, "tqx")) { + // parse Google Visualization API options + // https://developers.google.com/chart/interactive/docs/dev/implementing_data_source + char *tqx_name, *tqx_value; + + while(value) { + tqx_value = mystrsep(&value, ";"); + if(!tqx_value || !*tqx_value) continue; + + tqx_name = mystrsep(&tqx_value, ":"); + if(!tqx_name || !*tqx_name) continue; + if(!tqx_value || !*tqx_value) continue; + + if(!strcmp(tqx_name, "version")) + google_version = tqx_value; + else if(!strcmp(tqx_name, "reqId")) + google_reqId = tqx_value; + else if(!strcmp(tqx_name, "sig")) { + google_sig = tqx_value; + google_timestamp = strtoul(google_sig, NULL, 0); + } + else if(!strcmp(tqx_name, "out")) { + google_out = tqx_value; + format = web_client_api_request_v1_data_google_format(google_out); + } + else if(!strcmp(tqx_name, "responseHandler")) + responseHandler = tqx_value; + else if(!strcmp(tqx_name, "outFileName")) + outFileName = tqx_value; + } + } + } + + if(!chart || !*chart) { + buffer_sprintf(w->response.data, "No chart id is given at the request."); + goto cleanup; + } + + RRDSET *st = rrdset_find(chart); + if(!st) st = rrdset_find_byname(chart); + if(!st) { + buffer_sprintf(w->response.data, "Chart '%s' is not found.", chart); + ret = 404; + goto cleanup; + } + + long long before = (before_str && *before_str)?atol(before_str):0; + long long after = (after_str && *after_str) ?atol(after_str):0; + int points = (points_str && *points_str)?atoi(points_str):0; + + debug(D_WEB_CLIENT, "%llu: API command 'data' for chart '%s', dimensions '%s', after '%lld', before '%lld', points '%d', group '%u', format '%u', options '0x%08x'" + , w->id + , chart + , (dimensions)?buffer_tostring(dimensions):"" + , after + , before + , points + , group + , format + , options + ); + + if(outFileName && *outFileName) { + buffer_sprintf(w->response.header, "Content-Disposition: attachment; filename=\"%s\"\r\n", outFileName); + error("generating outfilename header: '%s'", outFileName); + } + + if(format == DATASOURCE_DATATABLE_JSONP) { + if(responseHandler == NULL) + responseHandler = "google.visualization.Query.setResponse"; + + debug(D_WEB_CLIENT_ACCESS, "%llu: GOOGLE JSON/JSONP: version = '%s', reqId = '%s', sig = '%s', out = '%s', responseHandler = '%s', outFileName = '%s'", + w->id, google_version, google_reqId, google_sig, google_out, responseHandler, outFileName + ); + + buffer_sprintf(w->response.data, + "%s({version:'%s',reqId:'%s',status:'ok',sig:'%lu',table:", + responseHandler, google_version, google_reqId, st->last_updated.tv_sec); + } + else if(format == DATASOURCE_JSONP) { + if(responseHandler == NULL) + responseHandler = "callback"; + + buffer_strcat(w->response.data, responseHandler); + buffer_strcat(w->response.data, "("); + } + + ret = rrd2format(st, w->response.data, dimensions, format, points, after, before, group, options, &last_timestamp_in_data); + + if(format == DATASOURCE_DATATABLE_JSONP) { + if(google_timestamp < last_timestamp_in_data) + buffer_strcat(w->response.data, "});"); + + else { + // the client already has the latest data + buffer_flush(w->response.data); + buffer_sprintf(w->response.data, + "%s({version:'%s',reqId:'%s',status:'error',errors:[{reason:'not_modified',message:'Data not modified'}]});", + responseHandler, google_version, google_reqId); + } + } + else if(format == DATASOURCE_JSONP) + buffer_strcat(w->response.data, ");"); + +cleanup: + if(dimensions) buffer_free(dimensions); + return ret; +} + +int web_client_api_request_v1(struct web_client *w, char *url) +{ + // get the command + char *tok = mystrsep(&url, "/?&"); + debug(D_WEB_CLIENT, "%llu: Searching for API v1 command '%s'.", w->id, tok); + + if(strcmp(tok, "data") == 0) + return web_client_api_request_v1_data(w, url); + else if(strcmp(tok, "chart") == 0) + return web_client_api_request_v1_chart(w, url); + else if(strcmp(tok, "charts") == 0) + return web_client_api_request_v1_charts(w, url); + + buffer_flush(w->response.data); + buffer_sprintf(w->response.data, "Unsupported v1 API command: %s", tok); + return 404; +} + +int web_client_api_request(struct web_client *w, char *url) +{ + // get the api version + char *tok = mystrsep(&url, "/?&"); + debug(D_WEB_CLIENT, "%llu: Searching for API version '%s'.", w->id, tok); + + if(strcmp(tok, "v1") == 0) + return web_client_api_request_v1(w, url); + + buffer_flush(w->response.data); + buffer_sprintf(w->response.data, "Unsupported API version: %s", tok); + return 404; +} + +int web_client_data_request(struct web_client *w, char *url, int datasource_type) +{ + char *args = strchr(url, '?'); + if(args) { + *args='\0'; + args = &args[1]; + } + + // get the name of the data to show + char *tok = mystrsep(&url, "/"); + debug(D_WEB_CLIENT, "%llu: Searching for RRD data with name '%s'.", w->id, tok); + + // do we have such a data set? + RRDSET *st = rrdset_find_byname(tok); + if(!st) st = rrdset_find(tok); + if(!st) { + // we don't have it + // try to send a file with that name + buffer_flush(w->response.data); + return(mysendfile(w, tok)); + } + + // we have it + debug(D_WEB_CLIENT, "%llu: Found RRD data with name '%s'.", w->id, tok); + + // how many entries does the client want? + long lines = rrd_default_history_entries; + long group_count = 1; + time_t after = 0, before = 0; + int group_method = GROUP_AVERAGE; + int nonzero = 0; + + if(url) { + // parse the lines required + tok = mystrsep(&url, "/"); + if(tok) lines = atoi(tok); + if(lines < 1) lines = 1; + } + if(url) { + // parse the group count required + tok = mystrsep(&url, "/"); + if(tok) group_count = atoi(tok); + if(group_count < 1) group_count = 1; + //if(group_count > save_history / 20) group_count = save_history / 20; + } + if(url) { + // parse the grouping method required + tok = mystrsep(&url, "/"); + if(strcmp(tok, "max") == 0) group_method = GROUP_MAX; + else if(strcmp(tok, "average") == 0) group_method = GROUP_AVERAGE; + else if(strcmp(tok, "sum") == 0) group_method = GROUP_SUM; + else debug(D_WEB_CLIENT, "%llu: Unknown group method '%s'", w->id, tok); + } + if(url) { + // parse after time + tok = mystrsep(&url, "/"); + if(tok) after = strtoul(tok, NULL, 10); + if(after < 0) after = 0; + } + if(url) { + // parse before time + tok = mystrsep(&url, "/"); + if(tok) before = strtoul(tok, NULL, 10); + if(before < 0) before = 0; + } + if(url) { + // parse nonzero + tok = mystrsep(&url, "/"); + if(tok && strcmp(tok, "nonzero") == 0) nonzero = 1; + } + + w->response.data->contenttype = CT_APPLICATION_JSON; + buffer_flush(w->response.data); + + char *google_version = "0.6"; + char *google_reqId = "0"; + char *google_sig = "0"; + char *google_out = "json"; + char *google_responseHandler = "google.visualization.Query.setResponse"; + char *google_outFileName = NULL; + time_t last_timestamp_in_data = 0; + if(datasource_type == DATASOURCE_DATATABLE_JSON || datasource_type == DATASOURCE_DATATABLE_JSONP) { + + w->response.data->contenttype = CT_APPLICATION_X_JAVASCRIPT; + + while(args) { + tok = mystrsep(&args, "&"); + if(tok) { + char *name = mystrsep(&tok, "="); + if(name && strcmp(name, "tqx") == 0) { + char *key = mystrsep(&tok, ":"); + char *value = mystrsep(&tok, ";"); + if(key && value && *key && *value) { + if(strcmp(key, "version") == 0) + google_version = value; + + else if(strcmp(key, "reqId") == 0) + google_reqId = value; + + else if(strcmp(key, "sig") == 0) + google_sig = value; + + else if(strcmp(key, "out") == 0) + google_out = value; + + else if(strcmp(key, "responseHandler") == 0) + google_responseHandler = value; + + else if(strcmp(key, "outFileName") == 0) + google_outFileName = value; + } + } + } + } + + debug(D_WEB_CLIENT_ACCESS, "%llu: GOOGLE JSONP: version = '%s', reqId = '%s', sig = '%s', out = '%s', responseHandler = '%s', outFileName = '%s'", + w->id, google_version, google_reqId, google_sig, google_out, google_responseHandler, google_outFileName + ); + + if(datasource_type == DATASOURCE_DATATABLE_JSONP) { + last_timestamp_in_data = strtoul(google_sig, NULL, 0); + + // check the client wants json + if(strcmp(google_out, "json") != 0) { + buffer_sprintf(w->response.data, + "%s({version:'%s',reqId:'%s',status:'error',errors:[{reason:'invalid_query',message:'output format is not supported',detailed_message:'the format %s requested is not supported by netdata.'}]});", + google_responseHandler, google_version, google_reqId, google_out); + return 200; + } + } + } + + if(datasource_type == DATASOURCE_DATATABLE_JSONP) { + buffer_sprintf(w->response.data, + "%s({version:'%s',reqId:'%s',status:'ok',sig:'%lu',table:", + google_responseHandler, google_version, google_reqId, st->last_updated.tv_sec); + } + + debug(D_WEB_CLIENT_ACCESS, "%llu: Sending RRD data '%s' (id %s, %d lines, %d group, %d group_method, %lu after, %lu before).", w->id, st->name, st->id, lines, group_count, group_method, after, before); + time_t timestamp_in_data = rrd_stats_json(datasource_type, st, w->response.data, lines, group_count, group_method, after, before, nonzero); + + if(datasource_type == DATASOURCE_DATATABLE_JSONP) { + if(timestamp_in_data > last_timestamp_in_data) + buffer_strcat(w->response.data, "});"); + + else { + // the client already has the latest data + buffer_flush(w->response.data); + buffer_sprintf(w->response.data, + "%s({version:'%s',reqId:'%s',status:'error',errors:[{reason:'not_modified',message:'Data not modified'}]});", + google_responseHandler, google_version, google_reqId); + } + } + + return 200; +} + +/* +int web_client_parse_request(struct web_client *w) { + // protocol + // hostname + // path + // query string name-value + // http version + // method + // http request headers name-value + + web_client_clean_request(w); + + debug(D_WEB_DATA, "%llu: Processing data buffer of %d bytes: '%s'.", w->id, w->response.data->bytes, w->response.data->buffer); + + char *buf = w->response.data->buffer; + char *line, *tok; + + // ------------------------------------------------------------------------ + // the first line + + if(buf && (line = strsep(&buf, "\r\n"))) { + // method + if(line && (tok = strsep(&line, " "))) { + w->request.protocol = strdup(tok); + } + else goto cleanup; + + // url + } + else goto cleanup; + + // ------------------------------------------------------------------------ + // the rest of the lines + + while(buf && (line = strsep(&buf, "\r\n"))) { + while(line && (tok = strsep(&line, ": "))) { + } + } + + char *url = NULL; + + +cleanup: + web_client_clean_request(w); + return 0; +} +*/ + +void web_client_process(struct web_client *w) { + int code = 500; + ssize_t bytes; + int enable_gzip = 0; + + w->wait_receive = 0; + + // check if we have an empty line (end of HTTP header) + if(strstr(w->response.data->buffer, "\r\n\r\n")) { + global_statistics_lock(); + global_statistics.web_requests++; + global_statistics_unlock(); + + gettimeofday(&w->tv_in, NULL); + debug(D_WEB_DATA, "%llu: Processing data buffer of %d bytes: '%s'.", w->id, w->response.data->len, w->response.data->buffer); + + // check if the client requested keep-alive HTTP + if(strcasestr(w->response.data->buffer, "Connection: keep-alive")) w->keepalive = 1; + else w->keepalive = 0; + +#ifdef NETDATA_WITH_ZLIB + // check if the client accepts deflate + if(web_enable_gzip && strstr(w->response.data->buffer, "gzip")) + enable_gzip = 1; +#endif // NETDATA_WITH_ZLIB + + int datasource_type = DATASOURCE_DATATABLE_JSONP; + //if(strstr(w->response.data->buffer, "X-DataSource-Auth")) + // datasource_type = DATASOURCE_GOOGLE_JSON; + + char *buf = (char *)buffer_tostring(w->response.data); + char *tok = strsep(&buf, " \r\n"); + char *url = NULL; + char *pointer_to_free = NULL; // keep url_decode() allocated buffer + + w->mode = WEB_CLIENT_MODE_NORMAL; + + if(buf && strcmp(tok, "GET") == 0) { + tok = strsep(&buf, " \r\n"); + pointer_to_free = url = url_decode(tok); + debug(D_WEB_CLIENT, "%llu: Processing HTTP GET on url '%s'.", w->id, url); + } + else if(buf && strcmp(tok, "OPTIONS") == 0) { + tok = strsep(&buf, " \r\n"); + pointer_to_free = url = url_decode(tok); + debug(D_WEB_CLIENT, "%llu: Processing HTTP OPTIONS on url '%s'.", w->id, url); + w->mode = WEB_CLIENT_MODE_OPTIONS; + } + else if (buf && strcmp(tok, "POST") == 0) { + w->keepalive = 0; + tok = strsep(&buf, " \r\n"); + pointer_to_free = url = url_decode(tok); + debug(D_WEB_CLIENT, "%llu: I don't know how to handle POST with form data. Assuming it is a GET on url '%s'.", w->id, url); + } + + w->last_url[0] = '\0'; + + if(w->mode == WEB_CLIENT_MODE_OPTIONS) { + strncpy(w->last_url, url, URL_MAX); + w->last_url[URL_MAX] = '\0'; + + code = 200; + w->response.data->contenttype = CT_TEXT_PLAIN; + buffer_flush(w->response.data); + buffer_strcat(w->response.data, "OK"); + } + else if(url) { +#ifdef NETDATA_WITH_ZLIB + if(enable_gzip) + web_client_enable_deflate(w); +#endif + + strncpy(w->last_url, url, URL_MAX); + w->last_url[URL_MAX] = '\0'; + + tok = mystrsep(&url, "/?"); + + debug(D_WEB_CLIENT, "%llu: Processing command '%s'.", w->id, tok); + + if(strcmp(tok, "api") == 0) { + // the client is requesting api access + datasource_type = DATASOURCE_JSON; + code = web_client_api_request(w, url); + } +#ifdef NETDATA_INTERNAL_CHECKS + else if(strcmp(tok, "exit") == 0) { + netdata_exit = 1; + code = 200; + w->response.data->contenttype = CT_TEXT_PLAIN; + buffer_flush(w->response.data); + buffer_strcat(w->response.data, "will do"); + } +#endif + else if(strcmp(tok, WEB_PATH_DATA) == 0) { // "data" + // the client is requesting rrd data + datasource_type = DATASOURCE_JSON; + code = web_client_data_request(w, url, datasource_type); + } + else if(strcmp(tok, WEB_PATH_DATASOURCE) == 0) { // "datasource" + // the client is requesting google datasource + code = web_client_data_request(w, url, datasource_type); + } + else if(strcmp(tok, WEB_PATH_GRAPH) == 0) { // "graph" + // the client is requesting an rrd graph + + // get the name of the data to show + tok = mystrsep(&url, "/?&"); + debug(D_WEB_CLIENT, "%llu: Searching for RRD data with name '%s'.", w->id, tok); + + // do we have such a data set? + RRDSET *st = rrdset_find_byname(tok); + if(!st) st = rrdset_find(tok); + if(!st) { + // we don't have it + // try to send a file with that name + buffer_flush(w->response.data); + code = mysendfile(w, tok); + } + else { + code = 200; + debug(D_WEB_CLIENT_ACCESS, "%llu: Sending %s.json of RRD_STATS...", w->id, st->name); + w->response.data->contenttype = CT_APPLICATION_JSON; + buffer_flush(w->response.data); + rrd_stats_graph_json(st, url, w->response.data); + } + } +#ifdef NETDATA_INTERNAL_CHECKS + else if(strcmp(tok, "debug") == 0) { + buffer_flush(w->response.data); + + // get the name of the data to show + tok = mystrsep(&url, "/?&"); + debug(D_WEB_CLIENT, "%llu: Searching for RRD data with name '%s'.", w->id, tok); + + // do we have such a data set? + RRDSET *st = rrdset_find_byname(tok); + if(!st) st = rrdset_find(tok); + if(!st) { + code = 404; + buffer_sprintf(w->response.data, "Chart %s is not found.\r\n", tok); + debug(D_WEB_CLIENT_ACCESS, "%llu: %s is not found.", w->id, tok); + } + else { + code = 200; + debug_flags |= D_RRD_STATS; + st->debug = st->debug?0:1; + buffer_sprintf(w->response.data, "Chart %s has now debug %s.\r\n", tok, st->debug?"enabled":"disabled"); + debug(D_WEB_CLIENT_ACCESS, "%llu: debug for %s is %s.", w->id, tok, st->debug?"enabled":"disabled"); + } + } + else if(strcmp(tok, "mirror") == 0) { + code = 200; + + debug(D_WEB_CLIENT_ACCESS, "%llu: Mirroring...", w->id); + + // replace the zero bytes with spaces + buffer_char_replace(w->response.data, '\0', ' '); + + // just leave the buffer as is + // it will be copied back to the client + } +#endif + else if(strcmp(tok, "list") == 0) { + code = 200; + + debug(D_WEB_CLIENT_ACCESS, "%llu: Sending list of RRD_STATS...", w->id); + + buffer_flush(w->response.data); + RRDSET *st = rrdset_root; + + for ( ; st ; st = st->next ) + buffer_sprintf(w->response.data, "%s\n", st->name); + } + else if(strcmp(tok, "all.json") == 0) { + code = 200; + debug(D_WEB_CLIENT_ACCESS, "%llu: Sending JSON list of all monitors of RRD_STATS...", w->id); + + w->response.data->contenttype = CT_APPLICATION_JSON; + buffer_flush(w->response.data); + rrd_stats_all_json(w->response.data); + } + else if(strcmp(tok, "netdata.conf") == 0) { + code = 200; + debug(D_WEB_CLIENT_ACCESS, "%llu: Sending netdata.conf ...", w->id); + + w->response.data->contenttype = CT_TEXT_PLAIN; + buffer_flush(w->response.data); + generate_config(w->response.data, 0); + } + else { + char filename[FILENAME_MAX+1]; + url = filename; + strncpy(filename, w->last_url, FILENAME_MAX); + filename[FILENAME_MAX] = '\0'; + tok = mystrsep(&url, "?"); + buffer_flush(w->response.data); + code = mysendfile(w, (tok && *tok)?tok:"/"); + } + } + else { + strcpy(w->last_url, "not a valid response"); + + if(buf) debug(D_WEB_CLIENT_ACCESS, "%llu: Cannot understand '%s'.", w->id, buf); + + code = 500; + buffer_flush(w->response.data); + buffer_strcat(w->response.data, "I don't understand you...\r\n"); + } + + // free url_decode() buffer + if(pointer_to_free) free(pointer_to_free); + } + else if(w->response.data->len > 8192) { + strcpy(w->last_url, "too big request"); + + debug(D_WEB_CLIENT_ACCESS, "%llu: Received request is too big.", w->id); + + code = 400; + buffer_flush(w->response.data); + buffer_strcat(w->response.data, "Received request is too big.\r\n"); + } + else { + // wait for more data + w->wait_receive = 1; + return; + } + + gettimeofday(&w->tv_ready, NULL); + w->response.data->date = time(NULL); + w->response.sent = 0; + w->response.code = code; + + // prepare the HTTP response header + debug(D_WEB_CLIENT, "%llu: Generating HTTP header with response %d.", w->id, code); + + char *content_type_string; + switch(w->response.data->contenttype) { + case CT_TEXT_HTML: + content_type_string = "text/html; charset=utf-8"; + break; + + case CT_APPLICATION_XML: + content_type_string = "application/xml; charset=utf-8"; + break; + + case CT_APPLICATION_JSON: + content_type_string = "application/json; charset=utf-8"; + break; + + case CT_APPLICATION_X_JAVASCRIPT: + content_type_string = "application/x-javascript; charset=utf-8"; + break; + + case CT_TEXT_CSS: + content_type_string = "text/css; charset=utf-8"; + break; + + case CT_TEXT_XML: + content_type_string = "text/xml; charset=utf-8"; + break; + + case CT_TEXT_XSL: + content_type_string = "text/xsl; charset=utf-8"; + break; + + case CT_APPLICATION_OCTET_STREAM: + content_type_string = "application/octet-stream"; + break; + + case CT_IMAGE_SVG_XML: + content_type_string = "image/svg+xml"; + break; + + case CT_APPLICATION_X_FONT_TRUETYPE: + content_type_string = "application/x-font-truetype"; + break; + + case CT_APPLICATION_X_FONT_OPENTYPE: + content_type_string = "application/x-font-opentype"; + break; + + case CT_APPLICATION_FONT_WOFF: + content_type_string = "application/font-woff"; + break; + + case CT_APPLICATION_FONT_WOFF2: + content_type_string = "application/font-woff2"; + break; + + case CT_APPLICATION_VND_MS_FONTOBJ: + content_type_string = "application/vnd.ms-fontobject"; + break; + + case CT_IMAGE_PNG: + content_type_string = "image/png"; + break; + + case CT_IMAGE_JPG: + content_type_string = "image/jpeg"; + break; + + case CT_IMAGE_GIF: + content_type_string = "image/gif"; + break; + + case CT_IMAGE_XICON: + content_type_string = "image/x-icon"; + break; + + case CT_IMAGE_BMP: + content_type_string = "image/bmp"; + break; + + case CT_IMAGE_ICNS: + content_type_string = "image/icns"; + break; + + default: + case CT_TEXT_PLAIN: + content_type_string = "text/plain; charset=utf-8"; + break; + } + + char *code_msg; + switch(code) { + case 200: + code_msg = "OK"; + break; + + case 307: + code_msg = "Temporary Redirect"; + break; + + case 400: + code_msg = "Bad Request"; + break; + + case 403: + code_msg = "Forbidden"; + break; + + case 404: + code_msg = "Not Found"; + break; + + default: + code_msg = "Internal Server Error"; + break; + } + + char date[100]; + struct tm tmbuf, *tm = gmtime_r(&w->response.data->date, &tmbuf); + strftime(date, sizeof(date), "%a, %d %b %Y %H:%M:%S %Z", tm); + + buffer_sprintf(w->response.header_output, + "HTTP/1.1 %d %s\r\n" + "Connection: %s\r\n" + "Server: NetData Embedded HTTP Server\r\n" + "Content-Type: %s\r\n" + "Access-Control-Allow-Origin: *\r\n" + "Access-Control-Allow-Methods: GET, OPTIONS\r\n" + "Access-Control-Allow-Headers: accept, x-requested-with\r\n" + "Access-Control-Max-Age: 86400\r\n" + "Date: %s\r\n" + , code, code_msg + , w->keepalive?"keep-alive":"close" + , content_type_string + , date + ); + + if(buffer_strlen(w->response.header)) + buffer_strcat(w->response.header_output, buffer_tostring(w->response.header)); + + if(w->mode == WEB_CLIENT_MODE_NORMAL && (w->response.data->options & WB_CONTENT_NO_CACHEABLE)) { + buffer_sprintf(w->response.header_output, + "Expires: %s\r\n" + "Cache-Control: no-cache\r\n" + , date); + } + else { + char edate[100]; + time_t et = w->response.data->date + (86400 * 14); + struct tm etmbuf, *etm = gmtime_r(&et, &etmbuf); + strftime(edate, sizeof(edate), "%a, %d %b %Y %H:%M:%S %Z", etm); + + buffer_sprintf(w->response.header_output, + "Expires: %s\r\n" + "Cache-Control: public\r\n" + , edate); + } + + // if we know the content length, put it + if(!w->response.zoutput && (w->response.data->len || w->response.rlen)) + buffer_sprintf(w->response.header_output, + "Content-Length: %ld\r\n" + , w->response.data->len? w->response.data->len: w->response.rlen + ); + else if(!w->response.zoutput) + w->keepalive = 0; // content-length is required for keep-alive + + if(w->response.zoutput) { + buffer_strcat(w->response.header_output, + "Content-Encoding: gzip\r\n" + "Transfer-Encoding: chunked\r\n" + ); + } + + buffer_strcat(w->response.header_output, "\r\n"); + + // disable TCP_NODELAY, to buffer the header + int flag = 0; + if(setsockopt(w->ofd, IPPROTO_TCP, TCP_NODELAY, (char *) &flag, sizeof(int)) != 0) + error("%llu: failed to disable TCP_NODELAY on socket.", w->id); + + // sent the HTTP header + debug(D_WEB_DATA, "%llu: Sending response HTTP header of size %d: '%s'" + , w->id + , buffer_strlen(w->response.header_output) + , buffer_tostring(w->response.header_output) + ); + + bytes = send(w->ofd, buffer_tostring(w->response.header_output), buffer_strlen(w->response.header_output), 0); + if(bytes != (ssize_t) buffer_strlen(w->response.header_output)) + error("%llu: HTTP Header failed to be sent (I sent %d bytes but the system sent %d bytes)." + , w->id + , buffer_strlen(w->response.header_output) + , bytes); + else { + global_statistics_lock(); + global_statistics.bytes_sent += bytes; + global_statistics_unlock(); + } + + // enable TCP_NODELAY, to send all data immediately at the next send() + flag = 1; + if(setsockopt(w->ofd, IPPROTO_TCP, TCP_NODELAY, (char *) &flag, sizeof(int)) != 0) error("%llu: failed to enable TCP_NODELAY on socket.", w->id); + + // enable sending immediately if we have data + if(w->response.data->len) w->wait_send = 1; + else w->wait_send = 0; + + // pretty logging + switch(w->mode) { + case WEB_CLIENT_MODE_OPTIONS: + debug(D_WEB_CLIENT, "%llu: Done preparing the OPTIONS response. Sending data (%d bytes) to client.", w->id, w->response.data->len); + break; + + case WEB_CLIENT_MODE_NORMAL: + debug(D_WEB_CLIENT, "%llu: Done preparing the response. Sending data (%d bytes) to client.", w->id, w->response.data->len); + break; + + case WEB_CLIENT_MODE_FILECOPY: + if(w->response.rlen) { + debug(D_WEB_CLIENT, "%llu: Done preparing the response. Will be sending data file of %d bytes to client.", w->id, w->response.rlen); + w->wait_receive = 1; + + /* + // utilize the kernel sendfile() for copying the file to the socket. + // this block of code can be commented, without anything missing. + // when it is commented, the program will copy the data using async I/O. + { + long len = sendfile(w->ofd, w->ifd, NULL, w->response.data->rbytes); + if(len != w->response.data->rbytes) error("%llu: sendfile() should copy %ld bytes, but copied %ld. Falling back to manual copy.", w->id, w->response.data->rbytes, len); + else web_client_reset(w); + } + */ + } + else + debug(D_WEB_CLIENT, "%llu: Done preparing the response. Will be sending an unknown amount of bytes to client.", w->id); + break; + + default: + fatal("%llu: Unknown client mode %d.", w->id, w->mode); + break; + } +} + +long web_client_send_chunk_header(struct web_client *w, long len) +{ + debug(D_DEFLATE, "%llu: OPEN CHUNK of %d bytes (hex: %x).", w->id, len, len); + char buf[1024]; + sprintf(buf, "%lX\r\n", len); + ssize_t bytes = send(w->ofd, buf, strlen(buf), MSG_DONTWAIT); + + if(bytes > 0) debug(D_DEFLATE, "%llu: Sent chunk header %d bytes.", w->id, bytes); + else if(bytes == 0) debug(D_DEFLATE, "%llu: Did not send chunk header to the client.", w->id); + else debug(D_DEFLATE, "%llu: Failed to send chunk header to client.", w->id); + + return bytes; +} + +long web_client_send_chunk_close(struct web_client *w) +{ + //debug(D_DEFLATE, "%llu: CLOSE CHUNK.", w->id); + + ssize_t bytes = send(w->ofd, "\r\n", 2, MSG_DONTWAIT); + + if(bytes > 0) debug(D_DEFLATE, "%llu: Sent chunk suffix %d bytes.", w->id, bytes); + else if(bytes == 0) debug(D_DEFLATE, "%llu: Did not send chunk suffix to the client.", w->id); + else debug(D_DEFLATE, "%llu: Failed to send chunk suffix to client.", w->id); + + return bytes; +} + +long web_client_send_chunk_finalize(struct web_client *w) +{ + //debug(D_DEFLATE, "%llu: FINALIZE CHUNK.", w->id); + + ssize_t bytes = send(w->ofd, "\r\n0\r\n\r\n", 7, MSG_DONTWAIT); + + if(bytes > 0) debug(D_DEFLATE, "%llu: Sent chunk suffix %d bytes.", w->id, bytes); + else if(bytes == 0) debug(D_DEFLATE, "%llu: Did not send chunk suffix to the client.", w->id); + else debug(D_DEFLATE, "%llu: Failed to send chunk suffix to client.", w->id); + + return bytes; +} + +#ifdef NETDATA_WITH_ZLIB +long web_client_send_deflate(struct web_client *w) +{ + long len = 0, t = 0; + + // when using compression, + // w->response.sent is the amount of bytes passed through compression + + debug(D_DEFLATE, "%llu: web_client_send_deflate(): w->response.data->len = %d, w->response.sent = %d, w->response.zhave = %d, w->response.zsent = %d, w->response.zstream.avail_in = %d, w->response.zstream.avail_out = %d, w->response.zstream.total_in = %d, w->response.zstream.total_out = %d.", w->id, w->response.data->len, w->response.sent, w->response.zhave, w->response.zsent, w->response.zstream.avail_in, w->response.zstream.avail_out, w->response.zstream.total_in, w->response.zstream.total_out); + + if(w->response.data->len - w->response.sent == 0 && w->response.zstream.avail_in == 0 && w->response.zhave == w->response.zsent && w->response.zstream.avail_out != 0) { + // there is nothing to send + + debug(D_WEB_CLIENT, "%llu: Out of output data.", w->id); + + // finalize the chunk + if(w->response.sent != 0) + t += web_client_send_chunk_finalize(w); + + // there can be two cases for this + // A. we have done everything + // B. we temporarily have nothing to send, waiting for the buffer to be filled by ifd + + if(w->mode == WEB_CLIENT_MODE_FILECOPY && w->wait_receive && w->ifd != w->ofd && w->response.rlen && w->response.rlen > w->response.data->len) { + // we have to wait, more data will come + debug(D_WEB_CLIENT, "%llu: Waiting for more data to become available.", w->id); + w->wait_send = 0; + return(0); + } + + if(w->keepalive == 0) { + debug(D_WEB_CLIENT, "%llu: Closing (keep-alive is not enabled). %ld bytes sent.", w->id, w->response.sent); + errno = 0; + return(-1); + } + + // reset the client + web_client_reset(w); + debug(D_WEB_CLIENT, "%llu: Done sending all data on socket. Waiting for next request on the same socket.", w->id); + return(0); + } + + if(w->response.zhave == w->response.zsent) { + // compress more input data + + // close the previous open chunk + if(w->response.sent != 0) t += web_client_send_chunk_close(w); + + debug(D_DEFLATE, "%llu: Compressing %d new bytes starting from %d (and %d left behind).", w->id, (w->response.data->len - w->response.sent), w->response.sent, w->response.zstream.avail_in); + + // give the compressor all the data not passed through the compressor yet + if(w->response.data->len > w->response.sent) { +#ifdef NETDATA_INTERNAL_CHECKS + if((long)w->response.sent - (long)w->response.zstream.avail_in < 0) + error("internal error: avail_in is corrupted."); +#endif + w->response.zstream.next_in = (Bytef *)&w->response.data->buffer[w->response.sent - w->response.zstream.avail_in]; + w->response.zstream.avail_in += (uInt) (w->response.data->len - w->response.sent); + } + + // reset the compressor output buffer + w->response.zstream.next_out = w->response.zbuffer; + w->response.zstream.avail_out = ZLIB_CHUNK; + + // ask for FINISH if we have all the input + int flush = Z_SYNC_FLUSH; + if(w->mode == WEB_CLIENT_MODE_NORMAL + || (w->mode == WEB_CLIENT_MODE_FILECOPY && !w->wait_receive && w->response.data->len == w->response.rlen)) { + flush = Z_FINISH; + debug(D_DEFLATE, "%llu: Requesting Z_FINISH, if possible.", w->id); + } + else { + debug(D_DEFLATE, "%llu: Requesting Z_SYNC_FLUSH.", w->id); + } + + // compress + if(deflate(&w->response.zstream, flush) == Z_STREAM_ERROR) { + error("%llu: Compression failed. Closing down client.", w->id); + web_client_reset(w); + return(-1); + } + + w->response.zhave = ZLIB_CHUNK - w->response.zstream.avail_out; + w->response.zsent = 0; + + // keep track of the bytes passed through the compressor + w->response.sent = w->response.data->len; + + debug(D_DEFLATE, "%llu: Compression produced %d bytes.", w->id, w->response.zhave); + + // open a new chunk + t += web_client_send_chunk_header(w, w->response.zhave); + } + + debug(D_WEB_CLIENT, "%llu: Sending %d bytes of data (+%d of chunk header).", w->id, w->response.zhave - w->response.zsent, t); + + len = send(w->ofd, &w->response.zbuffer[w->response.zsent], (size_t) (w->response.zhave - w->response.zsent), MSG_DONTWAIT); + if(len > 0) { + w->response.zsent += len; + if(t > 0) len += t; + debug(D_WEB_CLIENT, "%llu: Sent %d bytes.", w->id, len); + } + else if(len == 0) debug(D_WEB_CLIENT, "%llu: Did not send any bytes to the client (zhave = %ld, zsent = %ld, need to send = %ld).", w->id, w->response.zhave, w->response.zsent, w->response.zhave - w->response.zsent); + else debug(D_WEB_CLIENT, "%llu: Failed to send data to client. Reason: %s", w->id, strerror(errno)); + + return(len); +} +#endif // NETDATA_WITH_ZLIB + +long web_client_send(struct web_client *w) +{ +#ifdef NETDATA_WITH_ZLIB + if(likely(w->response.zoutput)) return web_client_send_deflate(w); +#endif // NETDATA_WITH_ZLIB + + long bytes; + + if(unlikely(w->response.data->len - w->response.sent == 0)) { + // there is nothing to send + + debug(D_WEB_CLIENT, "%llu: Out of output data.", w->id); + + // there can be two cases for this + // A. we have done everything + // B. we temporarily have nothing to send, waiting for the buffer to be filled by ifd + + if(w->mode == WEB_CLIENT_MODE_FILECOPY && w->wait_receive && w->ifd != w->ofd && w->response.rlen && w->response.rlen > w->response.data->len) { + // we have to wait, more data will come + debug(D_WEB_CLIENT, "%llu: Waiting for more data to become available.", w->id); + w->wait_send = 0; + return(0); + } + + if(unlikely(w->keepalive == 0)) { + debug(D_WEB_CLIENT, "%llu: Closing (keep-alive is not enabled). %ld bytes sent.", w->id, w->response.sent); + errno = 0; + return(-1); + } + + web_client_reset(w); + debug(D_WEB_CLIENT, "%llu: Done sending all data on socket. Waiting for next request on the same socket.", w->id); + return(0); + } + + bytes = send(w->ofd, &w->response.data->buffer[w->response.sent], w->response.data->len - w->response.sent, MSG_DONTWAIT); + if(likely(bytes > 0)) { + w->response.sent += bytes; + debug(D_WEB_CLIENT, "%llu: Sent %d bytes.", w->id, bytes); + } + else if(likely(bytes == 0)) debug(D_WEB_CLIENT, "%llu: Did not send any bytes to the client.", w->id); + else debug(D_WEB_CLIENT, "%llu: Failed to send data to client.", w->id); + + return(bytes); +} + +long web_client_receive(struct web_client *w) +{ + // do we have any space for more data? + buffer_need_bytes(w->response.data, WEB_REQUEST_LENGTH); + + long left = w->response.data->size - w->response.data->len; + long bytes; + + if(unlikely(w->mode == WEB_CLIENT_MODE_FILECOPY)) + bytes = read(w->ifd, &w->response.data->buffer[w->response.data->len], (size_t) (left - 1)); + else + bytes = recv(w->ifd, &w->response.data->buffer[w->response.data->len], (size_t) (left - 1), MSG_DONTWAIT); + + if(likely(bytes > 0)) { + size_t old = w->response.data->len; + w->response.data->len += bytes; + w->response.data->buffer[w->response.data->len] = '\0'; + + debug(D_WEB_CLIENT, "%llu: Received %d bytes.", w->id, bytes); + debug(D_WEB_DATA, "%llu: Received data: '%s'.", w->id, &w->response.data->buffer[old]); + + if(w->mode == WEB_CLIENT_MODE_FILECOPY) { + w->wait_send = 1; + if(w->response.rlen && w->response.data->len >= w->response.rlen) w->wait_receive = 0; + } + } + else if(likely(bytes == 0)) { + debug(D_WEB_CLIENT, "%llu: Out of input data.", w->id); + + // if we cannot read, it means we have an error on input. + // if however, we are copying a file from ifd to ofd, we should not return an error. + // in this case, the error should be generated when the file has been sent to the client. + + if(w->mode == WEB_CLIENT_MODE_FILECOPY) { + // we are copying data from ifd to ofd + // let it finish copying... + w->wait_receive = 0; + debug(D_WEB_CLIENT, "%llu: Disabling input.", w->id); + } + else { + bytes = -1; + errno = 0; + } + } + + return(bytes); +} + + +// -------------------------------------------------------------------------------------- +// the thread of a single client + +// 1. waits for input and output, using async I/O +// 2. it processes HTTP requests +// 3. it generates HTTP responses +// 4. it copies data from input to output if mode is FILECOPY + +void *web_client_main(void *ptr) +{ + if(pthread_setcanceltype(PTHREAD_CANCEL_DEFERRED, NULL) != 0) + error("Cannot set pthread cancel type to DEFERRED."); + + if(pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, NULL) != 0) + error("Cannot set pthread cancel state to ENABLE."); + + struct timeval tv; + struct web_client *w = ptr; + int retval; + fd_set ifds, ofds, efds; + int fdmax = 0; + + log_access("%llu: %s port %s connected on thread task id %d", w->id, w->client_ip, w->client_port, gettid()); + + for(;;) { + FD_ZERO (&ifds); + FD_ZERO (&ofds); + FD_ZERO (&efds); + + FD_SET(w->ifd, &efds); + + if(w->ifd != w->ofd) + FD_SET(w->ofd, &efds); + + if (w->wait_receive) { + FD_SET(w->ifd, &ifds); + if(w->ifd > fdmax) fdmax = w->ifd; + } + + if (w->wait_send) { + FD_SET(w->ofd, &ofds); + if(w->ofd > fdmax) fdmax = w->ofd; + } + + tv.tv_sec = web_client_timeout; + tv.tv_usec = 0; + + debug(D_WEB_CLIENT, "%llu: Waiting socket async I/O for %s %s", w->id, w->wait_receive?"INPUT":"", w->wait_send?"OUTPUT":""); + retval = select(fdmax+1, &ifds, &ofds, &efds, &tv); + + if(retval == -1) { + debug(D_WEB_CLIENT_ACCESS, "%llu: LISTENER: select() failed.", w->id); + continue; + } + else if(!retval) { + // timeout + debug(D_WEB_CLIENT_ACCESS, "%llu: LISTENER: timeout.", w->id); + break; + } + + if(FD_ISSET(w->ifd, &efds)) { + debug(D_WEB_CLIENT_ACCESS, "%llu: Received error on input socket.", w->id); + break; + } + + if(FD_ISSET(w->ofd, &efds)) { + debug(D_WEB_CLIENT_ACCESS, "%llu: Received error on output socket.", w->id); + break; + } + + if(w->wait_send && FD_ISSET(w->ofd, &ofds)) { + long bytes; + if((bytes = web_client_send(w)) < 0) { + debug(D_WEB_CLIENT, "%llu: Cannot send data to client. Closing client.", w->id); + errno = 0; + break; + } + + global_statistics_lock(); + global_statistics.bytes_sent += bytes; + global_statistics_unlock(); + } + + if(w->wait_receive && FD_ISSET(w->ifd, &ifds)) { + long bytes; + if((bytes = web_client_receive(w)) < 0) { + debug(D_WEB_CLIENT, "%llu: Cannot receive data from client. Closing client.", w->id); + errno = 0; + break; + } + + if(w->mode == WEB_CLIENT_MODE_NORMAL) { + debug(D_WEB_CLIENT, "%llu: Attempting to process received data (%ld bytes).", w->id, bytes); + // info("%llu: Attempting to process received data (%ld bytes).", w->id, bytes); + web_client_process(w); + } + + global_statistics_lock(); + global_statistics.bytes_received += bytes; + global_statistics_unlock(); + } + } + + log_access("%llu: %s port %s disconnected from thread task id %d", w->id, w->client_ip, w->client_port, gettid()); + debug(D_WEB_CLIENT, "%llu: done...", w->id); + + web_client_reset(w); + w->obsolete = 1; + + return NULL; +} diff --git a/src/web_client.h b/src/web_client.h new file mode 100755 index 000000000..14cd91bc6 --- /dev/null +++ b/src/web_client.h @@ -0,0 +1,90 @@ + +#ifdef NETDATA_WITH_ZLIB +#include <zlib.h> +#endif + +#include <sys/time.h> +#include <string.h> +#include <sys/socket.h> +#include <netinet/in.h> +#include <arpa/inet.h> +#include <netdb.h> + +#include "web_buffer.h" + +#define DEFAULT_DISCONNECT_IDLE_WEB_CLIENTS_AFTER_SECONDS 60 +extern int web_client_timeout; +extern int web_enable_gzip; + +#ifndef NETDATA_WEB_CLIENT_H +#define NETDATA_WEB_CLIENT_H 1 + +#define WEB_CLIENT_MODE_NORMAL 0 +#define WEB_CLIENT_MODE_FILECOPY 1 +#define WEB_CLIENT_MODE_OPTIONS 2 + +#define URL_MAX 8192 +#define ZLIB_CHUNK 16384 +#define HTTP_RESPONSE_HEADER_SIZE 4096 + +struct response { + BUFFER *header; // our response header + BUFFER *header_output; // internal use + BUFFER *data; // our response data buffer + + int code; // the HTTP response code + + size_t rlen; // if non-zero, the excepted size of ifd (input) + size_t sent; // current data length sent to output + + int zoutput; // if set to 1, web_client_send() will send compressed data +#ifdef NETDATA_WITH_ZLIB + z_stream zstream; // zlib stream for sending compressed output to client + Bytef zbuffer[ZLIB_CHUNK]; // temporary buffer for storing compressed output + long zsent; // the compressed bytes we have sent to the client + long zhave; // the compressed bytes that we have to send + int zinitialized; +#endif + +}; + +struct web_client { + unsigned long long id; + + char client_ip[NI_MAXHOST+1]; + char client_port[NI_MAXSERV+1]; + + char last_url[URL_MAX+1]; + + struct timeval tv_in, tv_ready; + + int mode; + int keepalive; + + struct sockaddr_storage clientaddr; + + pthread_t thread; // the thread servicing this client + int obsolete; // if set to 1, the listener will remove this client + + int ifd; + int ofd; + + struct response response; + + int wait_receive; + int wait_send; + + struct web_client *prev; + struct web_client *next; +}; + +extern struct web_client *web_clients; + +extern uid_t web_files_uid(void); + +extern struct web_client *web_client_create(int listener); +extern struct web_client *web_client_free(struct web_client *w); + +extern void *web_client_main(void *ptr); + +#endif diff --git a/src/web_server.c b/src/web_server.c new file mode 100755 index 000000000..cae94acad --- /dev/null +++ b/src/web_server.c @@ -0,0 +1,225 @@ +#ifdef HAVE_CONFIG_H +#include <config.h> +#endif +#include <unistd.h> +#include <stdlib.h> +#include <sys/types.h> +#include <sys/socket.h> +#include <netinet/in.h> +#include <arpa/inet.h> +#include <errno.h> +#include <pthread.h> +#include <sys/stat.h> +#include <fcntl.h> +#include <netinet/tcp.h> +#include <malloc.h> + +#include "common.h" +#include "log.h" +#include "appconfig.h" +#include "url.h" +#include "web_buffer.h" +#include "web_client.h" +#include "web_server.h" +#include "global_statistics.h" +#include "rrd.h" +#include "rrd2json.h" + +int listen_backlog = LISTEN_BACKLOG; + +int listen_fd = -1; +int listen_port = LISTEN_PORT; + +static void log_allocations(void) +{ + static int mem = 0; + + struct mallinfo mi; + + mi = mallinfo(); + if(mi.uordblks > mem) { + int clients = 0; + struct web_client *w; + for(w = web_clients; w ; w = w->next) clients++; + + info("Allocated memory increased from %d to %d (increased by %d bytes). There are %d web clients connected.", mem, mi.uordblks, mi.uordblks - mem, clients); + mem = mi.uordblks; + } +} + +int create_listen_socket4(int port, int listen_backlog) +{ + int sock; + int sockopt = 1; + struct sockaddr_in name; + + debug(D_LISTENER, "IPv4 creating new listening socket on port %d", port); + + sock = socket(AF_INET, SOCK_STREAM, 0); + if(sock < 0) { + error("IPv4 socket() failed."); + return -1; + } + + /* avoid "address already in use" */ + setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, (void*)&sockopt, sizeof(sockopt)); + + memset(&name, 0, sizeof(struct sockaddr_in)); + name.sin_family = AF_INET; + name.sin_port = htons (port); + name.sin_addr.s_addr = htonl (INADDR_ANY); + + if(bind (sock, (struct sockaddr *) &name, sizeof (name)) < 0) { + close(sock); + error("IPv4 bind() failed."); + return -1; + } + + if(listen(sock, listen_backlog) < 0) { + close(sock); + fatal("IPv4 listen() failed."); + return -1; + } + + debug(D_LISTENER, "IPv4 listening port %d created", port); + return sock; +} + +int create_listen_socket6(int port, int listen_backlog) +{ + int sock = -1; + int sockopt = 1; + struct sockaddr_in6 name; + + debug(D_LISTENER, "IPv6 creating new listening socket on port %d", port); + + sock = socket(AF_INET6, SOCK_STREAM, 0); + if (sock < 0) { + error("IPv6 socket() failed."); + return -1; + } + + /* avoid "address already in use" */ + setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, (void*)&sockopt, sizeof(sockopt)); + + memset(&name, 0, sizeof(struct sockaddr_in6)); + name.sin6_family = AF_INET6; + name.sin6_port = htons ((uint16_t) port); + name.sin6_addr = in6addr_any; + name.sin6_scope_id = 0; + + if (bind (sock, (struct sockaddr *) &name, sizeof (name)) < 0) { + close(sock); + error("IPv6 bind() failed."); + return -1; + } + + if (listen(sock, listen_backlog) < 0) { + close(sock); + fatal("IPv6 listen() failed."); + return -1; + } + + debug(D_LISTENER, "IPv6 listening port %d created", port); + return sock; +} + + +// -------------------------------------------------------------------------------------- +// the main socket listener + +// 1. it accepts new incoming requests on our port +// 2. creates a new web_client for each connection received +// 3. spawns a new pthread to serve the client (this is optimal for keep-alive clients) +// 4. cleans up old web_clients that their pthreads have been exited + +void *socket_listen_main(void *ptr) +{ + if(ptr) { ; } + + info("WEB SERVER thread created with task id %d", gettid()); + + struct web_client *w; + struct timeval tv; + int retval; + + if(ptr) { ; } + + if(pthread_setcanceltype(PTHREAD_CANCEL_DEFERRED, NULL) != 0) + error("Cannot set pthread cancel type to DEFERRED."); + + if(pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, NULL) != 0) + error("Cannot set pthread cancel state to ENABLE."); + + web_client_timeout = (int) config_get_number("global", "disconnect idle web clients after seconds", DEFAULT_DISCONNECT_IDLE_WEB_CLIENTS_AFTER_SECONDS); + web_enable_gzip = config_get_boolean("global", "enable web responses gzip compression", web_enable_gzip); + + if(listen_fd < 0) fatal("LISTENER: Listen socket is not ready."); + + fd_set ifds, ofds, efds; + int fdmax = listen_fd; + + FD_ZERO (&ifds); + FD_ZERO (&ofds); + FD_ZERO (&efds); + + for(;;) { + tv.tv_sec = 0; + tv.tv_usec = 200000; + + if(listen_fd >= 0) { + FD_SET(listen_fd, &ifds); + FD_SET(listen_fd, &efds); + } + + // debug(D_WEB_CLIENT, "LISTENER: Waiting..."); + retval = select(fdmax+1, &ifds, &ofds, &efds, &tv); + + if(retval == -1) { + error("LISTENER: select() failed."); + continue; + } + else if(retval) { + // check for new incoming connections + if(FD_ISSET(listen_fd, &ifds)) { + w = web_client_create(listen_fd); + if(unlikely(!w)) { + // no need for error log - web_client_create already logged the error + continue; + } + + if(pthread_create(&w->thread, NULL, web_client_main, w) != 0) { + error("%llu: failed to create new thread for web client."); + w->obsolete = 1; + } + else if(pthread_detach(w->thread) != 0) { + error("%llu: Cannot request detach of newly created web client thread.", w->id); + w->obsolete = 1; + } + } + else debug(D_WEB_CLIENT, "LISTENER: select() didn't do anything."); + + } + //else { + // debug(D_WEB_CLIENT, "LISTENER: select() timeout."); + //} + + // cleanup unused clients + for(w = web_clients; w ; w = w?w->next:NULL) { + if(w->obsolete) { + debug(D_WEB_CLIENT, "%llu: Removing client.", w->id); + // pthread_join(w->thread, NULL); + w = web_client_free(w); + log_allocations(); + } + } + } + + error("LISTENER: exit!"); + + if(listen_fd >= 0) close(listen_fd); + exit(2); + + return NULL; +} + diff --git a/src/web_server.h b/src/web_server.h new file mode 100755 index 000000000..0d1e2da92 --- /dev/null +++ b/src/web_server.h @@ -0,0 +1,20 @@ +#ifndef NETDATA_WEB_SERVER_H +#define NETDATA_WEB_SERVER_H 1 + +#define WEB_PATH_FILE "file" +#define WEB_PATH_DATA "data" +#define WEB_PATH_DATASOURCE "datasource" +#define WEB_PATH_GRAPH "graph" + +#define LISTEN_PORT 19999 +#define LISTEN_BACKLOG 100 + +extern int listen_backlog; +extern int listen_fd; +extern int listen_port; + +extern int create_listen_socket4(int port, int listen_backlog); +extern int create_listen_socket6(int port, int listen_backlog); +extern void *socket_listen_main(void *ptr); + +#endif /* NETDATA_WEB_SERVER_H */ |