diff options
Diffstat (limited to '')
-rw-r--r-- | libnetdata/libnetdata.c | 2097 |
1 files changed, 2097 insertions, 0 deletions
diff --git a/libnetdata/libnetdata.c b/libnetdata/libnetdata.c new file mode 100644 index 00000000..37319a88 --- /dev/null +++ b/libnetdata/libnetdata.c @@ -0,0 +1,2097 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "libnetdata.h" + +#ifdef __APPLE__ +#define INHERIT_NONE 0 +#endif /* __APPLE__ */ +#if defined(__FreeBSD__) || defined(__APPLE__) +# define O_NOATIME 0 +# define MADV_DONTFORK INHERIT_NONE +#endif /* __FreeBSD__ || __APPLE__*/ + +struct rlimit rlimit_nofile = { .rlim_cur = 1024, .rlim_max = 1024 }; + +#ifdef MADV_MERGEABLE +int enable_ksm = 1; +#else +int enable_ksm = 0; +#endif + +volatile sig_atomic_t netdata_exit = 0; +const char *program_version = VERSION; + +#define MAX_JUDY_SIZE_TO_ARAL 24 +static bool judy_sizes_config[MAX_JUDY_SIZE_TO_ARAL + 1] = { + [3] = true, + [4] = true, + [5] = true, + [6] = true, + [7] = true, + [8] = true, + [10] = true, + [11] = true, + [15] = true, + [23] = true, +}; +static ARAL *judy_sizes_aral[MAX_JUDY_SIZE_TO_ARAL + 1] = {}; + +struct aral_statistics judy_sizes_aral_statistics = {}; + +void aral_judy_init(void) { + for(size_t Words = 0; Words <= MAX_JUDY_SIZE_TO_ARAL; Words++) + if(judy_sizes_config[Words]) { + char buf[30+1]; + snprintfz(buf, sizeof(buf) - 1, "judy-%zu", Words * sizeof(Word_t)); + judy_sizes_aral[Words] = aral_create( + buf, + Words * sizeof(Word_t), + 0, + 65536, + &judy_sizes_aral_statistics, + NULL, NULL, false, false); + } +} + +size_t judy_aral_overhead(void) { + return aral_overhead_from_stats(&judy_sizes_aral_statistics); +} + +size_t judy_aral_structures(void) { + return aral_structures_from_stats(&judy_sizes_aral_statistics); +} + +static ARAL *judy_size_aral(Word_t Words) { + if(Words <= MAX_JUDY_SIZE_TO_ARAL && judy_sizes_aral[Words]) + return judy_sizes_aral[Words]; + + return NULL; +} + +inline Word_t JudyMalloc(Word_t Words) { + Word_t Addr; + + ARAL *ar = judy_size_aral(Words); + if(ar) + Addr = (Word_t) aral_mallocz(ar); + else + Addr = (Word_t) mallocz(Words * sizeof(Word_t)); + + return(Addr); +} + +inline void JudyFree(void * PWord, Word_t Words) { + ARAL *ar = judy_size_aral(Words); + if(ar) + aral_freez(ar, PWord); + else + freez(PWord); +} + +Word_t JudyMallocVirtual(Word_t Words) { + return JudyMalloc(Words); +} + +void JudyFreeVirtual(void * PWord, Word_t Words) { + JudyFree(PWord, Words); +} + +// ---------------------------------------------------------------------------- +// memory allocation functions that handle failures + +// although netdata does not use memory allocations too often (netdata tries to +// maintain its memory footprint stable during runtime, i.e. all buffers are +// allocated during initialization and are adapted to current use throughout +// its lifetime), these can be used to override the default system allocation +// routines. + +#ifdef NETDATA_TRACE_ALLOCATIONS +#warning NETDATA_TRACE_ALLOCATIONS ENABLED +#include "Judy.h" + +#if defined(HAVE_DLSYM) && defined(ENABLE_DLSYM) +#include <dlfcn.h> + +typedef void (*libc_function_t)(void); + +static void *malloc_first_run(size_t size); +static void *(*libc_malloc)(size_t) = malloc_first_run; + +static void *calloc_first_run(size_t n, size_t size); +static void *(*libc_calloc)(size_t, size_t) = calloc_first_run; + +static void *realloc_first_run(void *ptr, size_t size); +static void *(*libc_realloc)(void *, size_t) = realloc_first_run; + +static void free_first_run(void *ptr); +static void (*libc_free)(void *) = free_first_run; + +static char *strdup_first_run(const char *s); +static char *(*libc_strdup)(const char *) = strdup_first_run; + +static size_t malloc_usable_size_first_run(void *ptr); +#ifdef HAVE_MALLOC_USABLE_SIZE +static size_t (*libc_malloc_usable_size)(void *) = malloc_usable_size_first_run; +#else +static size_t (*libc_malloc_usable_size)(void *) = NULL; +#endif + +static void link_system_library_function(libc_function_t *func_pptr, const char *name, bool required) { + *func_pptr = dlsym(RTLD_NEXT, name); + if(!*func_pptr && required) { + fprintf(stderr, "FATAL: Cannot find system's %s() function.\n", name); + abort(); + } +} + +static void *malloc_first_run(size_t size) { + link_system_library_function((libc_function_t *) &libc_malloc, "malloc", true); + return libc_malloc(size); +} + +static void *calloc_first_run(size_t n, size_t size) { + link_system_library_function((libc_function_t *) &libc_calloc, "calloc", true); + return libc_calloc(n, size); +} + +static void *realloc_first_run(void *ptr, size_t size) { + link_system_library_function((libc_function_t *) &libc_realloc, "realloc", true); + return libc_realloc(ptr, size); +} + +static void free_first_run(void *ptr) { + link_system_library_function((libc_function_t *) &libc_free, "free", true); + libc_free(ptr); +} + +static char *strdup_first_run(const char *s) { + link_system_library_function((libc_function_t *) &libc_strdup, "strdup", true); + return libc_strdup(s); +} + +static size_t malloc_usable_size_first_run(void *ptr) { + link_system_library_function((libc_function_t *) &libc_malloc_usable_size, "malloc_usable_size", false); + + if(libc_malloc_usable_size) + return libc_malloc_usable_size(ptr); + else + return 0; +} + +void *malloc(size_t size) { + return mallocz(size); +} + +void *calloc(size_t n, size_t size) { + return callocz(n, size); +} + +void *realloc(void *ptr, size_t size) { + return reallocz(ptr, size); +} + +void *reallocarray(void *ptr, size_t n, size_t size) { + return reallocz(ptr, n * size); +} + +void free(void *ptr) { + freez(ptr); +} + +char *strdup(const char *s) { + return strdupz(s); +} + +size_t malloc_usable_size(void *ptr) { + return mallocz_usable_size(ptr); +} +#else // !HAVE_DLSYM + +static void *(*libc_malloc)(size_t) = malloc; +static void *(*libc_calloc)(size_t, size_t) = calloc; +static void *(*libc_realloc)(void *, size_t) = realloc; +static void (*libc_free)(void *) = free; + +#ifdef HAVE_MALLOC_USABLE_SIZE +static size_t (*libc_malloc_usable_size)(void *) = malloc_usable_size; +#else +static size_t (*libc_malloc_usable_size)(void *) = NULL; +#endif + +#endif // HAVE_DLSYM + + +void posix_memfree(void *ptr) { + libc_free(ptr); +} + +struct malloc_header_signature { + uint32_t magic; + uint32_t size; + struct malloc_trace *trace; +}; + +struct malloc_header { + struct malloc_header_signature signature; + uint8_t padding[(sizeof(struct malloc_header_signature) % MALLOC_ALIGNMENT) ? MALLOC_ALIGNMENT - (sizeof(struct malloc_header_signature) % MALLOC_ALIGNMENT) : 0]; + uint8_t data[]; +}; + +static size_t malloc_header_size = sizeof(struct malloc_header); + +int malloc_trace_compare(void *A, void *B) { + struct malloc_trace *a = A; + struct malloc_trace *b = B; + return strcmp(a->function, b->function); +} + +static avl_tree_lock malloc_trace_index = { + .avl_tree = { + .root = NULL, + .compar = malloc_trace_compare}, + .rwlock = AVL_LOCK_INITIALIZER +}; + +int malloc_trace_walkthrough(int (*callback)(void *item, void *data), void *data) { + return avl_traverse_lock(&malloc_trace_index, callback, data); +} + +NEVERNULL WARNUNUSED +static struct malloc_trace *malloc_trace_find_or_create(const char *file, const char *function, size_t line) { + struct malloc_trace tmp = { + .line = line, + .function = function, + .file = file, + }; + + struct malloc_trace *t = (struct malloc_trace *)avl_search_lock(&malloc_trace_index, (avl_t *)&tmp); + if(!t) { + t = libc_calloc(1, sizeof(struct malloc_trace)); + if(!t) fatal("No memory"); + t->line = line; + t->function = function; + t->file = file; + + struct malloc_trace *t2 = (struct malloc_trace *)avl_insert_lock(&malloc_trace_index, (avl_t *)t); + if(t2 != t) + free(t); + + t = t2; + } + + if(!t) + fatal("Cannot insert to AVL"); + + return t; +} + +void malloc_trace_mmap(size_t size) { + struct malloc_trace *p = malloc_trace_find_or_create("unknown", "netdata_mmap", 1); + size_t_atomic_count(add, p->mmap_calls, 1); + size_t_atomic_count(add, p->allocations, 1); + size_t_atomic_bytes(add, p->bytes, size); +} + +void malloc_trace_munmap(size_t size) { + struct malloc_trace *p = malloc_trace_find_or_create("unknown", "netdata_mmap", 1); + size_t_atomic_count(add, p->munmap_calls, 1); + size_t_atomic_count(sub, p->allocations, 1); + size_t_atomic_bytes(sub, p->bytes, size); +} + +void *mallocz_int(size_t size, const char *file, const char *function, size_t line) { + struct malloc_trace *p = malloc_trace_find_or_create(file, function, line); + + size_t_atomic_count(add, p->malloc_calls, 1); + size_t_atomic_count(add, p->allocations, 1); + size_t_atomic_bytes(add, p->bytes, size); + + struct malloc_header *t = (struct malloc_header *)libc_malloc(malloc_header_size + size); + if (unlikely(!t)) fatal("mallocz() cannot allocate %zu bytes of memory (%zu with header).", size, malloc_header_size + size); + t->signature.magic = 0x0BADCAFE; + t->signature.trace = p; + t->signature.size = size; + +#ifdef NETDATA_INTERNAL_CHECKS + for(ssize_t i = 0; i < (ssize_t)sizeof(t->padding) ;i++) // signed to avoid compiler warning when zero-padded + t->padding[i] = 0xFF; +#endif + + return (void *)&t->data; +} + +void *callocz_int(size_t nmemb, size_t size, const char *file, const char *function, size_t line) { + struct malloc_trace *p = malloc_trace_find_or_create(file, function, line); + size = nmemb * size; + + size_t_atomic_count(add, p->calloc_calls, 1); + size_t_atomic_count(add, p->allocations, 1); + size_t_atomic_bytes(add, p->bytes, size); + + struct malloc_header *t = (struct malloc_header *)libc_calloc(1, malloc_header_size + size); + if (unlikely(!t)) fatal("mallocz() cannot allocate %zu bytes of memory (%zu with header).", size, malloc_header_size + size); + t->signature.magic = 0x0BADCAFE; + t->signature.trace = p; + t->signature.size = size; + +#ifdef NETDATA_INTERNAL_CHECKS + for(ssize_t i = 0; i < (ssize_t)sizeof(t->padding) ;i++) // signed to avoid compiler warning when zero-padded + t->padding[i] = 0xFF; +#endif + + return &t->data; +} + +char *strdupz_int(const char *s, const char *file, const char *function, size_t line) { + struct malloc_trace *p = malloc_trace_find_or_create(file, function, line); + size_t size = strlen(s) + 1; + + size_t_atomic_count(add, p->strdup_calls, 1); + size_t_atomic_count(add, p->allocations, 1); + size_t_atomic_bytes(add, p->bytes, size); + + struct malloc_header *t = (struct malloc_header *)libc_malloc(malloc_header_size + size); + if (unlikely(!t)) fatal("strdupz() cannot allocate %zu bytes of memory (%zu with header).", size, malloc_header_size + size); + t->signature.magic = 0x0BADCAFE; + t->signature.trace = p; + t->signature.size = size; + +#ifdef NETDATA_INTERNAL_CHECKS + for(ssize_t i = 0; i < (ssize_t)sizeof(t->padding) ;i++) // signed to avoid compiler warning when zero-padded + t->padding[i] = 0xFF; +#endif + + memcpy(&t->data, s, size); + return (char *)&t->data; +} + +static struct malloc_header *malloc_get_header(void *ptr, const char *caller, const char *file, const char *function, size_t line) { + uint8_t *ret = (uint8_t *)ptr - malloc_header_size; + struct malloc_header *t = (struct malloc_header *)ret; + + if(t->signature.magic != 0x0BADCAFE) { + netdata_log_error("pointer %p is not our pointer (called %s() from %zu@%s, %s()).", ptr, caller, line, file, function); + return NULL; + } + + return t; +} + +void *reallocz_int(void *ptr, size_t size, const char *file, const char *function, size_t line) { + if(!ptr) return mallocz_int(size, file, function, line); + + struct malloc_header *t = malloc_get_header(ptr, __FUNCTION__, file, function, line); + if(!t) + return libc_realloc(ptr, size); + + if(t->signature.size == size) return ptr; + size_t_atomic_count(add, t->signature.trace->free_calls, 1); + size_t_atomic_count(sub, t->signature.trace->allocations, 1); + size_t_atomic_bytes(sub, t->signature.trace->bytes, t->signature.size); + + struct malloc_trace *p = malloc_trace_find_or_create(file, function, line); + size_t_atomic_count(add, p->realloc_calls, 1); + size_t_atomic_count(add, p->allocations, 1); + size_t_atomic_bytes(add, p->bytes, size); + + t = (struct malloc_header *)libc_realloc(t, malloc_header_size + size); + if (unlikely(!t)) fatal("reallocz() cannot allocate %zu bytes of memory (%zu with header).", size, malloc_header_size + size); + t->signature.magic = 0x0BADCAFE; + t->signature.trace = p; + t->signature.size = size; + +#ifdef NETDATA_INTERNAL_CHECKS + for(ssize_t i = 0; i < (ssize_t)sizeof(t->padding) ;i++) // signed to avoid compiler warning when zero-padded + t->padding[i] = 0xFF; +#endif + + return (void *)&t->data; +} + +size_t mallocz_usable_size_int(void *ptr, const char *file, const char *function, size_t line) { + if(unlikely(!ptr)) return 0; + + struct malloc_header *t = malloc_get_header(ptr, __FUNCTION__, file, function, line); + if(!t) { + if(libc_malloc_usable_size) + return libc_malloc_usable_size(ptr); + else + return 0; + } + + return t->signature.size; +} + +void freez_int(void *ptr, const char *file, const char *function, size_t line) { + if(unlikely(!ptr)) return; + + struct malloc_header *t = malloc_get_header(ptr, __FUNCTION__, file, function, line); + if(!t) { + libc_free(ptr); + return; + } + + size_t_atomic_count(add, t->signature.trace->free_calls, 1); + size_t_atomic_count(sub, t->signature.trace->allocations, 1); + size_t_atomic_bytes(sub, t->signature.trace->bytes, t->signature.size); + +#ifdef NETDATA_INTERNAL_CHECKS + // it should crash if it is used after freeing it + memset(t, 0, malloc_header_size + t->signature.size); +#endif + + libc_free(t); +} +#else + +char *strdupz(const char *s) { + char *t = strdup(s); + if (unlikely(!t)) fatal("Cannot strdup() string '%s'", s); + return t; +} + +// If ptr is NULL, no operation is performed. +void freez(void *ptr) { + free(ptr); +} + +void *mallocz(size_t size) { + void *p = malloc(size); + if (unlikely(!p)) fatal("Cannot allocate %zu bytes of memory.", size); + return p; +} + +void *callocz(size_t nmemb, size_t size) { + void *p = calloc(nmemb, size); + if (unlikely(!p)) fatal("Cannot allocate %zu bytes of memory.", nmemb * size); + return p; +} + +void *reallocz(void *ptr, size_t size) { + void *p = realloc(ptr, size); + if (unlikely(!p)) fatal("Cannot re-allocate memory to %zu bytes.", size); + return p; +} + +void posix_memfree(void *ptr) { + free(ptr); +} + +#endif + +// -------------------------------------------------------------------------------------------------------------------- + +void json_escape_string(char *dst, const char *src, size_t size) { + const char *t; + char *d = dst, *e = &dst[size - 1]; + + for(t = src; *t && d < e ;t++) { + if(unlikely(*t == '\\' || *t == '"')) { + if(unlikely(d + 1 >= e)) break; + *d++ = '\\'; + } + *d++ = *t; + } + + *d = '\0'; +} + +void json_fix_string(char *s) { + unsigned char c; + while((c = (unsigned char)*s)) { + if(unlikely(c == '\\')) + *s++ = '/'; + else if(unlikely(c == '"')) + *s++ = '\''; + else if(unlikely(isspace(c) || iscntrl(c))) + *s++ = ' '; + else if(unlikely(!isprint(c) || c > 127)) + *s++ = '_'; + else + s++; + } +} + +unsigned char netdata_map_chart_names[256] = { + [0] = '\0', // + [1] = '_', // + [2] = '_', // + [3] = '_', // + [4] = '_', // + [5] = '_', // + [6] = '_', // + [7] = '_', // + [8] = '_', // + [9] = '_', // + [10] = '_', // + [11] = '_', // + [12] = '_', // + [13] = '_', // + [14] = '_', // + [15] = '_', // + [16] = '_', // + [17] = '_', // + [18] = '_', // + [19] = '_', // + [20] = '_', // + [21] = '_', // + [22] = '_', // + [23] = '_', // + [24] = '_', // + [25] = '_', // + [26] = '_', // + [27] = '_', // + [28] = '_', // + [29] = '_', // + [30] = '_', // + [31] = '_', // + [32] = '_', // + [33] = '_', // ! + [34] = '_', // " + [35] = '_', // # + [36] = '_', // $ + [37] = '_', // % + [38] = '_', // & + [39] = '_', // ' + [40] = '_', // ( + [41] = '_', // ) + [42] = '_', // * + [43] = '_', // + + [44] = '.', // , + [45] = '-', // - + [46] = '.', // . + [47] = '/', // / + [48] = '0', // 0 + [49] = '1', // 1 + [50] = '2', // 2 + [51] = '3', // 3 + [52] = '4', // 4 + [53] = '5', // 5 + [54] = '6', // 6 + [55] = '7', // 7 + [56] = '8', // 8 + [57] = '9', // 9 + [58] = '_', // : + [59] = '_', // ; + [60] = '_', // < + [61] = '_', // = + [62] = '_', // > + [63] = '_', // ? + [64] = '_', // @ + [65] = 'a', // A + [66] = 'b', // B + [67] = 'c', // C + [68] = 'd', // D + [69] = 'e', // E + [70] = 'f', // F + [71] = 'g', // G + [72] = 'h', // H + [73] = 'i', // I + [74] = 'j', // J + [75] = 'k', // K + [76] = 'l', // L + [77] = 'm', // M + [78] = 'n', // N + [79] = 'o', // O + [80] = 'p', // P + [81] = 'q', // Q + [82] = 'r', // R + [83] = 's', // S + [84] = 't', // T + [85] = 'u', // U + [86] = 'v', // V + [87] = 'w', // W + [88] = 'x', // X + [89] = 'y', // Y + [90] = 'z', // Z + [91] = '_', // [ + [92] = '/', // backslash + [93] = '_', // ] + [94] = '_', // ^ + [95] = '_', // _ + [96] = '_', // ` + [97] = 'a', // a + [98] = 'b', // b + [99] = 'c', // c + [100] = 'd', // d + [101] = 'e', // e + [102] = 'f', // f + [103] = 'g', // g + [104] = 'h', // h + [105] = 'i', // i + [106] = 'j', // j + [107] = 'k', // k + [108] = 'l', // l + [109] = 'm', // m + [110] = 'n', // n + [111] = 'o', // o + [112] = 'p', // p + [113] = 'q', // q + [114] = 'r', // r + [115] = 's', // s + [116] = 't', // t + [117] = 'u', // u + [118] = 'v', // v + [119] = 'w', // w + [120] = 'x', // x + [121] = 'y', // y + [122] = 'z', // z + [123] = '_', // { + [124] = '_', // | + [125] = '_', // } + [126] = '_', // ~ + [127] = '_', // + [128] = '_', // + [129] = '_', // + [130] = '_', // + [131] = '_', // + [132] = '_', // + [133] = '_', // + [134] = '_', // + [135] = '_', // + [136] = '_', // + [137] = '_', // + [138] = '_', // + [139] = '_', // + [140] = '_', // + [141] = '_', // + [142] = '_', // + [143] = '_', // + [144] = '_', // + [145] = '_', // + [146] = '_', // + [147] = '_', // + [148] = '_', // + [149] = '_', // + [150] = '_', // + [151] = '_', // + [152] = '_', // + [153] = '_', // + [154] = '_', // + [155] = '_', // + [156] = '_', // + [157] = '_', // + [158] = '_', // + [159] = '_', // + [160] = '_', // + [161] = '_', // + [162] = '_', // + [163] = '_', // + [164] = '_', // + [165] = '_', // + [166] = '_', // + [167] = '_', // + [168] = '_', // + [169] = '_', // + [170] = '_', // + [171] = '_', // + [172] = '_', // + [173] = '_', // + [174] = '_', // + [175] = '_', // + [176] = '_', // + [177] = '_', // + [178] = '_', // + [179] = '_', // + [180] = '_', // + [181] = '_', // + [182] = '_', // + [183] = '_', // + [184] = '_', // + [185] = '_', // + [186] = '_', // + [187] = '_', // + [188] = '_', // + [189] = '_', // + [190] = '_', // + [191] = '_', // + [192] = '_', // + [193] = '_', // + [194] = '_', // + [195] = '_', // + [196] = '_', // + [197] = '_', // + [198] = '_', // + [199] = '_', // + [200] = '_', // + [201] = '_', // + [202] = '_', // + [203] = '_', // + [204] = '_', // + [205] = '_', // + [206] = '_', // + [207] = '_', // + [208] = '_', // + [209] = '_', // + [210] = '_', // + [211] = '_', // + [212] = '_', // + [213] = '_', // + [214] = '_', // + [215] = '_', // + [216] = '_', // + [217] = '_', // + [218] = '_', // + [219] = '_', // + [220] = '_', // + [221] = '_', // + [222] = '_', // + [223] = '_', // + [224] = '_', // + [225] = '_', // + [226] = '_', // + [227] = '_', // + [228] = '_', // + [229] = '_', // + [230] = '_', // + [231] = '_', // + [232] = '_', // + [233] = '_', // + [234] = '_', // + [235] = '_', // + [236] = '_', // + [237] = '_', // + [238] = '_', // + [239] = '_', // + [240] = '_', // + [241] = '_', // + [242] = '_', // + [243] = '_', // + [244] = '_', // + [245] = '_', // + [246] = '_', // + [247] = '_', // + [248] = '_', // + [249] = '_', // + [250] = '_', // + [251] = '_', // + [252] = '_', // + [253] = '_', // + [254] = '_', // + [255] = '_' // +}; + +// make sure the supplied string +// is good for a netdata chart/dimension ID/NAME +void netdata_fix_chart_name(char *s) { + while ((*s = netdata_map_chart_names[(unsigned char) *s])) s++; +} + +unsigned char netdata_map_chart_ids[256] = { + [0] = '\0', // + [1] = '_', // + [2] = '_', // + [3] = '_', // + [4] = '_', // + [5] = '_', // + [6] = '_', // + [7] = '_', // + [8] = '_', // + [9] = '_', // + [10] = '_', // + [11] = '_', // + [12] = '_', // + [13] = '_', // + [14] = '_', // + [15] = '_', // + [16] = '_', // + [17] = '_', // + [18] = '_', // + [19] = '_', // + [20] = '_', // + [21] = '_', // + [22] = '_', // + [23] = '_', // + [24] = '_', // + [25] = '_', // + [26] = '_', // + [27] = '_', // + [28] = '_', // + [29] = '_', // + [30] = '_', // + [31] = '_', // + [32] = '_', // + [33] = '_', // ! + [34] = '_', // " + [35] = '_', // # + [36] = '_', // $ + [37] = '_', // % + [38] = '_', // & + [39] = '_', // ' + [40] = '_', // ( + [41] = '_', // ) + [42] = '_', // * + [43] = '_', // + + [44] = '.', // , + [45] = '-', // - + [46] = '.', // . + [47] = '_', // / + [48] = '0', // 0 + [49] = '1', // 1 + [50] = '2', // 2 + [51] = '3', // 3 + [52] = '4', // 4 + [53] = '5', // 5 + [54] = '6', // 6 + [55] = '7', // 7 + [56] = '8', // 8 + [57] = '9', // 9 + [58] = '_', // : + [59] = '_', // ; + [60] = '_', // < + [61] = '_', // = + [62] = '_', // > + [63] = '_', // ? + [64] = '_', // @ + [65] = 'a', // A + [66] = 'b', // B + [67] = 'c', // C + [68] = 'd', // D + [69] = 'e', // E + [70] = 'f', // F + [71] = 'g', // G + [72] = 'h', // H + [73] = 'i', // I + [74] = 'j', // J + [75] = 'k', // K + [76] = 'l', // L + [77] = 'm', // M + [78] = 'n', // N + [79] = 'o', // O + [80] = 'p', // P + [81] = 'q', // Q + [82] = 'r', // R + [83] = 's', // S + [84] = 't', // T + [85] = 'u', // U + [86] = 'v', // V + [87] = 'w', // W + [88] = 'x', // X + [89] = 'y', // Y + [90] = 'z', // Z + [91] = '_', // [ + [92] = '_', // backslash + [93] = '_', // ] + [94] = '_', // ^ + [95] = '_', // _ + [96] = '_', // ` + [97] = 'a', // a + [98] = 'b', // b + [99] = 'c', // c + [100] = 'd', // d + [101] = 'e', // e + [102] = 'f', // f + [103] = 'g', // g + [104] = 'h', // h + [105] = 'i', // i + [106] = 'j', // j + [107] = 'k', // k + [108] = 'l', // l + [109] = 'm', // m + [110] = 'n', // n + [111] = 'o', // o + [112] = 'p', // p + [113] = 'q', // q + [114] = 'r', // r + [115] = 's', // s + [116] = 't', // t + [117] = 'u', // u + [118] = 'v', // v + [119] = 'w', // w + [120] = 'x', // x + [121] = 'y', // y + [122] = 'z', // z + [123] = '_', // { + [124] = '_', // | + [125] = '_', // } + [126] = '_', // ~ + [127] = '_', // + [128] = '_', // + [129] = '_', // + [130] = '_', // + [131] = '_', // + [132] = '_', // + [133] = '_', // + [134] = '_', // + [135] = '_', // + [136] = '_', // + [137] = '_', // + [138] = '_', // + [139] = '_', // + [140] = '_', // + [141] = '_', // + [142] = '_', // + [143] = '_', // + [144] = '_', // + [145] = '_', // + [146] = '_', // + [147] = '_', // + [148] = '_', // + [149] = '_', // + [150] = '_', // + [151] = '_', // + [152] = '_', // + [153] = '_', // + [154] = '_', // + [155] = '_', // + [156] = '_', // + [157] = '_', // + [158] = '_', // + [159] = '_', // + [160] = '_', // + [161] = '_', // + [162] = '_', // + [163] = '_', // + [164] = '_', // + [165] = '_', // + [166] = '_', // + [167] = '_', // + [168] = '_', // + [169] = '_', // + [170] = '_', // + [171] = '_', // + [172] = '_', // + [173] = '_', // + [174] = '_', // + [175] = '_', // + [176] = '_', // + [177] = '_', // + [178] = '_', // + [179] = '_', // + [180] = '_', // + [181] = '_', // + [182] = '_', // + [183] = '_', // + [184] = '_', // + [185] = '_', // + [186] = '_', // + [187] = '_', // + [188] = '_', // + [189] = '_', // + [190] = '_', // + [191] = '_', // + [192] = '_', // + [193] = '_', // + [194] = '_', // + [195] = '_', // + [196] = '_', // + [197] = '_', // + [198] = '_', // + [199] = '_', // + [200] = '_', // + [201] = '_', // + [202] = '_', // + [203] = '_', // + [204] = '_', // + [205] = '_', // + [206] = '_', // + [207] = '_', // + [208] = '_', // + [209] = '_', // + [210] = '_', // + [211] = '_', // + [212] = '_', // + [213] = '_', // + [214] = '_', // + [215] = '_', // + [216] = '_', // + [217] = '_', // + [218] = '_', // + [219] = '_', // + [220] = '_', // + [221] = '_', // + [222] = '_', // + [223] = '_', // + [224] = '_', // + [225] = '_', // + [226] = '_', // + [227] = '_', // + [228] = '_', // + [229] = '_', // + [230] = '_', // + [231] = '_', // + [232] = '_', // + [233] = '_', // + [234] = '_', // + [235] = '_', // + [236] = '_', // + [237] = '_', // + [238] = '_', // + [239] = '_', // + [240] = '_', // + [241] = '_', // + [242] = '_', // + [243] = '_', // + [244] = '_', // + [245] = '_', // + [246] = '_', // + [247] = '_', // + [248] = '_', // + [249] = '_', // + [250] = '_', // + [251] = '_', // + [252] = '_', // + [253] = '_', // + [254] = '_', // + [255] = '_' // +}; + +// make sure the supplied string +// is good for a netdata chart/dimension ID/NAME +void netdata_fix_chart_id(char *s) { + while ((*s = netdata_map_chart_ids[(unsigned char) *s])) s++; +} + +static int memory_file_open(const char *filename, size_t size) { + // netdata_log_info("memory_file_open('%s', %zu", filename, size); + + int fd = open(filename, O_RDWR | O_CREAT | O_NOATIME, 0664); + if (fd != -1) { + if (lseek(fd, size, SEEK_SET) == (off_t) size) { + if (write(fd, "", 1) == 1) { + if (ftruncate(fd, size)) + netdata_log_error("Cannot truncate file '%s' to size %zu. Will use the larger file.", filename, size); + } + else + netdata_log_error("Cannot write to file '%s' at position %zu.", filename, size); + } + else + netdata_log_error("Cannot seek file '%s' to size %zu.", filename, size); + } + else + netdata_log_error("Cannot create/open file '%s'.", filename); + + return fd; +} + +inline int madvise_sequential(void *mem, size_t len) { + static int logger = 1; + int ret = madvise(mem, len, MADV_SEQUENTIAL); + + if (ret != 0 && logger-- > 0) + netdata_log_error("madvise(MADV_SEQUENTIAL) failed."); + return ret; +} + +inline int madvise_random(void *mem, size_t len) { + static int logger = 1; + int ret = madvise(mem, len, MADV_RANDOM); + + if (ret != 0 && logger-- > 0) + netdata_log_error("madvise(MADV_RANDOM) failed."); + return ret; +} + +inline int madvise_dontfork(void *mem, size_t len) { + static int logger = 1; + int ret = madvise(mem, len, MADV_DONTFORK); + + if (ret != 0 && logger-- > 0) + netdata_log_error("madvise(MADV_DONTFORK) failed."); + return ret; +} + +inline int madvise_willneed(void *mem, size_t len) { + static int logger = 1; + int ret = madvise(mem, len, MADV_WILLNEED); + + if (ret != 0 && logger-- > 0) + netdata_log_error("madvise(MADV_WILLNEED) failed."); + return ret; +} + +inline int madvise_dontneed(void *mem, size_t len) { + static int logger = 1; + int ret = madvise(mem, len, MADV_DONTNEED); + + if (ret != 0 && logger-- > 0) + netdata_log_error("madvise(MADV_DONTNEED) failed."); + return ret; +} + +inline int madvise_dontdump(void *mem __maybe_unused, size_t len __maybe_unused) { +#if __linux__ + static int logger = 1; + int ret = madvise(mem, len, MADV_DONTDUMP); + + if (ret != 0 && logger-- > 0) + netdata_log_error("madvise(MADV_DONTDUMP) failed."); + return ret; +#else + return 0; +#endif +} + +inline int madvise_mergeable(void *mem __maybe_unused, size_t len __maybe_unused) { +#ifdef MADV_MERGEABLE + static int logger = 1; + int ret = madvise(mem, len, MADV_MERGEABLE); + + if (ret != 0 && logger-- > 0) + netdata_log_error("madvise(MADV_MERGEABLE) failed."); + return ret; +#else + return 0; +#endif +} + +void *netdata_mmap(const char *filename, size_t size, int flags, int ksm, bool read_only, int *open_fd) +{ + // netdata_log_info("netdata_mmap('%s', %zu", filename, size); + + // MAP_SHARED is used in memory mode map + // MAP_PRIVATE is used in memory mode ram and save + + if(unlikely(!(flags & MAP_SHARED) && !(flags & MAP_PRIVATE))) + fatal("Neither MAP_SHARED or MAP_PRIVATE were given to netdata_mmap()"); + + if(unlikely((flags & MAP_SHARED) && (flags & MAP_PRIVATE))) + fatal("Both MAP_SHARED and MAP_PRIVATE were given to netdata_mmap()"); + + if(unlikely((flags & MAP_SHARED) && (!filename || !*filename))) + fatal("MAP_SHARED requested, without a filename to netdata_mmap()"); + + // don't enable ksm is the global setting is disabled + if(unlikely(!enable_ksm)) ksm = 0; + + // KSM only merges anonymous (private) pages, never pagecache (file) pages + // but MAP_PRIVATE without MAP_ANONYMOUS it fails too, so we need it always + if((flags & MAP_PRIVATE)) flags |= MAP_ANONYMOUS; + + int fd = -1; + void *mem = MAP_FAILED; + + if(filename && *filename) { + // open/create the file to be used + fd = memory_file_open(filename, size); + if(fd == -1) goto cleanup; + } + + int fd_for_mmap = fd; + if(fd != -1 && (flags & MAP_PRIVATE)) { + // this is MAP_PRIVATE allocation + // no need for mmap() to use our fd + // we will copy the file into the memory allocated + fd_for_mmap = -1; + } + + mem = mmap(NULL, size, read_only ? PROT_READ : PROT_READ | PROT_WRITE, flags, fd_for_mmap, 0); + if (mem != MAP_FAILED) { + +#ifdef NETDATA_TRACE_ALLOCATIONS + malloc_trace_mmap(size); +#endif + + // if we have a file open, but we didn't give it to mmap(), + // we have to read the file into the memory block we allocated + if(fd != -1 && fd_for_mmap == -1) { + if (lseek(fd, 0, SEEK_SET) == 0) { + if (read(fd, mem, size) != (ssize_t) size) + netdata_log_info("Cannot read from file '%s'", filename); + } + else netdata_log_info("Cannot seek to beginning of file '%s'.", filename); + } + + // madvise_sequential(mem, size); + madvise_dontfork(mem, size); + madvise_dontdump(mem, size); + // if(flags & MAP_SHARED) madvise_willneed(mem, size); + if(ksm) madvise_mergeable(mem, size); + } + +cleanup: + if(fd != -1) { + if (open_fd) + *open_fd = fd; + else + close(fd); + } + if(mem == MAP_FAILED) return NULL; + errno = 0; + return mem; +} + +int netdata_munmap(void *ptr, size_t size) { +#ifdef NETDATA_TRACE_ALLOCATIONS + malloc_trace_munmap(size); +#endif + return munmap(ptr, size); +} + +int memory_file_save(const char *filename, void *mem, size_t size) { + char tmpfilename[FILENAME_MAX + 1]; + + snprintfz(tmpfilename, FILENAME_MAX, "%s.%ld.tmp", filename, (long) getpid()); + + int fd = open(tmpfilename, O_RDWR | O_CREAT | O_NOATIME, 0664); + if (fd < 0) { + netdata_log_error("Cannot create/open file '%s'.", filename); + return -1; + } + + if (write(fd, mem, size) != (ssize_t) size) { + netdata_log_error("Cannot write to file '%s' %ld bytes.", filename, (long) size); + close(fd); + return -1; + } + + close(fd); + + if (rename(tmpfilename, filename)) { + netdata_log_error("Cannot rename '%s' to '%s'", tmpfilename, filename); + return -1; + } + + return 0; +} + +int fd_is_valid(int fd) { + return fcntl(fd, F_GETFD) != -1 || errno != EBADF; +} + +char *fgets_trim_len(char *buf, size_t buf_size, FILE *fp, size_t *len) { + char *s = fgets(buf, (int)buf_size, fp); + if (!s) return NULL; + + char *t = s; + if (*t != '\0') { + // find the string end + while (*++t != '\0'); + + // trim trailing spaces/newlines/tabs + while (--t > s && *t == '\n') + *t = '\0'; + } + + if (len) + *len = t - s + 1; + + return s; +} + +// vsnprintfz() returns the number of bytes actually written - after possible truncation +int vsnprintfz(char *dst, size_t n, const char *fmt, va_list args) { + if(unlikely(!n)) return 0; + + int size = vsnprintf(dst, n, fmt, args); + dst[n - 1] = '\0'; + + if (unlikely((size_t) size >= n)) size = (int)(n - 1); + + return size; +} + +// snprintfz() returns the number of bytes actually written - after possible truncation +int snprintfz(char *dst, size_t n, const char *fmt, ...) { + va_list args; + + va_start(args, fmt); + int ret = vsnprintfz(dst, n, fmt, args); + va_end(args); + + return ret; +} + +/* +// poor man cycle counting +static unsigned long tsc; +void begin_tsc(void) { + unsigned long a, d; + asm volatile ("cpuid\nrdtsc" : "=a" (a), "=d" (d) : "0" (0) : "ebx", "ecx"); + tsc = ((unsigned long)d << 32) | (unsigned long)a; +} +unsigned long end_tsc(void) { + unsigned long a, d; + asm volatile ("rdtscp" : "=a" (a), "=d" (d) : : "ecx"); + return (((unsigned long)d << 32) | (unsigned long)a) - tsc; +} +*/ + +int recursively_delete_dir(const char *path, const char *reason) { + DIR *dir = opendir(path); + if(!dir) { + netdata_log_error("Cannot read %s directory to be deleted '%s'", reason?reason:"", path); + return -1; + } + + int ret = 0; + struct dirent *de = NULL; + while((de = readdir(dir))) { + if(de->d_type == DT_DIR + && ( + (de->d_name[0] == '.' && de->d_name[1] == '\0') + || (de->d_name[0] == '.' && de->d_name[1] == '.' && de->d_name[2] == '\0') + )) + continue; + + char fullpath[FILENAME_MAX + 1]; + snprintfz(fullpath, FILENAME_MAX, "%s/%s", path, de->d_name); + + if(de->d_type == DT_DIR) { + int r = recursively_delete_dir(fullpath, reason); + if(r > 0) ret += r; + continue; + } + + netdata_log_info("Deleting %s file '%s'", reason?reason:"", fullpath); + if(unlikely(unlink(fullpath) == -1)) + netdata_log_error("Cannot delete %s file '%s'", reason?reason:"", fullpath); + else + ret++; + } + + netdata_log_info("Deleting empty directory '%s'", path); + if(unlikely(rmdir(path) == -1)) + netdata_log_error("Cannot delete empty directory '%s'", path); + else + ret++; + + closedir(dir); + + return ret; +} + +static int is_virtual_filesystem(const char *path, char **reason) { + +#if defined(__APPLE__) || defined(__FreeBSD__) + (void)path; + (void)reason; +#else + struct statfs stat; + // stat.f_fsid.__val[0] is a file system id + // stat.f_fsid.__val[1] is the inode + // so their combination uniquely identifies the file/dir + + if (statfs(path, &stat) == -1) { + if(reason) *reason = "failed to statfs()"; + return -1; + } + + if(stat.f_fsid.__val[0] != 0 || stat.f_fsid.__val[1] != 0) { + errno = EINVAL; + if(reason) *reason = "is not a virtual file system"; + return -1; + } +#endif + + return 0; +} + +int verify_netdata_host_prefix(bool log_msg) { + if(!netdata_configured_host_prefix) + netdata_configured_host_prefix = ""; + + if(!*netdata_configured_host_prefix) + return 0; + + char buffer[FILENAME_MAX + 1]; + char *path = netdata_configured_host_prefix; + char *reason = "unknown reason"; + errno = 0; + + struct stat sb; + if (stat(path, &sb) == -1) { + reason = "failed to stat()"; + goto failed; + } + + if((sb.st_mode & S_IFMT) != S_IFDIR) { + errno = EINVAL; + reason = "is not a directory"; + goto failed; + } + + path = buffer; + snprintfz(path, FILENAME_MAX, "%s/proc", netdata_configured_host_prefix); + if(is_virtual_filesystem(path, &reason) == -1) + goto failed; + + snprintfz(path, FILENAME_MAX, "%s/sys", netdata_configured_host_prefix); + if(is_virtual_filesystem(path, &reason) == -1) + goto failed; + + if (netdata_configured_host_prefix && *netdata_configured_host_prefix) { + if (log_msg) + netdata_log_info("Using host prefix directory '%s'", netdata_configured_host_prefix); + } + + return 0; + +failed: + if (log_msg) + netdata_log_error("Ignoring host prefix '%s': path '%s' %s", netdata_configured_host_prefix, path, reason); + netdata_configured_host_prefix = ""; + return -1; +} + +char *strdupz_path_subpath(const char *path, const char *subpath) { + if(unlikely(!path || !*path)) path = "."; + if(unlikely(!subpath)) subpath = ""; + + // skip trailing slashes in path + size_t len = strlen(path); + while(len > 0 && path[len - 1] == '/') len--; + + // skip leading slashes in subpath + while(subpath[0] == '/') subpath++; + + // if the last character in path is / and (there is a subpath or path is now empty) + // keep the trailing slash in path and remove the additional slash + char *slash = "/"; + if(path[len] == '/' && (*subpath || len == 0)) { + slash = ""; + len++; + } + else if(!*subpath) { + // there is no subpath + // no need for trailing slash + slash = ""; + } + + char buffer[FILENAME_MAX + 1]; + snprintfz(buffer, FILENAME_MAX, "%.*s%s%s", (int)len, path, slash, subpath); + return strdupz(buffer); +} + +int path_is_dir(const char *path, const char *subpath) { + char *s = strdupz_path_subpath(path, subpath); + + size_t max_links = 100; + + int is_dir = 0; + struct stat statbuf; + while(max_links-- && stat(s, &statbuf) == 0) { + if((statbuf.st_mode & S_IFMT) == S_IFDIR) { + is_dir = 1; + break; + } + else if((statbuf.st_mode & S_IFMT) == S_IFLNK) { + char buffer[FILENAME_MAX + 1]; + ssize_t l = readlink(s, buffer, FILENAME_MAX); + if(l > 0) { + buffer[l] = '\0'; + freez(s); + s = strdupz(buffer); + continue; + } + else { + is_dir = 0; + break; + } + } + else { + is_dir = 0; + break; + } + } + + freez(s); + return is_dir; +} + +int path_is_file(const char *path, const char *subpath) { + char *s = strdupz_path_subpath(path, subpath); + + size_t max_links = 100; + + int is_file = 0; + struct stat statbuf; + while(max_links-- && stat(s, &statbuf) == 0) { + if((statbuf.st_mode & S_IFMT) == S_IFREG) { + is_file = 1; + break; + } + else if((statbuf.st_mode & S_IFMT) == S_IFLNK) { + char buffer[FILENAME_MAX + 1]; + ssize_t l = readlink(s, buffer, FILENAME_MAX); + if(l > 0) { + buffer[l] = '\0'; + freez(s); + s = strdupz(buffer); + continue; + } + else { + is_file = 0; + break; + } + } + else { + is_file = 0; + break; + } + } + + freez(s); + return is_file; +} + +void recursive_config_double_dir_load(const char *user_path, const char *stock_path, const char *subpath, int (*callback)(const char *filename, void *data), void *data, size_t depth) { + if(depth > 3) { + netdata_log_error("CONFIG: Max directory depth reached while reading user path '%s', stock path '%s', subpath '%s'", user_path, stock_path, subpath); + return; + } + + char *udir = strdupz_path_subpath(user_path, subpath); + char *sdir = strdupz_path_subpath(stock_path, subpath); + + netdata_log_debug(D_HEALTH, "CONFIG traversing user-config directory '%s', stock config directory '%s'", udir, sdir); + + DIR *dir = opendir(udir); + if (!dir) { + netdata_log_error("CONFIG cannot open user-config directory '%s'.", udir); + } + else { + struct dirent *de = NULL; + while((de = readdir(dir))) { + if(de->d_type == DT_DIR || de->d_type == DT_LNK) { + if( !de->d_name[0] || + (de->d_name[0] == '.' && de->d_name[1] == '\0') || + (de->d_name[0] == '.' && de->d_name[1] == '.' && de->d_name[2] == '\0') + ) { + netdata_log_debug(D_HEALTH, "CONFIG ignoring user-config directory '%s/%s'", udir, de->d_name); + continue; + } + + if(path_is_dir(udir, de->d_name)) { + recursive_config_double_dir_load(udir, sdir, de->d_name, callback, data, depth + 1); + continue; + } + } + + if(de->d_type == DT_UNKNOWN || de->d_type == DT_REG || de->d_type == DT_LNK) { + size_t len = strlen(de->d_name); + if(path_is_file(udir, de->d_name) && + len > 5 && !strcmp(&de->d_name[len - 5], ".conf")) { + char *filename = strdupz_path_subpath(udir, de->d_name); + netdata_log_debug(D_HEALTH, "CONFIG calling callback for user file '%s'", filename); + callback(filename, data); + freez(filename); + continue; + } + } + + netdata_log_debug(D_HEALTH, "CONFIG ignoring user-config file '%s/%s' of type %d", udir, de->d_name, (int)de->d_type); + } + + closedir(dir); + } + + netdata_log_debug(D_HEALTH, "CONFIG traversing stock config directory '%s', user config directory '%s'", sdir, udir); + + dir = opendir(sdir); + if (!dir) { + netdata_log_error("CONFIG cannot open stock config directory '%s'.", sdir); + } + else { + if (strcmp(udir, sdir)) { + struct dirent *de = NULL; + while((de = readdir(dir))) { + if(de->d_type == DT_DIR || de->d_type == DT_LNK) { + if( !de->d_name[0] || + (de->d_name[0] == '.' && de->d_name[1] == '\0') || + (de->d_name[0] == '.' && de->d_name[1] == '.' && de->d_name[2] == '\0') + ) { + netdata_log_debug(D_HEALTH, "CONFIG ignoring stock config directory '%s/%s'", sdir, de->d_name); + continue; + } + + if(path_is_dir(sdir, de->d_name)) { + // we recurse in stock subdirectory, only when there is no corresponding + // user subdirectory - to avoid reading the files twice + + if(!path_is_dir(udir, de->d_name)) + recursive_config_double_dir_load(udir, sdir, de->d_name, callback, data, depth + 1); + + continue; + } + } + + if(de->d_type == DT_UNKNOWN || de->d_type == DT_REG || de->d_type == DT_LNK) { + size_t len = strlen(de->d_name); + if(path_is_file(sdir, de->d_name) && !path_is_file(udir, de->d_name) && + len > 5 && !strcmp(&de->d_name[len - 5], ".conf")) { + char *filename = strdupz_path_subpath(sdir, de->d_name); + netdata_log_debug(D_HEALTH, "CONFIG calling callback for stock file '%s'", filename); + callback(filename, data); + freez(filename); + continue; + } + + } + + netdata_log_debug(D_HEALTH, "CONFIG ignoring stock-config file '%s/%s' of type %d", udir, de->d_name, (int)de->d_type); + } + } + closedir(dir); + } + + netdata_log_debug(D_HEALTH, "CONFIG done traversing user-config directory '%s', stock config directory '%s'", udir, sdir); + + freez(udir); + freez(sdir); +} + +// Returns the number of bytes read from the file if file_size is not NULL. +// The actual buffer has an extra byte set to zero (not included in the count). +char *read_by_filename(const char *filename, long *file_size) +{ + FILE *f = fopen(filename, "r"); + if (!f) + return NULL; + if (fseek(f, 0, SEEK_END) < 0) { + fclose(f); + return NULL; + } + long size = ftell(f); + if (size <= 0 || fseek(f, 0, SEEK_END) < 0) { + fclose(f); + return NULL; + } + char *contents = callocz(size + 1, 1); + if (!contents) { + fclose(f); + return NULL; + } + if (fseek(f, 0, SEEK_SET) < 0) { + fclose(f); + freez(contents); + return NULL; + } + size_t res = fread(contents, 1, size, f); + if ( res != (size_t)size) { + freez(contents); + fclose(f); + return NULL; + } + fclose(f); + if (file_size) + *file_size = size; + return contents; +} + +char *find_and_replace(const char *src, const char *find, const char *replace, const char *where) +{ + size_t size = strlen(src) + 1; + size_t find_len = strlen(find); + size_t repl_len = strlen(replace); + char *value, *dst; + + if (likely(where)) + size += (repl_len - find_len); + + value = mallocz(size); + dst = value; + + if (likely(where)) { + size_t count = where - src; + + memmove(dst, src, count); + src += count; + dst += count; + + memmove(dst, replace, repl_len); + src += find_len; + dst += repl_len; + } + + strcpy(dst, src); + + return value; +} + + +BUFFER *run_command_and_get_output_to_buffer(const char *command, int max_line_length) { + BUFFER *wb = buffer_create(0, NULL); + + pid_t pid; + FILE *fp = netdata_popen(command, &pid, NULL); + + if(fp) { + char buffer[max_line_length + 1]; + while (fgets(buffer, max_line_length, fp)) { + buffer[max_line_length] = '\0'; + buffer_strcat(wb, buffer); + } + } + else { + buffer_free(wb); + netdata_log_error("Failed to execute command '%s'.", command); + return NULL; + } + + netdata_pclose(NULL, fp, pid); + return wb; +} + +bool run_command_and_copy_output_to_stdout(const char *command, int max_line_length) { + pid_t pid; + FILE *fp = netdata_popen(command, &pid, NULL); + + if(fp) { + char buffer[max_line_length + 1]; + while (fgets(buffer, max_line_length, fp)) + fprintf(stdout, "%s", buffer); + } + else { + netdata_log_error("Failed to execute command '%s'.", command); + return false; + } + + netdata_pclose(NULL, fp, pid); + return true; +} + +void for_each_open_fd(OPEN_FD_ACTION action, OPEN_FD_EXCLUDE excluded_fds){ + int fd; + + switch(action){ + case OPEN_FD_ACTION_CLOSE: + if(!(excluded_fds & OPEN_FD_EXCLUDE_STDIN)) (void)close(STDIN_FILENO); + if(!(excluded_fds & OPEN_FD_EXCLUDE_STDOUT)) (void)close(STDOUT_FILENO); + if(!(excluded_fds & OPEN_FD_EXCLUDE_STDERR)) (void)close(STDERR_FILENO); +#if defined(HAVE_CLOSE_RANGE) + if(close_range(STDERR_FILENO + 1, ~0U, 0) == 0) return; + netdata_log_error("close_range() failed, will try to close fds one by one"); +#endif + break; + case OPEN_FD_ACTION_FD_CLOEXEC: + if(!(excluded_fds & OPEN_FD_EXCLUDE_STDIN)) (void)fcntl(STDIN_FILENO, F_SETFD, FD_CLOEXEC); + if(!(excluded_fds & OPEN_FD_EXCLUDE_STDOUT)) (void)fcntl(STDOUT_FILENO, F_SETFD, FD_CLOEXEC); + if(!(excluded_fds & OPEN_FD_EXCLUDE_STDERR)) (void)fcntl(STDERR_FILENO, F_SETFD, FD_CLOEXEC); +#if defined(HAVE_CLOSE_RANGE) && defined(CLOSE_RANGE_CLOEXEC) // Linux >= 5.11, FreeBSD >= 13.1 + if(close_range(STDERR_FILENO + 1, ~0U, CLOSE_RANGE_CLOEXEC) == 0) return; + netdata_log_error("close_range() failed, will try to mark fds for closing one by one"); +#endif + break; + default: + break; // do nothing + } + + DIR *dir = opendir("/proc/self/fd"); + if (dir == NULL) { + struct rlimit rl; + int open_max = -1; + + if(getrlimit(RLIMIT_NOFILE, &rl) == 0 && rl.rlim_max != RLIM_INFINITY) open_max = rl.rlim_max; +#ifdef _SC_OPEN_MAX + else open_max = sysconf(_SC_OPEN_MAX); +#endif + + if (open_max == -1) open_max = 65535; // 65535 arbitrary default if everything else fails + + for (fd = STDERR_FILENO + 1; fd < open_max; fd++) { + switch(action){ + case OPEN_FD_ACTION_CLOSE: + if(fd_is_valid(fd)) (void)close(fd); + break; + case OPEN_FD_ACTION_FD_CLOEXEC: + (void)fcntl(fd, F_SETFD, FD_CLOEXEC); + break; + default: + break; // do nothing + } + } + } else { + struct dirent *entry; + while ((entry = readdir(dir)) != NULL) { + fd = str2i(entry->d_name); + if(unlikely((fd == STDIN_FILENO ) || (fd == STDOUT_FILENO) || (fd == STDERR_FILENO) )) continue; + switch(action){ + case OPEN_FD_ACTION_CLOSE: + if(fd_is_valid(fd)) (void)close(fd); + break; + case OPEN_FD_ACTION_FD_CLOEXEC: + (void)fcntl(fd, F_SETFD, FD_CLOEXEC); + break; + default: + break; // do nothing + } + } + closedir(dir); + } +} + +struct timing_steps { + const char *name; + usec_t time; + size_t count; +} timing_steps[TIMING_STEP_MAX + 1] = { + [TIMING_STEP_INTERNAL] = { .name = "internal", .time = 0, }, + + [TIMING_STEP_BEGIN2_PREPARE] = { .name = "BEGIN2 prepare", .time = 0, }, + [TIMING_STEP_BEGIN2_FIND_CHART] = { .name = "BEGIN2 find chart", .time = 0, }, + [TIMING_STEP_BEGIN2_PARSE] = { .name = "BEGIN2 parse", .time = 0, }, + [TIMING_STEP_BEGIN2_ML] = { .name = "BEGIN2 ml", .time = 0, }, + [TIMING_STEP_BEGIN2_PROPAGATE] = { .name = "BEGIN2 propagate", .time = 0, }, + [TIMING_STEP_BEGIN2_STORE] = { .name = "BEGIN2 store", .time = 0, }, + + [TIMING_STEP_SET2_PREPARE] = { .name = "SET2 prepare", .time = 0, }, + [TIMING_STEP_SET2_LOOKUP_DIMENSION] = { .name = "SET2 find dimension", .time = 0, }, + [TIMING_STEP_SET2_PARSE] = { .name = "SET2 parse", .time = 0, }, + [TIMING_STEP_SET2_ML] = { .name = "SET2 ml", .time = 0, }, + [TIMING_STEP_SET2_PROPAGATE] = { .name = "SET2 propagate", .time = 0, }, + [TIMING_STEP_RRDSET_STORE_METRIC] = { .name = "SET2 rrdset store", .time = 0, }, + [TIMING_STEP_DBENGINE_FIRST_CHECK] = { .name = "db 1st check", .time = 0, }, + [TIMING_STEP_DBENGINE_CHECK_DATA] = { .name = "db check data", .time = 0, }, + [TIMING_STEP_DBENGINE_PACK] = { .name = "db pack", .time = 0, }, + [TIMING_STEP_DBENGINE_PAGE_FIN] = { .name = "db page fin", .time = 0, }, + [TIMING_STEP_DBENGINE_MRG_UPDATE] = { .name = "db mrg update", .time = 0, }, + [TIMING_STEP_DBENGINE_PAGE_ALLOC] = { .name = "db page alloc", .time = 0, }, + [TIMING_STEP_DBENGINE_CREATE_NEW_PAGE] = { .name = "db new page", .time = 0, }, + [TIMING_STEP_DBENGINE_FLUSH_PAGE] = { .name = "db page flush", .time = 0, }, + [TIMING_STEP_SET2_STORE] = { .name = "SET2 store", .time = 0, }, + + [TIMING_STEP_END2_PREPARE] = { .name = "END2 prepare", .time = 0, }, + [TIMING_STEP_END2_PUSH_V1] = { .name = "END2 push v1", .time = 0, }, + [TIMING_STEP_END2_ML] = { .name = "END2 ml", .time = 0, }, + [TIMING_STEP_END2_RRDSET] = { .name = "END2 rrdset", .time = 0, }, + [TIMING_STEP_END2_PROPAGATE] = { .name = "END2 propagate", .time = 0, }, + [TIMING_STEP_END2_STORE] = { .name = "END2 store", .time = 0, }, + + // terminator + [TIMING_STEP_MAX] = { .name = NULL, .time = 0, }, +}; + +void timing_action(TIMING_ACTION action, TIMING_STEP step) { + static __thread usec_t last_action_time = 0; + static struct timing_steps timings2[TIMING_STEP_MAX + 1] = {}; + + switch(action) { + case TIMING_ACTION_INIT: + last_action_time = now_monotonic_usec(); + break; + + case TIMING_ACTION_STEP: { + if(!last_action_time) + return; + + usec_t now = now_monotonic_usec(); + __atomic_add_fetch(&timing_steps[step].time, now - last_action_time, __ATOMIC_RELAXED); + __atomic_add_fetch(&timing_steps[step].count, 1, __ATOMIC_RELAXED); + last_action_time = now; + break; + } + + case TIMING_ACTION_FINISH: { + if(!last_action_time) + return; + + usec_t expected = __atomic_load_n(&timing_steps[TIMING_STEP_INTERNAL].time, __ATOMIC_RELAXED); + if(last_action_time - expected < 10 * USEC_PER_SEC) { + last_action_time = 0; + return; + } + + if(!__atomic_compare_exchange_n(&timing_steps[TIMING_STEP_INTERNAL].time, &expected, last_action_time, false, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST)) { + last_action_time = 0; + return; + } + + struct timing_steps timings3[TIMING_STEP_MAX + 1]; + memcpy(timings3, timing_steps, sizeof(timings3)); + + size_t total_reqs = 0; + usec_t total_usec = 0; + for(size_t t = 1; t < TIMING_STEP_MAX ; t++) { + total_usec += timings3[t].time - timings2[t].time; + total_reqs += timings3[t].count - timings2[t].count; + } + + BUFFER *wb = buffer_create(1024, NULL); + + for(size_t t = 1; t < TIMING_STEP_MAX ; t++) { + size_t requests = timings3[t].count - timings2[t].count; + if(!requests) continue; + + buffer_sprintf(wb, "TIMINGS REPORT: [%3zu. %-20s]: # %10zu, t %11.2f ms (%6.2f %%), avg %6.2f usec/run\n", + t, + timing_steps[t].name ? timing_steps[t].name : "x", + requests, + (double) (timings3[t].time - timings2[t].time) / (double)USEC_PER_MS, + (double) (timings3[t].time - timings2[t].time) * 100.0 / (double) total_usec, + (double) (timings3[t].time - timings2[t].time) / (double)requests + ); + } + + netdata_log_info("TIMINGS REPORT:\n%sTIMINGS REPORT: total # %10zu, t %11.2f ms", + buffer_tostring(wb), total_reqs, (double)total_usec / USEC_PER_MS); + + memcpy(timings2, timings3, sizeof(timings2)); + + last_action_time = 0; + buffer_free(wb); + } + } +} + +#ifdef ENABLE_HTTPS +int hash256_string(const unsigned char *string, size_t size, char *hash) { + EVP_MD_CTX *ctx; + ctx = EVP_MD_CTX_create(); + + if (!ctx) + return 0; + + if (!EVP_DigestInit(ctx, EVP_sha256())) { + EVP_MD_CTX_destroy(ctx); + return 0; + } + + if (!EVP_DigestUpdate(ctx, string, size)) { + EVP_MD_CTX_destroy(ctx); + return 0; + } + + if (!EVP_DigestFinal(ctx, (unsigned char *)hash, NULL)) { + EVP_MD_CTX_destroy(ctx); + return 0; + } + EVP_MD_CTX_destroy(ctx); + return 1; +} +#endif + + +bool rrdr_relative_window_to_absolute(time_t *after, time_t *before, time_t now) { + if(!now) now = now_realtime_sec(); + + int absolute_period_requested = -1; + time_t before_requested = *before; + time_t after_requested = *after; + + // allow relative for before (smaller than API_RELATIVE_TIME_MAX) + if(ABS(before_requested) <= API_RELATIVE_TIME_MAX) { + // if the user asked for a positive relative time, + // flip it to a negative + if(before_requested > 0) + before_requested = -before_requested; + + before_requested = now + before_requested; + absolute_period_requested = 0; + } + + // allow relative for after (smaller than API_RELATIVE_TIME_MAX) + if(ABS(after_requested) <= API_RELATIVE_TIME_MAX) { + if(after_requested > 0) + after_requested = -after_requested; + + // if the user didn't give an after, use the number of points + // to give a sane default + if(after_requested == 0) + after_requested = -600; + + // since the query engine now returns inclusive timestamps + // it is awkward to return 6 points when after=-5 is given + // so for relative queries we add 1 second, to give + // more predictable results to users. + after_requested = before_requested + after_requested + 1; + absolute_period_requested = 0; + } + + if(absolute_period_requested == -1) + absolute_period_requested = 1; + + // check if the parameters are flipped + if(after_requested > before_requested) { + long long t = before_requested; + before_requested = after_requested; + after_requested = t; + } + + // if the query requests future data + // shift the query back to be in the present time + // (this may also happen because of the rules above) + if(before_requested > now) { + time_t delta = before_requested - now; + before_requested -= delta; + after_requested -= delta; + } + + *before = before_requested; + *after = after_requested; + + return (absolute_period_requested != 1); +} + +// Returns 1 if an absolute period was requested or 0 if it was a relative period +bool rrdr_relative_window_to_absolute_query(time_t *after, time_t *before, time_t *now_ptr, bool unittest_running) { + time_t now = now_realtime_sec() - 1; + + if(now_ptr) + *now_ptr = now; + + time_t before_requested = *before; + time_t after_requested = *after; + + int absolute_period_requested = rrdr_relative_window_to_absolute(&after_requested, &before_requested, now); + + time_t absolute_minimum_time = now - (10 * 365 * 86400); + time_t absolute_maximum_time = now + (1 * 365 * 86400); + + if (after_requested < absolute_minimum_time && !unittest_running) + after_requested = absolute_minimum_time; + + if (after_requested > absolute_maximum_time && !unittest_running) + after_requested = absolute_maximum_time; + + if (before_requested < absolute_minimum_time && !unittest_running) + before_requested = absolute_minimum_time; + + if (before_requested > absolute_maximum_time && !unittest_running) + before_requested = absolute_maximum_time; + + *before = before_requested; + *after = after_requested; + + return (absolute_period_requested != 1); +} + +int netdata_base64_decode(const char *encoded, char *decoded, size_t decoded_size) { + static const unsigned char base64_table[256] = { + ['A'] = 0, ['B'] = 1, ['C'] = 2, ['D'] = 3, ['E'] = 4, ['F'] = 5, ['G'] = 6, ['H'] = 7, + ['I'] = 8, ['J'] = 9, ['K'] = 10, ['L'] = 11, ['M'] = 12, ['N'] = 13, ['O'] = 14, ['P'] = 15, + ['Q'] = 16, ['R'] = 17, ['S'] = 18, ['T'] = 19, ['U'] = 20, ['V'] = 21, ['W'] = 22, ['X'] = 23, + ['Y'] = 24, ['Z'] = 25, ['a'] = 26, ['b'] = 27, ['c'] = 28, ['d'] = 29, ['e'] = 30, ['f'] = 31, + ['g'] = 32, ['h'] = 33, ['i'] = 34, ['j'] = 35, ['k'] = 36, ['l'] = 37, ['m'] = 38, ['n'] = 39, + ['o'] = 40, ['p'] = 41, ['q'] = 42, ['r'] = 43, ['s'] = 44, ['t'] = 45, ['u'] = 46, ['v'] = 47, + ['w'] = 48, ['x'] = 49, ['y'] = 50, ['z'] = 51, ['0'] = 52, ['1'] = 53, ['2'] = 54, ['3'] = 55, + ['4'] = 56, ['5'] = 57, ['6'] = 58, ['7'] = 59, ['8'] = 60, ['9'] = 61, ['+'] = 62, ['/'] = 63, + [0 ... '+' - 1] = 255, + ['+' + 1 ... '/' - 1] = 255, + ['9' + 1 ... 'A' - 1] = 255, + ['Z' + 1 ... 'a' - 1] = 255, + ['z' + 1 ... 255] = 255 + }; + + size_t count = 0; + unsigned int tmp = 0; + int i, bit; + + if (decoded_size < 1) + return 0; // Buffer size must be at least 1 for null termination + + for (i = 0, bit = 0; encoded[i]; i++) { + unsigned char value = base64_table[(unsigned char)encoded[i]]; + if (value > 63) + return -1; // Invalid character in input + + tmp = tmp << 6 | value; + if (++bit == 4) { + if (count + 3 >= decoded_size) break; // Stop decoding if buffer is full + decoded[count++] = (tmp >> 16) & 0xFF; + decoded[count++] = (tmp >> 8) & 0xFF; + decoded[count++] = tmp & 0xFF; + tmp = 0; + bit = 0; + } + } + + if (bit > 0 && count + 1 < decoded_size) { + tmp <<= 6 * (4 - bit); + if (bit > 2 && count + 1 < decoded_size) decoded[count++] = (tmp >> 16) & 0xFF; + if (bit > 3 && count + 1 < decoded_size) decoded[count++] = (tmp >> 8) & 0xFF; + } + + decoded[count] = '\0'; // Null terminate the output string + return count; +} |