diff options
Diffstat (limited to 'libnetdata/procfile')
-rw-r--r-- | libnetdata/procfile/Makefile.am | 8 | ||||
-rw-r--r-- | libnetdata/procfile/README.md | 71 | ||||
-rw-r--r-- | libnetdata/procfile/procfile.c | 490 | ||||
-rw-r--r-- | libnetdata/procfile/procfile.h | 109 |
4 files changed, 0 insertions, 678 deletions
diff --git a/libnetdata/procfile/Makefile.am b/libnetdata/procfile/Makefile.am deleted file mode 100644 index 161784b8f..000000000 --- a/libnetdata/procfile/Makefile.am +++ /dev/null @@ -1,8 +0,0 @@ -# SPDX-License-Identifier: GPL-3.0-or-later - -AUTOMAKE_OPTIONS = subdir-objects -MAINTAINERCLEANFILES = $(srcdir)/Makefile.in - -dist_noinst_DATA = \ - README.md \ - $(NULL) diff --git a/libnetdata/procfile/README.md b/libnetdata/procfile/README.md deleted file mode 100644 index 37138bd19..000000000 --- a/libnetdata/procfile/README.md +++ /dev/null @@ -1,71 +0,0 @@ -<!-- -title: "PROCFILE" -custom_edit_url: https://github.com/netdata/netdata/edit/master/libnetdata/procfile/README.md -sidebar_label: "Procfile" -learn_status: "Published" -learn_topic_type: "Tasks" -learn_rel_path: "Developers/libnetdata" ---> - -# PROCFILE - -procfile is a library for reading text data files (i.e `/proc` files) in the fastest possible way. - -## How it works - -The library automatically adapts (through the iterations) its memory so that each file -is read with single `read()` call. - -Then the library splits the file into words, using the supplied separators. -The library also supported quoted words (i.e. strings within of which the separators are ignored). - -### Initialization - -Initially the caller: - -- calls `procfile_open()` to open the file and allocate the structures needed. - -### Iterations - -For each iteration, the caller: - -- calls `procfile_readall()` to read updated contents. - This call also rewinds (`lseek()` to 0) before reading it. - - For every file, a [BUFFER](https://github.com/netdata/netdata/blob/master/libnetdata/buffer/README.md) is used that is automatically adjusted to fit the entire - file contents of the file. So the file is read with a single `read()` call (providing atomicity / consistency when - the data are read from the kernel). - - Once the data are read, 2 arrays of pointers are updated: - - - a `words` array, pointing to each word in the data read - - a `lines` array, pointing to the first word for each line - - This is highly optimized. Both arrays are automatically adjusted to - fit all contents and are updated in a single pass on the data. - - The library provides a number of macros: - - - `procfile_lines()` returns the # of lines read - - `procfile_linewords()` returns the # of words in the given line - - `procfile_word()` returns a pointer the given word # - - `procfile_line()` returns a pointer to the first word of the given line # - - `procfile_lineword()` returns a pointer to the given word # of the given line # - -### Cleanup - -When the caller exits: - -- calls `procfile_free()` to close the file and free all memory used. - -### Performance - -- a **raspberry Pi 1** (the oldest single core one) can process 5.000+ `/proc` files per second. -- a **J1900 Celeron** processor can process 23.000+ `/proc` files per second per core. - -To achieve this kind of performance, the library tries to work in batches so that the code -and the data are inside the processor's caches. - -This library is extensively used in Netdata and its plugins. - - diff --git a/libnetdata/procfile/procfile.c b/libnetdata/procfile/procfile.c deleted file mode 100644 index 0bc731d68..000000000 --- a/libnetdata/procfile/procfile.c +++ /dev/null @@ -1,490 +0,0 @@ -// SPDX-License-Identifier: GPL-3.0-or-later - -#include "../libnetdata.h" - -#define PF_PREFIX "PROCFILE" - -#define PFWORDS_INCREASE_STEP 2000 -#define PFLINES_INCREASE_STEP 200 -#define PROCFILE_INCREMENT_BUFFER 4096 - -int procfile_open_flags = O_RDONLY; - -int procfile_adaptive_initial_allocation = 0; - -// if adaptive allocation is set, these store the -// max values we have seen so far -size_t procfile_max_lines = PFLINES_INCREASE_STEP; -size_t procfile_max_words = PFWORDS_INCREASE_STEP; -size_t procfile_max_allocation = PROCFILE_INCREMENT_BUFFER; - - -// ---------------------------------------------------------------------------- - -char *procfile_filename(procfile *ff) { - if(ff->filename) - return ff->filename; - - char filename[FILENAME_MAX + 1]; - char buffer[FILENAME_MAX + 1]; - snprintfz(buffer, FILENAME_MAX, "/proc/self/fd/%d", ff->fd); - - ssize_t l = readlink(buffer, filename, FILENAME_MAX); - if(unlikely(l == -1)) - snprintfz(filename, FILENAME_MAX, "unknown filename for fd %d", ff->fd); - else - filename[l] = '\0'; - - - ff->filename = strdupz(filename); - - // on non-linux systems, something like this will be needed - // fcntl(ff->fd, F_GETPATH, ff->filename) - - return ff->filename; -} - -// ---------------------------------------------------------------------------- -// An array of words - -static inline void procfile_words_add(procfile *ff, char *str) { - // netdata_log_debug(D_PROCFILE, PF_PREFIX ": adding word No %d: '%s'", fw->len, str); - - pfwords *fw = ff->words; - if(unlikely(fw->len == fw->size)) { - // netdata_log_debug(D_PROCFILE, PF_PREFIX ": expanding words"); - size_t minimum = PFWORDS_INCREASE_STEP; - size_t optimal = fw->size / 2; - size_t wanted = (optimal > minimum)?optimal:minimum; - - ff->words = fw = reallocz(fw, sizeof(pfwords) + (fw->size + wanted) * sizeof(char *)); - fw->size += wanted; - } - - fw->words[fw->len++] = str; -} - -NEVERNULL -static inline pfwords *procfile_words_create(void) { - // netdata_log_debug(D_PROCFILE, PF_PREFIX ": initializing words"); - - size_t size = (procfile_adaptive_initial_allocation) ? procfile_max_words : PFWORDS_INCREASE_STEP; - - pfwords *new = mallocz(sizeof(pfwords) + size * sizeof(char *)); - new->len = 0; - new->size = size; - return new; -} - -static inline void procfile_words_reset(pfwords *fw) { - // netdata_log_debug(D_PROCFILE, PF_PREFIX ": resetting words"); - fw->len = 0; -} - -static inline void procfile_words_free(pfwords *fw) { - // netdata_log_debug(D_PROCFILE, PF_PREFIX ": freeing words"); - - freez(fw); -} - - -// ---------------------------------------------------------------------------- -// An array of lines - -NEVERNULL -static inline size_t *procfile_lines_add(procfile *ff) { - // netdata_log_debug(D_PROCFILE, PF_PREFIX ": adding line %d at word %d", fl->len, first_word); - - pflines *fl = ff->lines; - if(unlikely(fl->len == fl->size)) { - // netdata_log_debug(D_PROCFILE, PF_PREFIX ": expanding lines"); - size_t minimum = PFLINES_INCREASE_STEP; - size_t optimal = fl->size / 2; - size_t wanted = (optimal > minimum)?optimal:minimum; - - ff->lines = fl = reallocz(fl, sizeof(pflines) + (fl->size + wanted) * sizeof(ffline)); - fl->size += wanted; - } - - ffline *ffl = &fl->lines[fl->len++]; - ffl->words = 0; - ffl->first = ff->words->len; - - return &ffl->words; -} - -NEVERNULL -static inline pflines *procfile_lines_create(void) { - // netdata_log_debug(D_PROCFILE, PF_PREFIX ": initializing lines"); - - size_t size = (unlikely(procfile_adaptive_initial_allocation)) ? procfile_max_words : PFLINES_INCREASE_STEP; - - pflines *new = mallocz(sizeof(pflines) + size * sizeof(ffline)); - new->len = 0; - new->size = size; - return new; -} - -static inline void procfile_lines_reset(pflines *fl) { - // netdata_log_debug(D_PROCFILE, PF_PREFIX ": resetting lines"); - - fl->len = 0; -} - -static inline void procfile_lines_free(pflines *fl) { - // netdata_log_debug(D_PROCFILE, PF_PREFIX ": freeing lines"); - - freez(fl); -} - - -// ---------------------------------------------------------------------------- -// The procfile - -void procfile_close(procfile *ff) { - if(unlikely(!ff)) return; - - netdata_log_debug(D_PROCFILE, PF_PREFIX ": Closing file '%s'", procfile_filename(ff)); - - freez(ff->filename); - procfile_lines_free(ff->lines); - procfile_words_free(ff->words); - - if(likely(ff->fd != -1)) close(ff->fd); - freez(ff); -} - -NOINLINE -static void procfile_parser(procfile *ff) { - // netdata_log_debug(D_PROCFILE, PF_PREFIX ": Parsing file '%s'", ff->filename); - - char *s = ff->data // our current position - , *e = &ff->data[ff->len] // the terminating null - , *t = ff->data; // the first character of a word (or quoted / parenthesized string) - - // the look up array to find our type of character - PF_CHAR_TYPE *separators = ff->separators; - - char quote = 0; // the quote character - only when in quoted string - size_t opened = 0; // counts the number of open parenthesis - - size_t *line_words = procfile_lines_add(ff); - - while(s < e) { - PF_CHAR_TYPE ct = separators[(unsigned char)(*s)]; - - // this is faster than a switch() - // read more here: http://lazarenko.me/switch/ - if(likely(ct == PF_CHAR_IS_WORD)) { - s++; - } - else if(likely(ct == PF_CHAR_IS_SEPARATOR)) { - if(!quote && !opened) { - if (s != t) { - // separator, but we have word before it - *s = '\0'; - procfile_words_add(ff, t); - (*line_words)++; - t = ++s; - } - else { - // separator at the beginning - // skip it - t = ++s; - } - } - else { - // we are inside a quote or parenthesized string - s++; - } - } - else if(likely(ct == PF_CHAR_IS_NEWLINE)) { - // end of line - - *s = '\0'; - procfile_words_add(ff, t); - (*line_words)++; - t = ++s; - - // netdata_log_debug(D_PROCFILE, PF_PREFIX ": ended line %d with %d words", l, ff->lines->lines[l].words); - - line_words = procfile_lines_add(ff); - } - else if(likely(ct == PF_CHAR_IS_QUOTE)) { - if(unlikely(!quote && s == t)) { - // quote opened at the beginning - quote = *s; - t = ++s; - } - else if(unlikely(quote && quote == *s)) { - // quote closed - quote = 0; - - *s = '\0'; - procfile_words_add(ff, t); - (*line_words)++; - t = ++s; - } - else - s++; - } - else if(likely(ct == PF_CHAR_IS_OPEN)) { - if(s == t) { - opened++; - t = ++s; - } - else if(opened) { - opened++; - s++; - } - else - s++; - } - else if(likely(ct == PF_CHAR_IS_CLOSE)) { - if(opened) { - opened--; - - if(!opened) { - *s = '\0'; - procfile_words_add(ff, t); - (*line_words)++; - t = ++s; - } - else - s++; - } - else - s++; - } - else - fatal("Internal Error: procfile_readall() does not handle all the cases."); - } - - if(likely(s > t && t < e)) { - // the last word - if(unlikely(ff->len >= ff->size)) { - // we are going to loose the last byte - s = &ff->data[ff->size - 1]; - } - - *s = '\0'; - procfile_words_add(ff, t); - (*line_words)++; - // t = ++s; - } -} - -procfile *procfile_readall(procfile *ff) { - // netdata_log_debug(D_PROCFILE, PF_PREFIX ": Reading file '%s'.", ff->filename); - - ff->len = 0; // zero the used size - ssize_t r = 1; // read at least once - while(r > 0) { - ssize_t s = ff->len; - ssize_t x = ff->size - s; - - if(unlikely(!x)) { - size_t minimum = PROCFILE_INCREMENT_BUFFER; - size_t optimal = ff->size / 2; - size_t wanted = (optimal > minimum)?optimal:minimum; - - netdata_log_debug(D_PROCFILE, PF_PREFIX ": Expanding data buffer for file '%s' by %zu bytes.", procfile_filename(ff), wanted); - ff = reallocz(ff, sizeof(procfile) + ff->size + wanted); - ff->size += wanted; - } - - netdata_log_debug(D_PROCFILE, "Reading file '%s', from position %zd with length %zd", procfile_filename(ff), s, (ssize_t)(ff->size - s)); - r = read(ff->fd, &ff->data[s], ff->size - s); - if(unlikely(r == -1)) { - if(unlikely(!(ff->flags & PROCFILE_FLAG_NO_ERROR_ON_FILE_IO))) collector_error(PF_PREFIX ": Cannot read from file '%s' on fd %d", procfile_filename(ff), ff->fd); - else if(unlikely(ff->flags & PROCFILE_FLAG_ERROR_ON_ERROR_LOG)) - netdata_log_error(PF_PREFIX ": Cannot read from file '%s' on fd %d", procfile_filename(ff), ff->fd); - procfile_close(ff); - return NULL; - } - - ff->len += r; - } - - // netdata_log_debug(D_PROCFILE, "Rewinding file '%s'", ff->filename); - if(unlikely(lseek(ff->fd, 0, SEEK_SET) == -1)) { - if(unlikely(!(ff->flags & PROCFILE_FLAG_NO_ERROR_ON_FILE_IO))) collector_error(PF_PREFIX ": Cannot rewind on file '%s'.", procfile_filename(ff)); - else if(unlikely(ff->flags & PROCFILE_FLAG_ERROR_ON_ERROR_LOG)) - netdata_log_error(PF_PREFIX ": Cannot rewind on file '%s'.", procfile_filename(ff)); - procfile_close(ff); - return NULL; - } - - procfile_lines_reset(ff->lines); - procfile_words_reset(ff->words); - procfile_parser(ff); - - if(unlikely(procfile_adaptive_initial_allocation)) { - if(unlikely(ff->len > procfile_max_allocation)) procfile_max_allocation = ff->len; - if(unlikely(ff->lines->len > procfile_max_lines)) procfile_max_lines = ff->lines->len; - if(unlikely(ff->words->len > procfile_max_words)) procfile_max_words = ff->words->len; - } - - // netdata_log_debug(D_PROCFILE, "File '%s' updated.", ff->filename); - return ff; -} - -static PF_CHAR_TYPE procfile_default_separators[256]; -__attribute__((constructor)) void procfile_initialize_default_separators(void) { - int i = 256; - while(i--) { - if(unlikely(i == '\n' || i == '\r')) - procfile_default_separators[i] = PF_CHAR_IS_NEWLINE; - - else if(unlikely(isspace(i) || !isprint(i))) - procfile_default_separators[i] = PF_CHAR_IS_SEPARATOR; - - else - procfile_default_separators[i] = PF_CHAR_IS_WORD; - } -} - -NOINLINE -static void procfile_set_separators(procfile *ff, const char *separators) { - // set the separators - if(unlikely(!separators)) - separators = " \t=|"; - - // copy the default - memcpy(ff->separators, procfile_default_separators, 256 * sizeof(PF_CHAR_TYPE)); - - PF_CHAR_TYPE *ffs = ff->separators; - const char *s = separators; - while(*s) - ffs[(int)*s++] = PF_CHAR_IS_SEPARATOR; -} - -void procfile_set_quotes(procfile *ff, const char *quotes) { - PF_CHAR_TYPE *ffs = ff->separators; - - // remove all quotes - int i = 256; - while(i--) - if(unlikely(ffs[i] == PF_CHAR_IS_QUOTE)) - ffs[i] = PF_CHAR_IS_WORD; - - // if nothing given, return - if(unlikely(!quotes || !*quotes)) - return; - - // set the quotes - const char *s = quotes; - while(*s) - ffs[(int)*s++] = PF_CHAR_IS_QUOTE; -} - -void procfile_set_open_close(procfile *ff, const char *open, const char *close) { - PF_CHAR_TYPE *ffs = ff->separators; - - // remove all open/close - int i = 256; - while(i--) - if(unlikely(ffs[i] == PF_CHAR_IS_OPEN || ffs[i] == PF_CHAR_IS_CLOSE)) - ffs[i] = PF_CHAR_IS_WORD; - - // if nothing given, return - if(unlikely(!open || !*open || !close || !*close)) - return; - - // set the openings - const char *s = open; - while(*s) - ffs[(int)*s++] = PF_CHAR_IS_OPEN; - - // set the closings - s = close; - while(*s) - ffs[(int)*s++] = PF_CHAR_IS_CLOSE; -} - -procfile *procfile_open(const char *filename, const char *separators, uint32_t flags) { - netdata_log_debug(D_PROCFILE, PF_PREFIX ": Opening file '%s'", filename); - - int fd = open(filename, procfile_open_flags, 0666); - if(unlikely(fd == -1)) { - if (unlikely(flags & PROCFILE_FLAG_ERROR_ON_ERROR_LOG)) - netdata_log_error(PF_PREFIX ": Cannot open file '%s'", filename); - else if (unlikely(!(flags & PROCFILE_FLAG_NO_ERROR_ON_FILE_IO))) { - if (errno == ENOENT) - collector_info(PF_PREFIX ": Cannot open file '%s'", filename); - else - collector_error(PF_PREFIX ": Cannot open file '%s'", filename); - } - return NULL; - } - - // netdata_log_info("PROCFILE: opened '%s' on fd %d", filename, fd); - - size_t size = (unlikely(procfile_adaptive_initial_allocation)) ? procfile_max_allocation : PROCFILE_INCREMENT_BUFFER; - procfile *ff = mallocz(sizeof(procfile) + size); - - //strncpyz(ff->filename, filename, FILENAME_MAX); - ff->filename = NULL; - ff->fd = fd; - ff->size = size; - ff->len = 0; - ff->flags = flags; - - ff->lines = procfile_lines_create(); - ff->words = procfile_words_create(); - - procfile_set_separators(ff, separators); - - netdata_log_debug(D_PROCFILE, "File '%s' opened.", filename); - return ff; -} - -procfile *procfile_reopen(procfile *ff, const char *filename, const char *separators, uint32_t flags) { - if(unlikely(!ff)) return procfile_open(filename, separators, flags); - - if(likely(ff->fd != -1)) { - // netdata_log_info("PROCFILE: closing fd %d", ff->fd); - close(ff->fd); - } - - ff->fd = open(filename, procfile_open_flags, 0666); - if(unlikely(ff->fd == -1)) { - procfile_close(ff); - return NULL; - } - - // netdata_log_info("PROCFILE: opened '%s' on fd %d", filename, ff->fd); - - //strncpyz(ff->filename, filename, FILENAME_MAX); - freez(ff->filename); - ff->filename = NULL; - ff->flags = flags; - - // do not do the separators again if NULL is given - if(likely(separators)) procfile_set_separators(ff, separators); - - return ff; -} - -// ---------------------------------------------------------------------------- -// example parsing of procfile data - -void procfile_print(procfile *ff) { - size_t lines = procfile_lines(ff), l; - char *s; - (void)s; - - netdata_log_debug(D_PROCFILE, "File '%s' with %zu lines and %zu words", procfile_filename(ff), ff->lines->len, ff->words->len); - - for(l = 0; likely(l < lines) ;l++) { - size_t words = procfile_linewords(ff, l); - - netdata_log_debug(D_PROCFILE, " line %zu starts at word %zu and has %zu words", l, ff->lines->lines[l].first, ff->lines->lines[l].words); - - size_t w; - for(w = 0; likely(w < words) ;w++) { - s = procfile_lineword(ff, l, w); - netdata_log_debug(D_PROCFILE, " [%zu.%zu] '%s'", l, w, s); - } - } -} diff --git a/libnetdata/procfile/procfile.h b/libnetdata/procfile/procfile.h deleted file mode 100644 index 8db5b45f4..000000000 --- a/libnetdata/procfile/procfile.h +++ /dev/null @@ -1,109 +0,0 @@ -// SPDX-License-Identifier: GPL-3.0-or-later - -#ifndef NETDATA_PROCFILE_H -#define NETDATA_PROCFILE_H 1 - -#include "../libnetdata.h" - -// ---------------------------------------------------------------------------- -// An array of words - -typedef struct { - size_t len; // used entries - size_t size; // capacity - char *words[]; // array of pointers -} pfwords; - - -// ---------------------------------------------------------------------------- -// An array of lines - -typedef struct { - size_t words; // how many words this line has - size_t first; // the id of the first word of this line - // in the words array -} ffline; - -typedef struct { - size_t len; // used entries - size_t size; // capacity - ffline lines[]; // array of lines -} pflines; - - -// ---------------------------------------------------------------------------- -// The procfile - -#define PROCFILE_FLAG_DEFAULT 0x00000000 // To store inside `collector.log` -#define PROCFILE_FLAG_NO_ERROR_ON_FILE_IO 0x00000001 // Do not store nothing -#define PROCFILE_FLAG_ERROR_ON_ERROR_LOG 0x00000002 // Store inside `error.log` - -typedef enum __attribute__ ((__packed__)) procfile_separator { - PF_CHAR_IS_SEPARATOR, - PF_CHAR_IS_NEWLINE, - PF_CHAR_IS_WORD, - PF_CHAR_IS_QUOTE, - PF_CHAR_IS_OPEN, - PF_CHAR_IS_CLOSE -} PF_CHAR_TYPE; - -typedef struct procfile { - char *filename; // not populated until procfile_filename() is called - uint32_t flags; - int fd; // the file descriptor - size_t len; // the bytes we have placed into data - size_t size; // the bytes we have allocated for data - pflines *lines; - pfwords *words; - PF_CHAR_TYPE separators[256]; - char data[]; // allocated buffer to keep file contents -} procfile; - -// close the proc file and free all related memory -void procfile_close(procfile *ff); - -// (re)read and parse the proc file -procfile *procfile_readall(procfile *ff); - -// open a /proc or /sys file -procfile *procfile_open(const char *filename, const char *separators, uint32_t flags); - -// re-open a file -// if separators == NULL, the last separators are used -procfile *procfile_reopen(procfile *ff, const char *filename, const char *separators, uint32_t flags); - -// example walk-through a procfile parsed file -void procfile_print(procfile *ff); - -void procfile_set_quotes(procfile *ff, const char *quotes); -void procfile_set_open_close(procfile *ff, const char *open, const char *close); - -char *procfile_filename(procfile *ff); - -// ---------------------------------------------------------------------------- - -// set to the O_XXXX flags, to have procfile_open and procfile_reopen use them when opening proc files -extern int procfile_open_flags; - -// set this to 1, to have procfile adapt its initial buffer allocation to the max allocation used so far -extern int procfile_adaptive_initial_allocation; - -// return the number of lines present -#define procfile_lines(ff) ((ff)->lines->len) - -// return the number of words of the Nth line -#define procfile_linewords(ff, line) (((line) < procfile_lines(ff)) ? (ff)->lines->lines[(line)].words : 0) - -// return the Nth word of the file, or empty string -#define procfile_word(ff, word) (((word) < (ff)->words->len) ? (ff)->words->words[(word)] : "") - -// return the first word of the Nth line, or empty string -#define procfile_line(ff, line) (((line) < procfile_lines(ff)) ? procfile_word((ff), (ff)->lines->lines[(line)].first) : "") - -// return the Nth word of the current line -#define procfile_lineword(ff, line, word) (((line) < procfile_lines(ff) && (word) < procfile_linewords((ff), (line))) ? procfile_word((ff), (ff)->lines->lines[(line)].first + (word)) : "") - -// Open file without logging file IO error if any -#define procfile_open_no_log(filename, separators, flags) procfile_open(filename, separators, flags | PROCFILE_FLAG_NO_ERROR_ON_FILE_IO) - -#endif /* NETDATA_PROCFILE_H */ |