Adding upstream version 1.44.3.upstream/1.44.3upstream

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

1 files changed, 501 insertions, 0 deletions

diff --git a/collectors/log2journal/log2journal.h b/collectors/log2journal/log2journal.h
new file mode 100644
index 00000000..834a5b13
--- /dev/null
+++ b/collectors/log2journal/log2journal.h
@@ -0,0 +1,501 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#ifndef NETDATA_LOG2JOURNAL_H
+#define NETDATA_LOG2JOURNAL_H
+
+// only for PACKAGE_VERSION
+#include "config.h"
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <dirent.h>
+#include <string.h>
+#include <stdbool.h>
+#include <string.h>
+#include <ctype.h>
+#include <math.h>
+#include <stdarg.h>
+#include <assert.h>
+
+// ----------------------------------------------------------------------------
+// logging
+
+// enable the compiler to check for printf like errors on our log2stderr() function
+static inline void log2stderr(const char *format, ...) __attribute__ ((format(__printf__, 1, 2)));
+static inline void log2stderr(const char *format, ...) {
+    va_list args;
+    va_start(args, format);
+    vfprintf(stderr, format, args);
+    va_end(args);
+    fprintf(stderr, "\n");
+}
+
+// ----------------------------------------------------------------------------
+// allocation functions abstraction
+
+static inline void *mallocz(size_t size) {
+    void *ptr = malloc(size);
+    if (!ptr) {
+        log2stderr("Fatal Error: Memory allocation failed. Requested size: %zu bytes.", size);
+        exit(EXIT_FAILURE);
+    }
+    return ptr;
+}
+
+static inline void *callocz(size_t elements, size_t size) {
+    void *ptr = calloc(elements, size);
+    if (!ptr) {
+        log2stderr("Fatal Error: Memory allocation failed. Requested size: %zu bytes.", elements * size);
+        exit(EXIT_FAILURE);
+    }
+    return ptr;
+}
+
+static inline void *reallocz(void *ptr, size_t size) {
+    void *new_ptr = realloc(ptr, size);
+    if (!new_ptr) {
+        log2stderr("Fatal Error: Memory reallocation failed. Requested size: %zu bytes.", size);
+        exit(EXIT_FAILURE);
+    }
+    return new_ptr;
+}
+
+static inline char *strdupz(const char *s) {
+    char *ptr = strdup(s);
+    if (!ptr) {
+        log2stderr("Fatal Error: Memory allocation failed in strdup.");
+        exit(EXIT_FAILURE);
+    }
+    return ptr;
+}
+
+static inline char *strndupz(const char *s, size_t n) {
+    char *ptr = strndup(s, n);
+    if (!ptr) {
+        log2stderr("Fatal Error: Memory allocation failed in strndup. Requested size: %zu bytes.", n);
+        exit(EXIT_FAILURE);
+    }
+    return ptr;
+}
+
+static inline void freez(void *ptr) {
+    if (ptr)
+        free(ptr);
+}
+
+// ----------------------------------------------------------------------------
+
+#define XXH_INLINE_ALL
+#include "../../libnetdata/xxhash.h"
+
+#define PCRE2_CODE_UNIT_WIDTH 8
+#include <pcre2.h>
+
+#ifdef HAVE_LIBYAML
+#include <yaml.h>
+#endif
+
+// ----------------------------------------------------------------------------
+// hashtable for HASHED_KEY
+
+// cleanup hashtable defines
+#undef SIMPLE_HASHTABLE_SORT_FUNCTION
+#undef SIMPLE_HASHTABLE_VALUE_TYPE
+#undef SIMPLE_HASHTABLE_NAME
+#undef NETDATA_SIMPLE_HASHTABLE_H
+
+struct hashed_key;
+static inline int compare_keys(struct hashed_key *k1, struct hashed_key *k2);
+#define SIMPLE_HASHTABLE_SORT_FUNCTION compare_keys
+#define SIMPLE_HASHTABLE_VALUE_TYPE struct hashed_key
+#define SIMPLE_HASHTABLE_NAME _KEY
+#include "../../libnetdata/simple_hashtable.h"
+
+// ----------------------------------------------------------------------------
+
+#define MAX_OUTPUT_KEYS 1024
+#define MAX_LINE_LENGTH (1024 * 1024)
+#define MAX_INJECTIONS (MAX_OUTPUT_KEYS / 2)
+#define MAX_REWRITES (MAX_OUTPUT_KEYS / 2)
+#define MAX_RENAMES (MAX_OUTPUT_KEYS / 2)
+
+#define JOURNAL_MAX_KEY_LEN 64              // according to systemd-journald
+#define JOURNAL_MAX_VALUE_LEN (48 * 1024)   // according to systemd-journald
+
+#define LOG2JOURNAL_CONFIG_PATH LIBCONFIG_DIR "/log2journal.d"
+
+// ----------------------------------------------------------------------------
+// character conversion for journal keys
+
+extern const char journal_key_characters_map[256];
+
+// ----------------------------------------------------------------------------
+// copy to buffer, while ensuring there is no buffer overflow
+
+static inline size_t copy_to_buffer(char *dst, size_t dst_size, const char *src, size_t src_len) {
+    if(dst_size < 2) {
+        if(dst_size == 1)
+            *dst = '\0';
+
+        return 0;
+    }
+
+    if(src_len <= dst_size - 1) {
+        memcpy(dst, src, src_len);
+        dst[src_len] = '\0';
+        return src_len;
+    }
+    else {
+        memcpy(dst, src, dst_size - 1);
+        dst[dst_size - 1] = '\0';
+        return dst_size - 1;
+    }
+}
+
+// ----------------------------------------------------------------------------
+// A dynamically sized, reusable text buffer,
+// allowing us to be fast (no allocations during iterations) while having the
+// smallest possible allocations.
+
+typedef struct txt {
+    char *txt;
+    uint32_t size;
+    uint32_t len;
+} TEXT;
+
+static inline void txt_cleanup(TEXT *t) {
+    if(!t)
+        return;
+
+    if(t->txt)
+        freez(t->txt);
+
+    t->txt = NULL;
+    t->size = 0;
+    t->len = 0;
+}
+
+static inline void txt_replace(TEXT *t, const char *s, size_t len) {
+    if(!s || !*s || len == 0) {
+        s = "";
+        len = 0;
+    }
+
+    if(len + 1 <= t->size) {
+        // the existing value allocation, fits our value
+
+        memcpy(t->txt, s, len);
+        t->txt[len] = '\0';
+        t->len = len;
+    }
+    else {
+        // no existing value allocation, or too small for our value
+        // cleanup and increase the buffer
+
+        txt_cleanup(t);
+
+        t->txt = strndupz(s, len);
+        t->size = len + 1;
+        t->len = len;
+    }
+}
+
+static inline void txt_expand_and_append(TEXT *t, const char *s, size_t len) {
+    if(len + 1 > (t->size - t->len)) {
+        size_t new_size = t->len + len + 1;
+        if(new_size < t->size * 2)
+            new_size = t->size * 2;
+
+        t->txt = reallocz(t->txt, new_size);
+        t->size = new_size;
+    }
+
+    char *copy_to = &t->txt[t->len];
+    memcpy(copy_to, s, len);
+    copy_to[len] = '\0';
+    t->len += len;
+}
+
+// ----------------------------------------------------------------------------
+
+typedef enum __attribute__((__packed__)) {
+    HK_NONE                 = 0,
+
+    // permanent flags - they are set once to optimize various decisions and lookups
+
+    HK_HASHTABLE_ALLOCATED  = (1 << 0), // this is key object allocated in the hashtable
+                                        // objects that do not have this, have a pointer to a key in the hashtable
+                                        // objects that have this, value a value allocated
+
+    HK_FILTERED             = (1 << 1), // we checked once if this key in filtered
+    HK_FILTERED_INCLUDED    = (1 << 2), // the result of the filtering was to include it in the output
+
+    HK_COLLISION_CHECKED    = (1 << 3), // we checked once for collision check of this key
+
+    HK_RENAMES_CHECKED      = (1 << 4), // we checked once if there are renames on this key
+    HK_HAS_RENAMES          = (1 << 5), // and we found there is a rename rule related to it
+
+    // ephemeral flags - they are unset at the end of each log line
+
+    HK_VALUE_FROM_LOG       = (1 << 14), // the value of this key has been read from the log (or from injection, duplication)
+    HK_VALUE_REWRITTEN      = (1 << 15), // the value of this key has been rewritten due to one of our rewrite rules
+
+} HASHED_KEY_FLAGS;
+
+typedef struct hashed_key {
+    const char *key;
+    uint32_t len;
+    HASHED_KEY_FLAGS flags;
+    XXH64_hash_t hash;
+    union {
+        struct hashed_key *hashtable_ptr; // HK_HASHTABLE_ALLOCATED is not set
+        TEXT value;                       // HK_HASHTABLE_ALLOCATED is set
+    };
+} HASHED_KEY;
+
+static inline void hashed_key_cleanup(HASHED_KEY *k) {
+    if(k->key) {
+        freez((void *)k->key);
+        k->key = NULL;
+    }
+
+    if(k->flags & HK_HASHTABLE_ALLOCATED)
+        txt_cleanup(&k->value);
+    else
+        k->hashtable_ptr = NULL;
+}
+
+static inline void hashed_key_set(HASHED_KEY *k, const char *name) {
+    hashed_key_cleanup(k);
+
+    k->key = strdupz(name);
+    k->len = strlen(k->key);
+    k->hash = XXH3_64bits(k->key, k->len);
+    k->flags = HK_NONE;
+}
+
+static inline void hashed_key_len_set(HASHED_KEY *k, const char *name, size_t len) {
+    hashed_key_cleanup(k);
+
+    k->key = strndupz(name, len);
+    k->len = len;
+    k->hash = XXH3_64bits(k->key, k->len);
+    k->flags = HK_NONE;
+}
+
+static inline bool hashed_keys_match(HASHED_KEY *k1, HASHED_KEY *k2) {
+    return ((k1 == k2) || (k1->hash == k2->hash && strcmp(k1->key, k2->key) == 0));
+}
+
+static inline int compare_keys(struct hashed_key *k1, struct hashed_key *k2) {
+    return strcmp(k1->key, k2->key);
+}
+
+// ----------------------------------------------------------------------------
+
+typedef struct search_pattern {
+    const char *pattern;
+    pcre2_code *re;
+    pcre2_match_data *match_data;
+    TEXT error;
+} SEARCH_PATTERN;
+
+void search_pattern_cleanup(SEARCH_PATTERN *sp);
+bool search_pattern_set(SEARCH_PATTERN *sp, const char *search_pattern, size_t search_pattern_len);
+
+static inline bool search_pattern_matches(SEARCH_PATTERN *sp, const char *value, size_t value_len) {
+    return pcre2_match(sp->re, (PCRE2_SPTR)value, value_len, 0, 0, sp->match_data, NULL) >= 0;
+}
+
+// ----------------------------------------------------------------------------
+
+typedef struct replacement_node {
+    HASHED_KEY name;
+    bool is_variable;
+    bool logged_error;
+
+    struct replacement_node *next;
+} REPLACE_NODE;
+
+void replace_node_free(REPLACE_NODE *rpn);
+
+typedef struct replace_pattern {
+    const char *pattern;
+    REPLACE_NODE *nodes;
+    bool has_variables;
+} REPLACE_PATTERN;
+
+void replace_pattern_cleanup(REPLACE_PATTERN *rp);
+bool replace_pattern_set(REPLACE_PATTERN *rp, const char *pattern);
+
+// ----------------------------------------------------------------------------
+
+typedef struct injection {
+    bool on_unmatched;
+    HASHED_KEY key;
+    REPLACE_PATTERN value;
+} INJECTION;
+
+void injection_cleanup(INJECTION *inj);
+
+// ----------------------------------------------------------------------------
+
+typedef struct key_rename {
+    HASHED_KEY new_key;
+    HASHED_KEY old_key;
+} RENAME;
+
+void rename_cleanup(RENAME *rn);
+
+// ----------------------------------------------------------------------------
+
+typedef enum __attribute__((__packed__)) {
+    RW_NONE                 = 0,
+    RW_MATCH_PCRE2          = (1 << 1), // a rewrite rule
+    RW_MATCH_NON_EMPTY      = (1 << 2), // a rewrite rule
+    RW_DONT_STOP            = (1 << 3),
+    RW_INJECT               = (1 << 4),
+} RW_FLAGS;
+
+typedef struct key_rewrite {
+    RW_FLAGS flags;
+    HASHED_KEY key;
+    union {
+        SEARCH_PATTERN match_pcre2;
+        REPLACE_PATTERN match_non_empty;
+    };
+    REPLACE_PATTERN value;
+} REWRITE;
+
+void rewrite_cleanup(REWRITE *rw);
+
+// ----------------------------------------------------------------------------
+// A job configuration and runtime structures
+
+typedef struct log_job {
+    bool show_config;
+
+    const char *pattern;
+    const char *prefix;
+
+    SIMPLE_HASHTABLE_KEY hashtable;
+
+    struct {
+        const char *buffer;
+        const char *trimmed;
+        size_t trimmed_len;
+        size_t size;
+        HASHED_KEY key;
+    } line;
+
+    struct {
+        SEARCH_PATTERN include;
+        SEARCH_PATTERN exclude;
+    } filter;
+
+    struct {
+        bool last_line_was_empty;
+        HASHED_KEY key;
+        TEXT current;
+    } filename;
+
+    struct {
+        uint32_t used;
+        INJECTION keys[MAX_INJECTIONS];
+    } injections;
+
+    struct {
+        HASHED_KEY key;
+        struct {
+            uint32_t used;
+            INJECTION keys[MAX_INJECTIONS];
+        } injections;
+    } unmatched;
+
+    struct {
+        uint32_t used;
+        REWRITE array[MAX_REWRITES];
+        TEXT tmp;
+    } rewrites;
+
+    struct {
+        uint32_t used;
+        RENAME array[MAX_RENAMES];
+    } renames;
+} LOG_JOB;
+
+// initialize a log job
+void log_job_init(LOG_JOB *jb);
+
+// free all resources consumed by the log job
+void log_job_cleanup(LOG_JOB *jb);
+
+// ----------------------------------------------------------------------------
+
+// the entry point to send key value pairs to the output
+// this implements the pipeline of processing renames, rewrites and duplications
+void log_job_send_extracted_key_value(LOG_JOB *jb, const char *key, const char *value, size_t len);
+
+// ----------------------------------------------------------------------------
+// configuration related
+
+// management of configuration to set settings
+bool log_job_filename_key_set(LOG_JOB *jb, const char *key, size_t key_len);
+bool log_job_key_prefix_set(LOG_JOB *jb, const char *prefix, size_t prefix_len);
+bool log_job_pattern_set(LOG_JOB *jb, const char *pattern, size_t pattern_len);
+bool log_job_injection_add(LOG_JOB *jb, const char *key, size_t key_len, const char *value, size_t value_len, bool unmatched);
+bool log_job_rewrite_add(LOG_JOB *jb, const char *key, RW_FLAGS flags, const char *search_pattern, const char *replace_pattern);
+bool log_job_rename_add(LOG_JOB *jb, const char *new_key, size_t new_key_len, const char *old_key, size_t old_key_len);
+bool log_job_include_pattern_set(LOG_JOB *jb, const char *pattern, size_t pattern_len);
+bool log_job_exclude_pattern_set(LOG_JOB *jb, const char *pattern, size_t pattern_len);
+
+// entry point to parse command line parameters
+bool log_job_command_line_parse_parameters(LOG_JOB *jb, int argc, char **argv);
+void log_job_command_line_help(const char *name);
+
+// ----------------------------------------------------------------------------
+// YAML configuration related
+
+#ifdef HAVE_LIBYAML
+bool yaml_parse_file(const char *config_file_path, LOG_JOB *jb);
+bool yaml_parse_config(const char *config_name, LOG_JOB *jb);
+#endif
+
+void log_job_configuration_to_yaml(LOG_JOB *jb);
+
+// ----------------------------------------------------------------------------
+// JSON parser
+
+typedef struct log_json_state LOG_JSON_STATE;
+LOG_JSON_STATE *json_parser_create(LOG_JOB *jb);
+void json_parser_destroy(LOG_JSON_STATE *js);
+const char *json_parser_error(LOG_JSON_STATE *js);
+bool json_parse_document(LOG_JSON_STATE *js, const char *txt);
+void json_test(void);
+
+size_t parse_surrogate(const char *s, char *d, size_t *remaining);
+
+// ----------------------------------------------------------------------------
+// logfmt parser
+
+typedef struct logfmt_state LOGFMT_STATE;
+LOGFMT_STATE *logfmt_parser_create(LOG_JOB *jb);
+void logfmt_parser_destroy(LOGFMT_STATE *lfs);
+const char *logfmt_parser_error(LOGFMT_STATE *lfs);
+bool logfmt_parse_document(LOGFMT_STATE *js, const char *txt);
+void logfmt_test(void);
+
+// ----------------------------------------------------------------------------
+// pcre2 parser
+
+typedef struct pcre2_state PCRE2_STATE;
+PCRE2_STATE *pcre2_parser_create(LOG_JOB *jb);
+void pcre2_parser_destroy(PCRE2_STATE *pcre2);
+const char *pcre2_parser_error(PCRE2_STATE *pcre2);
+bool pcre2_parse_document(PCRE2_STATE *pcre2, const char *txt, size_t len);
+bool pcre2_has_error(PCRE2_STATE *pcre2);
+void pcre2_test(void);
+
+void pcre2_get_error_in_buffer(char *msg, size_t msg_len, int rc, int pos);
+
+#endif //NETDATA_LOG2JOURNAL_H