diff options
Diffstat (limited to 'logsmanagement/parser.h')
| -rw-r--r-- | logsmanagement/parser.h | 436 |
1 files changed, 436 insertions, 0 deletions
diff --git a/logsmanagement/parser.h b/logsmanagement/parser.h new file mode 100644 index 00000000..c0cf284b --- /dev/null +++ b/logsmanagement/parser.h @@ -0,0 +1,436 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +/** @file parser.h + * @brief Header of parser.c + */ + +#ifndef PARSER_H_ +#define PARSER_H_ + +#include <regex.h> +#include "daemon/common.h" +#include "libnetdata/libnetdata.h" + +// Forward decleration +typedef struct log_parser_metrics Log_parser_metrics_t; + + +/* -------------------------------------------------------------------------- */ +/* Configuration-related */ +/* -------------------------------------------------------------------------- */ + +typedef enum{ + + CHART_COLLECTED_LOGS_TOTAL = 1 << 0, + CHART_COLLECTED_LOGS_RATE = 1 << 1, + + /* FLB_WEB_LOG charts */ + CHART_VHOST = 1 << 2, + CHART_PORT = 1 << 3, + CHART_IP_VERSION = 1 << 4, + CHART_REQ_CLIENT_CURRENT = 1 << 5, + CHART_REQ_CLIENT_ALL_TIME = 1 << 6, + CHART_REQ_METHODS = 1 << 7, + CHART_REQ_PROTO = 1 << 8, + CHART_BANDWIDTH = 1 << 9, + CHART_REQ_PROC_TIME = 1 << 10, + CHART_RESP_CODE_FAMILY = 1 << 11, + CHART_RESP_CODE = 1 << 12, + CHART_RESP_CODE_TYPE = 1 << 13, + CHART_SSL_PROTO = 1 << 14, + CHART_SSL_CIPHER = 1 << 15, + + /* FLB_SYSTEMD or FLB_SYSLOG charts */ + CHART_SYSLOG_PRIOR = 1 << 16, + CHART_SYSLOG_SEVER = 1 << 17, + CHART_SYSLOG_FACIL = 1 << 18, + + /* FLB_KMSG charts */ + CHART_KMSG_SUBSYSTEM = 1 << 19, + CHART_KMSG_DEVICE = 1 << 20, + + /* FLB_DOCKER_EV charts */ + CHART_DOCKER_EV_TYPE = 1 << 21, + CHART_DOCKER_EV_ACTION = 1 << 22, + + /* FLB_MQTT charts*/ + CHART_MQTT_TOPIC = 1 << 23 + +} chart_type_t; + +typedef struct log_parser_config{ + void *gen_config; /**< Pointer to (optional) generic configuration, as per use case. */ + unsigned long int chart_config; /**< Configuration of which charts to enable according to chart_type_t **/ +} Log_parser_config_t; + +/* -------------------------------------------------------------------------- */ + + +/* -------------------------------------------------------------------------- */ +/* Web Log parsing and metrics */ +/* -------------------------------------------------------------------------- */ + +#define VHOST_MAX_LEN 255 /**< Max vhost string length, inclding terminating \0 **/ +#define PORT_MAX_LEN 6 /**< Max port string length, inclding terminating \0 **/ +#define REQ_SCHEME_MAX_LEN 6 /**< Max request scheme length, including terminating \0 **/ +#define REQ_CLIENT_MAX_LEN 46 /**< https://superuser.com/questions/381022/how-many-characters-can-an-ip-address-be#comment2219013_381029 **/ +#define REQ_METHOD_MAX_LEN 18 /**< Max request method length, including terminating \0 **/ +#define REQ_URL_MAX_LEN 128 /**< Max request URL length, including terminating \0 **/ +#define REQ_PROTO_PREF_SIZE (sizeof("HTTP/") - 1) +#define REQ_PROTO_MAX_LEN 4 /**< Max request protocol numerical part length, including terminating \0 **/ +#define REQ_SIZE_MAX_LEN 11 /**< Max size of bytes received, including terminating \0 **/ +#define REQ_PROC_TIME_MAX_LEN 11 /**< Max size of request processing time, including terminating \0 **/ +#define REQ_RESP_CODE_MAX_LEN 4 /**< Max size of response code, including terminating \0 **/ +#define REQ_RESP_SIZE_MAX_LEN 11 /**< Max size of request response size, including terminating \0 **/ +#define UPS_RESP_TIME_MAX_LEN 10 /**< Max size of upstream response time, including terminating \0 **/ +#define SSL_PROTO_MAX_LEN 8 /**< Max SSL protocol length, inclding terminating \0 **/ +#define SSL_CIPHER_SUITE_MAX_LEN 256 /**< TODO: Check max len for ssl cipher suite string is indeed 256 **/ + +#define RESP_CODE_ARR_SIZE 501 /**< Size of resp_code array, assuming 500 valid resp codes + 1 for "other" **/ + +#define WEB_LOG_INVALID_HOST_STR "invalid" +#define WEB_LOG_INVALID_PORT -1 +#define WEB_LOG_INVALID_PORT_STR "inv" +#define WEB_LOG_INVALID_CLIENT_IP_STR WEB_LOG_INVALID_PORT_STR + +/* Web log configuration */ +#define ENABLE_PARSE_WEB_LOG_LINE_DEBUG 0 + +#define VHOST_BUFFS_SCALE_FACTOR 1.5 +#define PORT_BUFFS_SCALE_FACTOR 8 // Unlike Vhosts, ports are stored as integers, so scale factor can be bigger + + +typedef enum{ + VHOST_WITH_PORT, // nginx: $host:$server_port apache: %v:%p + VHOST, // nginx: $host ($http_host) apache: %v + PORT, // nginx: $server_port apache: %p + REQ_SCHEME, // nginx: $scheme apache: - + REQ_CLIENT, // nginx: $remote_addr apache: %a (%h) + REQ, // nginx: $request apache: %r + REQ_METHOD, // nginx: $request_method apache: %m + REQ_URL, // nginx: $request_uri apache: %U + REQ_PROTO, // nginx: $server_protocol apache: %H + REQ_SIZE, // nginx: $request_length apache: %I + REQ_PROC_TIME, // nginx: $request_time apache: %D + RESP_CODE, // nginx: $status apache: %s, %>s + RESP_SIZE, // nginx: $bytes_sent, $body_bytes_sent apache: %b, %O, %B // TODO: Should separate %b from %O ? + UPS_RESP_TIME, // nginx: $upstream_response_time apache: - + SSL_PROTO, // nginx: $ssl_protocol apache: - + SSL_CIPHER_SUITE, // nginx: $ssl_cipher apache: - + TIME, // nginx: $time_local apache: %t + CUSTOM +} web_log_line_field_t; + +typedef struct web_log_parser_config{ + web_log_line_field_t *fields; + int num_fields; /**< Number of strings in the fields array. **/ + char delimiter; /**< Delimiter that separates the fields in the log format. **/ + int verify_parsed_logs; /**< Boolean whether to try and verify parsed log fields or not **/ + int skip_timestamp_parsing; /**< Boolean whether to skip parsing of timestamp fields **/ +} Web_log_parser_config_t; + +static const char *const req_method_str[] = { + "ACL", + "BASELINE-CONTROL", + "BIND", + "CHECKIN", + "CHECKOUT", + "CONNECT", + "COPY", + "DELETE", + "GET", + "HEAD", + "LABEL", + "LINK", + "LOCK", + "MERGE", + "MKACTIVITY", + "MKCALENDAR", + "MKCOL", + "MKREDIRECTREF", + "MKWORKSPACE", + "MOVE", + "OPTIONS", + "ORDERPATCH", + "PATCH", + "POST", + "PRI", + "PROPFIND", + "PROPPATCH", + "PUT", + "REBIND", + "REPORT", + "SEARCH", + "TRACE", + "UNBIND", + "UNCHECKOUT", + "UNLINK", + "UNLOCK", + "UPDATE", + "UPDATEREDIRECTREF", + "-" +}; + +#define REQ_METHOD_ARR_SIZE (int)(sizeof(req_method_str) / sizeof(req_method_str[0])) + +typedef struct web_log_metrics{ + /* Web log metrics */ + struct log_parser_metrics_vhosts_array{ + struct log_parser_metrics_vhost{ + char name[VHOST_MAX_LEN]; /**< Name of the vhost **/ + int count; /**< Occurences of the vhost **/ + } *vhosts; + int size; /**< Size of vhosts array **/ + int size_max; + } vhost_arr; + struct log_parser_metrics_ports_array{ + struct log_parser_metrics_port{ + char name[PORT_MAX_LEN]; /**< Number of port in str */ + int port; /**< Number of port **/ + int count; /**< Occurences of the port **/ + } *ports; + int size; /**< Size of ports array **/ + int size_max; + } port_arr; + struct log_parser_metrics_ip_ver{ + int v4, v6, invalid; + } ip_ver; + /**< req_clients_current_arr is used by parser.c to save unique client IPs + * extracted per circular buffer item and also in p_file_info to save unique + * client IPs per collection (poll) iteration of plugin_logsmanagement.c. + * req_clients_alltime_arr is used in p_file_info to save unique client IPs + * of all time (and so ipv4_size and ipv6_size can only grow and are never reset to 0). **/ + struct log_parser_metrics_req_clients_array{ + char (*ipv4_req_clients)[REQ_CLIENT_MAX_LEN]; + int ipv4_size; + int ipv4_size_max; + char (*ipv6_req_clients)[REQ_CLIENT_MAX_LEN]; + int ipv6_size; + int ipv6_size_max; + } req_clients_current_arr, req_clients_alltime_arr; + int req_method[REQ_METHOD_ARR_SIZE]; + struct log_parser_metrics_req_proto{ + int http_1, http_1_1, http_2, other; + } req_proto; + struct log_parser_metrics_bandwidth{ + long long req_size, resp_size; + } bandwidth; + struct log_parser_metrics_req_proc_time{ + int min, max, sum, count; + } req_proc_time; + struct log_parser_metrics_resp_code_family{ + int resp_1xx, resp_2xx, resp_3xx, resp_4xx, resp_5xx, other; // TODO: Can there be "other"? + } resp_code_family; + /**< Array counting occurences of response codes. Each item represents the + * respective response code by adding 100 to its index, e.g. resp_code[102] + * counts how many 202 codes were detected. 501st item represents "other" */ + unsigned int resp_code[RESP_CODE_ARR_SIZE]; + struct log_parser_metrics_resp_code_type{ /* Note: 304 and 401 should be treated as resp_success */ + int resp_success, resp_redirect, resp_bad, resp_error, other; // TODO: Can there be "other"? + } resp_code_type; + struct log_parser_metrics_ssl_proto{ + int tlsv1, tlsv1_1, tlsv1_2, tlsv1_3, sslv2, sslv3, other; + } ssl_proto; + struct log_parser_metrics_ssl_cipher_array{ + struct log_parser_metrics_ssl_cipher{ + char name[SSL_CIPHER_SUITE_MAX_LEN]; /**< SSL cipher suite string **/ + int count; /**< Occurences of the SSL cipher **/ + } *ssl_ciphers; + int size; /**< Size of SSL ciphers array **/ + } ssl_cipher_arr; + int64_t timestamp; +} Web_log_metrics_t; + +typedef struct log_line_parsed{ + char vhost[VHOST_MAX_LEN]; + int port; + char req_scheme[REQ_SCHEME_MAX_LEN]; + char req_client[REQ_CLIENT_MAX_LEN]; + char req_method[REQ_METHOD_MAX_LEN]; + char req_URL[REQ_URL_MAX_LEN]; + char req_proto[REQ_PROTO_MAX_LEN]; + int req_size; + int req_proc_time; + int resp_code; + int resp_size; + int ups_resp_time; + char ssl_proto[SSL_PROTO_MAX_LEN]; + char ssl_cipher[SSL_CIPHER_SUITE_MAX_LEN]; + int64_t timestamp; + int parsing_errors; +} Log_line_parsed_t; + +Web_log_parser_config_t *read_web_log_parser_config(const char *log_format, const char delimiter); +#ifdef ENABLE_LOGSMANAGEMENT_TESTS +/* Used as public only for unit testing, normally defined as static */ +int count_fields(const char *line, const char delimiter); +#endif // ENABLE_LOGSMANAGEMENT_TESTS +void parse_web_log_line(const Web_log_parser_config_t *wblp_config, + char *line, const size_t line_len, + Log_line_parsed_t *log_line_parsed); +void extract_web_log_metrics(Log_parser_config_t *parser_config, + Log_line_parsed_t *line_parsed, + Web_log_metrics_t *metrics); +Web_log_parser_config_t *auto_detect_web_log_parser_config(char *line, const char delimiter); + +/* -------------------------------------------------------------------------- */ + + +/* -------------------------------------------------------------------------- */ +/* Kernel logs (kmsg) metrics */ +/* -------------------------------------------------------------------------- */ + +#define SYSLOG_SEVER_ARR_SIZE 9 /**< Number of severity levels plus 1 for 'unknown' **/ + +typedef struct metrics_dict_item{ + bool dim_initialized; + int num; + int num_new; +} metrics_dict_item_t; + +typedef struct kernel_metrics{ + unsigned int sever[SYSLOG_SEVER_ARR_SIZE]; /**< Syslog severity, 0-7 plus 1 space for 'unknown' **/ + DICTIONARY *subsystem; + DICTIONARY *device; +} Kernel_metrics_t; + +/* -------------------------------------------------------------------------- */ + + +/* -------------------------------------------------------------------------- */ +/* Systemd and Syslog metrics */ +/* -------------------------------------------------------------------------- */ + +#define SYSLOG_FACIL_ARR_SIZE 25 /**< Number of facility levels plus 1 for 'unknown' **/ +#define SYSLOG_PRIOR_ARR_SIZE 193 /**< Number of priority values plus 1 for 'unknown' **/ + +typedef struct systemd_metrics{ + unsigned int sever[SYSLOG_SEVER_ARR_SIZE]; /**< Syslog severity, 0-7 plus 1 space for 'unknown' **/ + unsigned int facil[SYSLOG_FACIL_ARR_SIZE]; /**< Syslog facility, 0-23 plus 1 space for 'unknown' **/ + unsigned int prior[SYSLOG_PRIOR_ARR_SIZE]; /**< Syslog priority value, 0-191 plus 1 space for 'unknown' **/ +} Systemd_metrics_t; + +/* -------------------------------------------------------------------------- */ + + +/* -------------------------------------------------------------------------- */ +/* Docker Events metrics */ +/* -------------------------------------------------------------------------- */ + +static const char *const docker_ev_type_string[] = { + "container", "image", "plugin", "volume", "network", "daemon", "service", "node", "secret", "config", "unknown" +}; + +#define NUM_OF_DOCKER_EV_TYPES ((int) (sizeof docker_ev_type_string / sizeof docker_ev_type_string[0])) + +#define NUM_OF_CONTAINER_ACTIONS 25 /**< == size of 'Containers actions' array, largest array in docker_ev_action_string **/ + +static const char *const docker_ev_action_string[NUM_OF_DOCKER_EV_TYPES][NUM_OF_CONTAINER_ACTIONS] = { + /* Order of arrays is important, it must match the order of docker_ev_type_string[] strings. */ + + /* Containers actions */ + {"attach", "commit", "copy", "create", "destroy", "detach", "die", "exec_create", "exec_detach", "exec_die", + "exec_start", "export", "health_status", "kill", "oom", "pause", "rename", "resize", "restart", "start", "stop", + "top", "unpause", "update", NULL}, + + /* Images actions */ + {"delete", "import", "load", "pull", "push", "save", "tag", "untag", NULL}, + + /* Plugins actions */ + {"enable", "disable", "install", "remove", NULL}, + + /* Volumes actions */ + {"create", "destroy", "mount", "unmount", NULL}, + + /* Networks actions */ + {"create", "connect", "destroy", "disconnect", "remove", NULL}, + + /* Daemons actions */ + {"reload", NULL}, + + /* Services actions */ + {"create", "remove", "update", NULL}, + + /* Nodes actions */ + {"create", "remove", "update", NULL}, + + /* Secrets actions */ + {"create", "remove", "update", NULL}, + + /* Configs actions */ + {"create", "remove", "update", NULL}, + + {"unknown", NULL} +}; + +typedef struct docker_ev_metrics{ + unsigned int ev_type[NUM_OF_DOCKER_EV_TYPES]; + unsigned int ev_action[NUM_OF_DOCKER_EV_TYPES][NUM_OF_CONTAINER_ACTIONS]; +} Docker_ev_metrics_t; + +/* -------------------------------------------------------------------------- */ + + +/* -------------------------------------------------------------------------- */ +/* MQTT metrics */ +/* -------------------------------------------------------------------------- */ + +typedef struct mqtt_metrics{ + DICTIONARY *topic; +} Mqtt_metrics_t; + +/* -------------------------------------------------------------------------- */ + + +/* -------------------------------------------------------------------------- */ +/* Regex / Keyword search */ +/* -------------------------------------------------------------------------- */ + +#define MAX_KEYWORD_LEN 100 /**< Max size of keyword used in keyword search, in bytes */ +#define MAX_REGEX_SIZE MAX_KEYWORD_LEN + 7 /**< Max size of regular expression (used in keyword search) in bytes **/ + +int search_keyword( char *src, size_t src_sz, + char *dest, size_t *dest_sz, + const char *keyword, regex_t *regex, + const int ignore_case); + +/* -------------------------------------------------------------------------- */ + + +/* -------------------------------------------------------------------------- */ +/* Custom Charts configuration and metrics */ +/* -------------------------------------------------------------------------- */ + +typedef struct log_parser_cus_config{ + char *chartname; /**< Chart name where the regex metrics will appear in **/ + char *regex_str; /**< String representation of the regex **/ + char *regex_name; /**< If regex is named, this is where its name is stored **/ + regex_t regex; /**< The compiled regex **/ +} Log_parser_cus_config_t; + +typedef struct log_parser_cus_metrics{ + unsigned long long count; +} Log_parser_cus_metrics_t; + +/* -------------------------------------------------------------------------- */ + + +/* -------------------------------------------------------------------------- */ +/* General / Other */ +/* -------------------------------------------------------------------------- */ + +struct log_parser_metrics{ + unsigned long long num_lines; + // struct timeval tv; + time_t last_update; + union { + Web_log_metrics_t *web_log; + Kernel_metrics_t *kernel; + Systemd_metrics_t *systemd; + Docker_ev_metrics_t *docker_ev; + Mqtt_metrics_t *mqtt; + }; + Log_parser_cus_metrics_t **parser_cus; /**< Array storing custom chart metrics structs **/ +} ; + +#endif // PARSER_H_ |