diff options
Diffstat (limited to 'logsmanagement/logsmanag_config.c')
| -rw-r--r-- | logsmanagement/logsmanag_config.c | 1410 |
1 files changed, 1410 insertions, 0 deletions
diff --git a/logsmanagement/logsmanag_config.c b/logsmanagement/logsmanag_config.c new file mode 100644 index 00000000..5be52389 --- /dev/null +++ b/logsmanagement/logsmanag_config.c @@ -0,0 +1,1410 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +/** @file logsmanag_config.c + * @brief This file includes functions to manage + * the logs management configuration. + */ + +#include "logsmanag_config.h" +#include "db_api.h" +#include "rrd_api/rrd_api.h" +#include "helper.h" + +g_logs_manag_config_t g_logs_manag_config = { + .update_every = UPDATE_EVERY, + .update_timeout = UPDATE_TIMEOUT_DEFAULT, + .use_log_timestamp = CONFIG_BOOLEAN_AUTO, + .circ_buff_max_size_in_mib = CIRCULAR_BUFF_DEFAULT_MAX_SIZE / (1 MiB), + .circ_buff_drop_logs = CIRCULAR_BUFF_DEFAULT_DROP_LOGS, + .compression_acceleration = COMPRESSION_ACCELERATION_DEFAULT, + .db_mode = GLOBAL_DB_MODE_DEFAULT, + .disk_space_limit_in_mib = DISK_SPACE_LIMIT_DEFAULT, + .buff_flush_to_db_interval = SAVE_BLOB_TO_DB_DEFAULT, + .enable_collected_logs_total = ENABLE_COLLECTED_LOGS_TOTAL_DEFAULT, + .enable_collected_logs_rate = ENABLE_COLLECTED_LOGS_RATE_DEFAULT, + .sd_journal_field_prefix = SD_JOURNAL_FIELD_PREFIX, + .do_sd_journal_send = SD_JOURNAL_SEND_DEFAULT +}; + +static logs_manag_db_mode_t db_mode_str_to_db_mode(const char *const db_mode_str){ + if(!db_mode_str || !*db_mode_str) return g_logs_manag_config.db_mode; + else if(!strcasecmp(db_mode_str, "full")) return LOGS_MANAG_DB_MODE_FULL; + else if(!strcasecmp(db_mode_str, "none")) return LOGS_MANAG_DB_MODE_NONE; + else return g_logs_manag_config.db_mode; +} + +static struct config log_management_config = { + .first_section = NULL, + .last_section = NULL, + .mutex = NETDATA_MUTEX_INITIALIZER, + .index = { + .avl_tree = { + .root = NULL, + .compar = appconfig_section_compare + }, + .rwlock = AVL_LOCK_INITIALIZER + } +}; + +static struct Chart_meta chart_types[] = { + {.type = FLB_TAIL, .init = generic_chart_init, .update = generic_chart_update}, + {.type = FLB_WEB_LOG, .init = web_log_chart_init, .update = web_log_chart_update}, + {.type = FLB_KMSG, .init = kernel_chart_init, .update = kernel_chart_update}, + {.type = FLB_SYSTEMD, .init = systemd_chart_init, .update = systemd_chart_update}, + {.type = FLB_DOCKER_EV, .init = docker_ev_chart_init, .update = docker_ev_chart_update}, + {.type = FLB_SYSLOG, .init = generic_chart_init, .update = generic_chart_update}, + {.type = FLB_SERIAL, .init = generic_chart_init, .update = generic_chart_update}, + {.type = FLB_MQTT, .init = mqtt_chart_init, .update = mqtt_chart_update} +}; + +char *get_user_config_dir(void){ + char *dir = getenv("NETDATA_USER_CONFIG_DIR"); + + return dir ? dir : CONFIG_DIR; +} + +char *get_stock_config_dir(void){ + char *dir = getenv("NETDATA_STOCK_CONFIG_DIR"); + + return dir ? dir : LIBCONFIG_DIR; +} + +char *get_log_dir(void){ + char *dir = getenv("NETDATA_LOG_DIR"); + + return dir ? dir : LOG_DIR; +} + +char *get_cache_dir(void){ + char *dir = getenv("NETDATA_CACHE_DIR"); + + return dir ? dir : CACHE_DIR; +} + +/** + * @brief Cleanup p_file_info struct + * @param p_file_info The struct of File_info type to be cleaned up. + * @todo Pass p_file_info by reference, so that it can be set to NULL. */ +static void p_file_info_destroy(void *arg){ + struct File_info *p_file_info = (struct File_info *) arg; + + // TODO: Clean up rrd / chart stuff. + // p_file_info->chart_meta + + if(unlikely(!p_file_info)){ + collector_info("p_file_info_destroy() called but p_file_info == NULL - already destroyed?"); + return; + } + + char chartname[100]; + snprintfz(chartname, 100, "%s", p_file_info->chartname ? p_file_info->chartname : "Unknown"); + collector_info("[%s]: p_file_info_destroy() cleanup...", chartname); + + __atomic_store_n(&p_file_info->state, LOG_SRC_EXITING, __ATOMIC_RELAXED); + + if(uv_is_active((uv_handle_t *) &p_file_info->flb_tmp_buff_cpy_timer)){ + uv_timer_stop(&p_file_info->flb_tmp_buff_cpy_timer); + if (!uv_is_closing((uv_handle_t *) &p_file_info->flb_tmp_buff_cpy_timer)) + uv_close((uv_handle_t *) &p_file_info->flb_tmp_buff_cpy_timer, NULL); + } + + // TODO: Need to do proper termination of DB threads and allocated memory. + if(p_file_info->db_writer_thread){ + uv_thread_join(p_file_info->db_writer_thread); + sqlite3_finalize(p_file_info->stmt_get_log_msg_metadata_asc); + sqlite3_finalize(p_file_info->stmt_get_log_msg_metadata_desc); + if(sqlite3_close(p_file_info->db) != SQLITE_OK) + collector_error("[%s]: Failed to close database", chartname); + freez(p_file_info->db_mut); + freez((void *) p_file_info->db_metadata); + freez((void *) p_file_info->db_dir); + freez(p_file_info->db_writer_thread); + } + + freez((void *) p_file_info->chartname); + freez(p_file_info->filename); + freez((void *) p_file_info->file_basename); + freez((void *) p_file_info->stream_guid); + + for(int i = 1; i <= BLOB_MAX_FILES; i++){ + if(p_file_info->blob_handles[i]){ + uv_fs_close(NULL, NULL, p_file_info->blob_handles[i], NULL); + p_file_info->blob_handles[i] = 0; + } + } + + if(p_file_info->circ_buff) + circ_buff_destroy(p_file_info->circ_buff); + + if(p_file_info->parser_metrics){ + switch(p_file_info->log_type){ + case FLB_WEB_LOG: { + if(p_file_info->parser_metrics->web_log) + freez(p_file_info->parser_metrics->web_log); + break; + } + case FLB_KMSG: { + if(p_file_info->parser_metrics->kernel){ + dictionary_destroy(p_file_info->parser_metrics->kernel->subsystem); + dictionary_destroy(p_file_info->parser_metrics->kernel->device); + freez(p_file_info->parser_metrics->kernel); + } + break; + } + case FLB_SYSTEMD: + case FLB_SYSLOG: { + if(p_file_info->parser_metrics->systemd) + freez(p_file_info->parser_metrics->systemd); + break; + } + case FLB_DOCKER_EV: { + if(p_file_info->parser_metrics->docker_ev) + freez(p_file_info->parser_metrics->docker_ev); + break; + } + case FLB_MQTT: { + if(p_file_info->parser_metrics->mqtt){ + dictionary_destroy(p_file_info->parser_metrics->mqtt->topic); + freez(p_file_info->parser_metrics->mqtt); + } + break; + } + default: + break; + } + + for(int i = 0; p_file_info->parser_cus_config && + p_file_info->parser_metrics->parser_cus && + p_file_info->parser_cus_config[i]; i++){ + freez(p_file_info->parser_cus_config[i]->chartname); + freez(p_file_info->parser_cus_config[i]->regex_str); + freez(p_file_info->parser_cus_config[i]->regex_name); + regfree(&p_file_info->parser_cus_config[i]->regex); + freez(p_file_info->parser_cus_config[i]); + freez(p_file_info->parser_metrics->parser_cus[i]); + } + + freez(p_file_info->parser_cus_config); + freez(p_file_info->parser_metrics->parser_cus); + + freez(p_file_info->parser_metrics); + } + + if(p_file_info->parser_config){ + freez(p_file_info->parser_config->gen_config); + freez(p_file_info->parser_config); + } + + Flb_output_config_t *output_next = p_file_info->flb_outputs; + while(output_next){ + Flb_output_config_t *output = output_next; + output_next = output_next->next; + + struct flb_output_config_param *param_next = output->param; + while(param_next){ + struct flb_output_config_param *param = param_next; + param_next = param->next; + freez(param->key); + freez(param->val); + freez(param); + } + freez(output->plugin); + freez(output); + } + + freez(p_file_info->flb_config); + + freez(p_file_info); + + collector_info("[%s]: p_file_info_destroy() cleanup done", chartname); +} + +void p_file_info_destroy_all(void){ + if(p_file_infos_arr){ + uv_thread_t thread_id[p_file_infos_arr->count]; + for(int i = 0; i < p_file_infos_arr->count; i++){ + fatal_assert(0 == uv_thread_create(&thread_id[i], p_file_info_destroy, p_file_infos_arr->data[i])); + } + for(int i = 0; i < p_file_infos_arr->count; i++){ + uv_thread_join(&thread_id[i]); + } + freez(p_file_infos_arr); + p_file_infos_arr = NULL; + } +} + +/** + * @brief Load logs management configuration. + * @returns 0 if success, + * -1 if config file not found + * -2 if p_flb_srvc_config if is NULL (no flb_srvc_config_t provided) + */ +int logs_manag_config_load( flb_srvc_config_t *p_flb_srvc_config, + Flb_socket_config_t **forward_in_config_p, + int g_update_every){ + int rc = LOGS_MANAG_CONFIG_LOAD_ERROR_OK; + char section[100]; + char temp_path[FILENAME_MAX + 1]; + + struct config logsmanagement_d_conf = { + .first_section = NULL, + .last_section = NULL, + .mutex = NETDATA_MUTEX_INITIALIZER, + .index = { + .avl_tree = { + .root = NULL, + .compar = appconfig_section_compare + }, + .rwlock = AVL_LOCK_INITIALIZER + } + }; + + char *filename = strdupz_path_subpath(get_user_config_dir(), "logsmanagement.d.conf"); + if(!appconfig_load(&logsmanagement_d_conf, filename, 0, NULL)) { + collector_info("CONFIG: cannot load user config '%s'. Will try stock config.", filename); + freez(filename); + + filename = strdupz_path_subpath(get_stock_config_dir(), "logsmanagement.d.conf"); + if(!appconfig_load(&logsmanagement_d_conf, filename, 0, NULL)){ + collector_error("CONFIG: cannot load stock config '%s'. Logs management will be disabled.", filename); + rc = LOGS_MANAG_CONFIG_LOAD_ERROR_NO_STOCK_CONFIG; + } + } + freez(filename); + + + /* [global] section */ + + snprintfz(section, 100, "global"); + + g_logs_manag_config.update_every = appconfig_get_number( + &logsmanagement_d_conf, + section, + "update every", + g_logs_manag_config.update_every); + + g_logs_manag_config.update_every = + g_update_every && g_update_every > g_logs_manag_config.update_every ? + g_update_every : g_logs_manag_config.update_every; + + g_logs_manag_config.update_timeout = appconfig_get_number( + &logsmanagement_d_conf, + section, + "update timeout", + UPDATE_TIMEOUT_DEFAULT); + + if(g_logs_manag_config.update_timeout < g_logs_manag_config.update_every) + g_logs_manag_config.update_timeout = g_logs_manag_config.update_every; + + g_logs_manag_config.use_log_timestamp = appconfig_get_boolean_ondemand( + &logsmanagement_d_conf, + section, + "use log timestamp", + g_logs_manag_config.use_log_timestamp); + + g_logs_manag_config.circ_buff_max_size_in_mib = appconfig_get_number( + &logsmanagement_d_conf, + section, + "circular buffer max size MiB", + g_logs_manag_config.circ_buff_max_size_in_mib); + + g_logs_manag_config.circ_buff_drop_logs = appconfig_get_boolean( + &logsmanagement_d_conf, + section, + "circular buffer drop logs if full", + g_logs_manag_config.circ_buff_drop_logs); + + g_logs_manag_config.compression_acceleration = appconfig_get_number( + &logsmanagement_d_conf, + section, + "compression acceleration", + g_logs_manag_config.compression_acceleration); + + g_logs_manag_config.enable_collected_logs_total = appconfig_get_boolean( + &logsmanagement_d_conf, + section, + "collected logs total chart enable", + g_logs_manag_config.enable_collected_logs_total); + + g_logs_manag_config.enable_collected_logs_rate = appconfig_get_boolean( + &logsmanagement_d_conf, + section, + "collected logs rate chart enable", + g_logs_manag_config.enable_collected_logs_rate); + + g_logs_manag_config.do_sd_journal_send = appconfig_get_boolean( + &logsmanagement_d_conf, + section, + "submit logs to system journal", + g_logs_manag_config.do_sd_journal_send); + + g_logs_manag_config.sd_journal_field_prefix = appconfig_get( + &logsmanagement_d_conf, + section, + "systemd journal fields prefix", + g_logs_manag_config.sd_journal_field_prefix); + + if(!rc){ + collector_info("CONFIG: [%s] update every: %d", section, g_logs_manag_config.update_every); + collector_info("CONFIG: [%s] update timeout: %d", section, g_logs_manag_config.update_timeout); + collector_info("CONFIG: [%s] use log timestamp: %d", section, g_logs_manag_config.use_log_timestamp); + collector_info("CONFIG: [%s] circular buffer max size MiB: %d", section, g_logs_manag_config.circ_buff_max_size_in_mib); + collector_info("CONFIG: [%s] circular buffer drop logs if full: %d", section, g_logs_manag_config.circ_buff_drop_logs); + collector_info("CONFIG: [%s] compression acceleration: %d", section, g_logs_manag_config.compression_acceleration); + collector_info("CONFIG: [%s] collected logs total chart enable: %d", section, g_logs_manag_config.enable_collected_logs_total); + collector_info("CONFIG: [%s] collected logs rate chart enable: %d", section, g_logs_manag_config.enable_collected_logs_rate); + collector_info("CONFIG: [%s] submit logs to system journal: %d", section, g_logs_manag_config.do_sd_journal_send); + collector_info("CONFIG: [%s] systemd journal fields prefix: %s", section, g_logs_manag_config.sd_journal_field_prefix); + } + + + /* [db] section */ + + snprintfz(section, 100, "db"); + + const char *const db_mode_str = appconfig_get( + &logsmanagement_d_conf, + section, + "db mode", + GLOBAL_DB_MODE_DEFAULT_STR); + g_logs_manag_config.db_mode = db_mode_str_to_db_mode(db_mode_str); + + snprintfz(temp_path, FILENAME_MAX, "%s" LOGS_MANAG_DB_SUBPATH, get_cache_dir()); + db_set_main_dir(appconfig_get(&logsmanagement_d_conf, section, "db dir", temp_path)); + + g_logs_manag_config.buff_flush_to_db_interval = appconfig_get_number( + &logsmanagement_d_conf, + section, + "circular buffer flush to db", + g_logs_manag_config.buff_flush_to_db_interval); + + g_logs_manag_config.disk_space_limit_in_mib = appconfig_get_number( + &logsmanagement_d_conf, + section, + "disk space limit MiB", + g_logs_manag_config.disk_space_limit_in_mib); + + if(!rc){ + collector_info("CONFIG: [%s] db mode: %s [%d]", section, db_mode_str, (int) g_logs_manag_config.db_mode); + collector_info("CONFIG: [%s] db dir: %s", section, temp_path); + collector_info("CONFIG: [%s] circular buffer flush to db: %d", section, g_logs_manag_config.buff_flush_to_db_interval); + collector_info("CONFIG: [%s] disk space limit MiB: %d", section, g_logs_manag_config.disk_space_limit_in_mib); + } + + + /* [forward input] section */ + + snprintfz(section, 100, "forward input"); + + const int fwd_enable = appconfig_get_boolean( + &logsmanagement_d_conf, + section, + "enabled", + CONFIG_BOOLEAN_NO); + + *forward_in_config_p = (Flb_socket_config_t *) callocz(1, sizeof(Flb_socket_config_t)); + + (*forward_in_config_p)->unix_path = appconfig_get( + &logsmanagement_d_conf, + section, + "unix path", + FLB_FORWARD_UNIX_PATH_DEFAULT); + + (*forward_in_config_p)->unix_perm = appconfig_get( + &logsmanagement_d_conf, + section, + "unix perm", + FLB_FORWARD_UNIX_PERM_DEFAULT); + + // TODO: Check if listen is in valid format + (*forward_in_config_p)->listen = appconfig_get( + &logsmanagement_d_conf, + section, + "listen", + FLB_FORWARD_ADDR_DEFAULT); + + (*forward_in_config_p)->port = appconfig_get( + &logsmanagement_d_conf, + section, + "port", + FLB_FORWARD_PORT_DEFAULT); + + if(!rc){ + collector_info("CONFIG: [%s] enabled: %s", section, fwd_enable ? "yes" : "no"); + collector_info("CONFIG: [%s] unix path: %s", section, (*forward_in_config_p)->unix_path); + collector_info("CONFIG: [%s] unix perm: %s", section, (*forward_in_config_p)->unix_perm); + collector_info("CONFIG: [%s] listen: %s", section, (*forward_in_config_p)->listen); + collector_info("CONFIG: [%s] port: %s", section, (*forward_in_config_p)->port); + } + + if(!fwd_enable) { + freez(*forward_in_config_p); + *forward_in_config_p = NULL; + } + + + /* [fluent bit] section */ + + snprintfz(section, 100, "fluent bit"); + + snprintfz(temp_path, FILENAME_MAX, "%s/%s", get_log_dir(), FLB_LOG_FILENAME_DEFAULT); + + if(p_flb_srvc_config){ + p_flb_srvc_config->flush = appconfig_get( + &logsmanagement_d_conf, + section, + "flush", + p_flb_srvc_config->flush); + + p_flb_srvc_config->http_listen = appconfig_get( + &logsmanagement_d_conf, + section, + "http listen", + p_flb_srvc_config->http_listen); + + p_flb_srvc_config->http_port = appconfig_get( + &logsmanagement_d_conf, + section, + "http port", + p_flb_srvc_config->http_port); + + p_flb_srvc_config->http_server = appconfig_get( + &logsmanagement_d_conf, + section, + "http server", + p_flb_srvc_config->http_server); + + p_flb_srvc_config->log_path = appconfig_get( + &logsmanagement_d_conf, + section, + "log file", + temp_path); + + p_flb_srvc_config->log_level = appconfig_get( + &logsmanagement_d_conf, + section, + "log level", + p_flb_srvc_config->log_level); + + p_flb_srvc_config->coro_stack_size = appconfig_get( + &logsmanagement_d_conf, + section, + "coro stack size", + p_flb_srvc_config->coro_stack_size); + } + else + rc = LOGS_MANAG_CONFIG_LOAD_ERROR_P_FLB_SRVC_NULL; + + if(!rc){ + collector_info("CONFIG: [%s] flush: %s", section, p_flb_srvc_config->flush); + collector_info("CONFIG: [%s] http listen: %s", section, p_flb_srvc_config->http_listen); + collector_info("CONFIG: [%s] http port: %s", section, p_flb_srvc_config->http_port); + collector_info("CONFIG: [%s] http server: %s", section, p_flb_srvc_config->http_server); + collector_info("CONFIG: [%s] log file: %s", section, p_flb_srvc_config->log_path); + collector_info("CONFIG: [%s] log level: %s", section, p_flb_srvc_config->log_level); + collector_info("CONFIG: [%s] coro stack size: %s", section, p_flb_srvc_config->coro_stack_size); + } + + return rc; +} + +static bool metrics_dict_conflict_cb(const DICTIONARY_ITEM *item __maybe_unused, void *old_value, void *new_value, void *data __maybe_unused){ + ((metrics_dict_item_t *)old_value)->num_new += ((metrics_dict_item_t *)new_value)->num_new; + return true; +} + +#define FLB_OUTPUT_PLUGIN_NAME_KEY "name" + +static int flb_output_param_get_cb(void *entry, void *data){ + struct config_option *option = (struct config_option *) entry; + Flb_output_config_t *flb_output = (Flb_output_config_t *) data; + + char *param_prefix = callocz(1, snprintf(NULL, 0, "output %d", MAX_OUTPUTS_PER_SOURCE) + 1); + sprintf(param_prefix, "output %d", flb_output->id); + size_t param_prefix_len = strlen(param_prefix); + + if(!strncasecmp(option->name, param_prefix, param_prefix_len)){ // param->name looks like "output 1 host" + char *param_key = &option->name[param_prefix_len]; // param_key should look like " host" + while(*param_key == ' ') param_key++; // remove whitespace so it looks like "host" + + if(*param_key && strcasecmp(param_key, FLB_OUTPUT_PLUGIN_NAME_KEY)){ // ignore param_key "name" + // debug_log( "config_option: name[%s], value[%s]", option->name, option->value); + // debug_log( "config option kv:[%s][%s]", param_key, option->value); + + struct flb_output_config_param **p = &flb_output->param; + while((*p) != NULL) p = &((*p)->next); // Go to last param of linked list + + (*p) = callocz(1, sizeof(struct flb_output_config_param)); + (*p)->key = strdupz(param_key); + (*p)->val = strdupz(option->value); + } + } + + freez(param_prefix); + + return 0; +} + +/** + * @brief Initialize logs management based on a section configuration. + * @note On error, calls p_file_info_destroy() to clean up before returning. + * @param config_section Section to read configuration from. + * @todo How to handle duplicate entries? + */ +static void config_section_init(uv_loop_t *main_loop, + struct section *config_section, + Flb_socket_config_t *forward_in_config, + flb_srvc_config_t *p_flb_srvc_config, + netdata_mutex_t *stdout_mut){ + + struct File_info *p_file_info = callocz(1, sizeof(struct File_info)); + + /* ------------------------------------------------------------------------- + * Check if config_section->name is valid and if so, use it as chartname. + * ------------------------------------------------------------------------- */ + if(config_section->name && *config_section->name){ + char tmp[LOGS_MANAG_CHARTNAME_SIZE] = {0}; + + snprintfz(tmp, sizeof(tmp), "%s%s", LOGS_MANAG_CHARTNAME_PREFIX, config_section->name); + + netdata_fix_chart_id(tmp); + + for(char *ch = (char *) tmp; *ch; ch++) + *ch = *ch == '.' ? '_' : *ch; // Convert dots to underscores + + p_file_info->chartname = strdupz(tmp); + + collector_info("[%s]: Initializing config loading", p_file_info->chartname); + } else { + collector_error("Invalid logs management config section."); + return p_file_info_destroy(p_file_info); + } + + + /* ------------------------------------------------------------------------- + * Check if this log source is enabled. + * ------------------------------------------------------------------------- */ + if(appconfig_get_boolean(&log_management_config, config_section->name, "enabled", CONFIG_BOOLEAN_NO)){ + collector_info("[%s]: enabled = yes", p_file_info->chartname); + } else { + collector_info("[%s]: enabled = no", p_file_info->chartname); + return p_file_info_destroy(p_file_info); + } + + + /* ------------------------------------------------------------------------- + * Check log type. + * ------------------------------------------------------------------------- */ + char *type = appconfig_get(&log_management_config, config_section->name, "log type", "flb_tail"); + if(!type || !*type) p_file_info->log_type = FLB_TAIL; // Default + else{ + if(!strcasecmp(type, "flb_tail")) p_file_info->log_type = FLB_TAIL; + else if (!strcasecmp(type, "flb_web_log")) p_file_info->log_type = FLB_WEB_LOG; + else if (!strcasecmp(type, "flb_kmsg")) p_file_info->log_type = FLB_KMSG; + else if (!strcasecmp(type, "flb_systemd")) p_file_info->log_type = FLB_SYSTEMD; + else if (!strcasecmp(type, "flb_docker_events")) p_file_info->log_type = FLB_DOCKER_EV; + else if (!strcasecmp(type, "flb_syslog")) p_file_info->log_type = FLB_SYSLOG; + else if (!strcasecmp(type, "flb_serial")) p_file_info->log_type = FLB_SERIAL; + else if (!strcasecmp(type, "flb_mqtt")) p_file_info->log_type = FLB_MQTT; + else p_file_info->log_type = FLB_TAIL; + } + freez(type); + collector_info("[%s]: log type = %s", p_file_info->chartname, log_src_type_t_str[p_file_info->log_type]); + + + /* ------------------------------------------------------------------------- + * Read log source. + * ------------------------------------------------------------------------- */ + char *source = appconfig_get(&log_management_config, config_section->name, "log source", "local"); + if(!source || !*source) p_file_info->log_source = LOG_SOURCE_LOCAL; // Default + else if(!strcasecmp(source, "forward")) p_file_info->log_source = LOG_SOURCE_FORWARD; + else p_file_info->log_source = LOG_SOURCE_LOCAL; + freez(source); + collector_info("[%s]: log source = %s", p_file_info->chartname, log_src_t_str[p_file_info->log_source]); + + if(p_file_info->log_source == LOG_SOURCE_FORWARD && !forward_in_config){ + collector_info("[%s]: forward_in_config == NULL - this log source will be disabled", p_file_info->chartname); + return p_file_info_destroy(p_file_info); + } + + + /* ------------------------------------------------------------------------- + * Read stream uuid. + * ------------------------------------------------------------------------- */ + p_file_info->stream_guid = appconfig_get(&log_management_config, config_section->name, "stream guid", ""); + collector_info("[%s]: stream guid = %s", p_file_info->chartname, p_file_info->stream_guid); + + + /* ------------------------------------------------------------------------- + * Read log path configuration and check if it is valid. + * ------------------------------------------------------------------------- */ + p_file_info->filename = appconfig_get(&log_management_config, config_section->name, "log path", LOG_PATH_AUTO); + if( /* path doesn't matter when log source is not local */ + (p_file_info->log_source == LOG_SOURCE_LOCAL) && + + /* FLB_SYSLOG is special case, may or may not require a path */ + (p_file_info->log_type != FLB_SYSLOG) && + + /* FLB_MQTT is special case, does not require a path */ + (p_file_info->log_type != FLB_MQTT) && + + (!p_file_info->filename /* Sanity check */ || + !*p_file_info->filename || + !strcmp(p_file_info->filename, LOG_PATH_AUTO) || + access(p_file_info->filename, R_OK) + )){ + + freez(p_file_info->filename); + p_file_info->filename = NULL; + + switch(p_file_info->log_type){ + case FLB_TAIL: + if(!strcasecmp(p_file_info->chartname, LOGS_MANAG_CHARTNAME_PREFIX "netdata_daemon_log")){ + char path[FILENAME_MAX + 1]; + snprintfz(path, FILENAME_MAX, "%s/daemon.log", get_log_dir()); + if(access(path, R_OK)) { + collector_error("[%s]: 'Netdata daemon.log' path (%s) invalid, unknown or needs permissions", + p_file_info->chartname, path); + return p_file_info_destroy(p_file_info); + } else p_file_info->filename = strdupz(path); + } else if(!strcasecmp(p_file_info->chartname, LOGS_MANAG_CHARTNAME_PREFIX "fluentbit_log")){ + if(access(p_flb_srvc_config->log_path, R_OK)){ + collector_error("[%s]: Netdata fluentbit.log path (%s) invalid, unknown or needs permissions", + p_file_info->chartname, p_flb_srvc_config->log_path); + return p_file_info_destroy(p_file_info); + } else p_file_info->filename = strdupz(p_flb_srvc_config->log_path); + } else if(!strcasecmp(p_file_info->chartname, LOGS_MANAG_CHARTNAME_PREFIX "auth_log_tail")){ + const char * const auth_path_default[] = { + "/var/log/auth.log", + NULL + }; + int i = 0; + while(auth_path_default[i] && access(auth_path_default[i], R_OK)){i++;}; + if(!auth_path_default[i]){ + collector_error("[%s]: auth.log path invalid, unknown or needs permissions", p_file_info->chartname); + return p_file_info_destroy(p_file_info); + } else p_file_info->filename = strdupz(auth_path_default[i]); + } else if(!strcasecmp(p_file_info->chartname, "syslog_tail")){ + const char * const syslog_path_default[] = { + "/var/log/syslog", /* Debian, Ubuntu */ + "/var/log/messages", /* RHEL, Red Hat, CentOS, Fedora */ + NULL + }; + int i = 0; + while(syslog_path_default[i] && access(syslog_path_default[i], R_OK)){i++;}; + if(!syslog_path_default[i]){ + collector_error("[%s]: syslog path invalid, unknown or needs permissions", p_file_info->chartname); + return p_file_info_destroy(p_file_info); + } else p_file_info->filename = strdupz(syslog_path_default[i]); + } + break; + case FLB_WEB_LOG: + if(!strcasecmp(p_file_info->chartname, LOGS_MANAG_CHARTNAME_PREFIX "apache_access_log")){ + const char * const apache_access_path_default[] = { + "/var/log/apache/access.log", + "/var/log/apache2/access.log", + "/var/log/apache2/access_log", + "/var/log/httpd/access_log", + "/var/log/httpd-access.log", + NULL + }; + int i = 0; + while(apache_access_path_default[i] && access(apache_access_path_default[i], R_OK)){i++;}; + if(!apache_access_path_default[i]){ + collector_error("[%s]: Apache access.log path invalid, unknown or needs permissions", p_file_info->chartname); + return p_file_info_destroy(p_file_info); + } else p_file_info->filename = strdupz(apache_access_path_default[i]); + } else if(!strcasecmp(p_file_info->chartname, LOGS_MANAG_CHARTNAME_PREFIX "nginx_access_log")){ + const char * const nginx_access_path_default[] = { + "/var/log/nginx/access.log", + NULL + }; + int i = 0; + while(nginx_access_path_default[i] && access(nginx_access_path_default[i], R_OK)){i++;}; + if(!nginx_access_path_default[i]){ + collector_error("[%s]: Nginx access.log path invalid, unknown or needs permissions", p_file_info->chartname); + return p_file_info_destroy(p_file_info); + } else p_file_info->filename = strdupz(nginx_access_path_default[i]); + } + break; + case FLB_KMSG: + if(access(KMSG_DEFAULT_PATH, R_OK)){ + collector_error("[%s]: kmsg default path invalid, unknown or needs permissions", p_file_info->chartname); + return p_file_info_destroy(p_file_info); + } else p_file_info->filename = strdupz(KMSG_DEFAULT_PATH); + break; + case FLB_SYSTEMD: + p_file_info->filename = strdupz(SYSTEMD_DEFAULT_PATH); + break; + case FLB_DOCKER_EV: + if(access(DOCKER_EV_DEFAULT_PATH, R_OK)){ + collector_error("[%s]: Docker socket default Unix path invalid, unknown or needs permissions", p_file_info->chartname); + return p_file_info_destroy(p_file_info); + } else p_file_info->filename = strdupz(DOCKER_EV_DEFAULT_PATH); + break; + default: + collector_error("[%s]: log path invalid or unknown", p_file_info->chartname); + return p_file_info_destroy(p_file_info); + } + } + p_file_info->file_basename = get_basename(p_file_info->filename); + collector_info("[%s]: p_file_info->filename: %s", p_file_info->chartname, + p_file_info->filename ? p_file_info->filename : "NULL"); + collector_info("[%s]: p_file_info->file_basename: %s", p_file_info->chartname, + p_file_info->file_basename ? p_file_info->file_basename : "NULL"); + if(unlikely(!p_file_info->filename)) return p_file_info_destroy(p_file_info); + + + /* ------------------------------------------------------------------------- + * Read "update every" and "update timeout" configuration. + * ------------------------------------------------------------------------- */ + p_file_info->update_every = appconfig_get_number( &log_management_config, config_section->name, + "update every", g_logs_manag_config.update_every); + collector_info("[%s]: update every = %d", p_file_info->chartname, p_file_info->update_every); + + p_file_info->update_timeout = appconfig_get_number( &log_management_config, config_section->name, + "update timeout", g_logs_manag_config.update_timeout); + if(p_file_info->update_timeout < p_file_info->update_every) p_file_info->update_timeout = p_file_info->update_every; + collector_info("[%s]: update timeout = %d", p_file_info->chartname, p_file_info->update_timeout); + + + /* ------------------------------------------------------------------------- + * Read "use log timestamp" configuration. + * ------------------------------------------------------------------------- */ + p_file_info->use_log_timestamp = appconfig_get_boolean_ondemand(&log_management_config, config_section->name, + "use log timestamp", + g_logs_manag_config.use_log_timestamp); + collector_info("[%s]: use log timestamp = %s", p_file_info->chartname, + p_file_info->use_log_timestamp ? "auto or yes" : "no"); + + + /* ------------------------------------------------------------------------- + * Read compression acceleration configuration. + * ------------------------------------------------------------------------- */ + p_file_info->compression_accel = appconfig_get_number( &log_management_config, config_section->name, + "compression acceleration", + g_logs_manag_config.compression_acceleration); + collector_info("[%s]: compression acceleration = %d", p_file_info->chartname, p_file_info->compression_accel); + + + /* ------------------------------------------------------------------------- + * Read DB mode. + * ------------------------------------------------------------------------- */ + const char *const db_mode_str = appconfig_get(&log_management_config, config_section->name, "db mode", NULL); + collector_info("[%s]: db mode = %s", p_file_info->chartname, db_mode_str ? db_mode_str : "NULL"); + p_file_info->db_mode = db_mode_str_to_db_mode(db_mode_str); + freez((void *)db_mode_str); + + + /* ------------------------------------------------------------------------- + * Read save logs from buffers to DB interval configuration. + * ------------------------------------------------------------------------- */ + p_file_info->buff_flush_to_db_interval = appconfig_get_number( &log_management_config, config_section->name, + "circular buffer flush to db", + g_logs_manag_config.buff_flush_to_db_interval); + if(p_file_info->buff_flush_to_db_interval > SAVE_BLOB_TO_DB_MAX) { + p_file_info->buff_flush_to_db_interval = SAVE_BLOB_TO_DB_MAX; + collector_info("[%s]: circular buffer flush to db out of range. Using maximum permitted value: %d", + p_file_info->chartname, p_file_info->buff_flush_to_db_interval); + + } else if(p_file_info->buff_flush_to_db_interval < SAVE_BLOB_TO_DB_MIN) { + p_file_info->buff_flush_to_db_interval = SAVE_BLOB_TO_DB_MIN; + collector_info("[%s]: circular buffer flush to db out of range. Using minimum permitted value: %d", + p_file_info->chartname, p_file_info->buff_flush_to_db_interval); + } + collector_info("[%s]: circular buffer flush to db = %d", p_file_info->chartname, p_file_info->buff_flush_to_db_interval); + + + /* ------------------------------------------------------------------------- + * Read BLOB max size configuration. + * ------------------------------------------------------------------------- */ + p_file_info->blob_max_size = appconfig_get_number( &log_management_config, config_section->name, + "disk space limit MiB", + g_logs_manag_config.disk_space_limit_in_mib) MiB / BLOB_MAX_FILES; + collector_info("[%s]: BLOB max size = %lld", p_file_info->chartname, (long long)p_file_info->blob_max_size); + + + /* ------------------------------------------------------------------------- + * Read configuration about sending logs to system journal. + * ------------------------------------------------------------------------- */ + p_file_info->do_sd_journal_send = appconfig_get_boolean(&log_management_config, config_section->name, + "submit logs to system journal", + g_logs_manag_config.do_sd_journal_send); + + /* ------------------------------------------------------------------------- + * Read collected logs chart configuration. + * ------------------------------------------------------------------------- */ + p_file_info->parser_config = callocz(1, sizeof(Log_parser_config_t)); + + if(appconfig_get_boolean(&log_management_config, config_section->name, + "collected logs total chart enable", + g_logs_manag_config.enable_collected_logs_total)){ + p_file_info->parser_config->chart_config |= CHART_COLLECTED_LOGS_TOTAL; + } + collector_info( "[%s]: collected logs total chart enable = %s", p_file_info->chartname, + (p_file_info->parser_config->chart_config & CHART_COLLECTED_LOGS_TOTAL) ? "yes" : "no"); + + if(appconfig_get_boolean(&log_management_config, config_section->name, + "collected logs rate chart enable", + g_logs_manag_config.enable_collected_logs_rate)){ + p_file_info->parser_config->chart_config |= CHART_COLLECTED_LOGS_RATE; + } + collector_info( "[%s]: collected logs rate chart enable = %s", p_file_info->chartname, + (p_file_info->parser_config->chart_config & CHART_COLLECTED_LOGS_RATE) ? "yes" : "no"); + + + /* ------------------------------------------------------------------------- + * Deal with log-type-specific configuration options. + * ------------------------------------------------------------------------- */ + + if(p_file_info->log_type == FLB_TAIL || p_file_info->log_type == FLB_WEB_LOG){ + Flb_tail_config_t *tail_config = callocz(1, sizeof(Flb_tail_config_t)); + if(appconfig_get_boolean(&log_management_config, config_section->name, "use inotify", CONFIG_BOOLEAN_YES)) + tail_config->use_inotify = 1; + collector_info( "[%s]: use inotify = %s", p_file_info->chartname, tail_config->use_inotify? "yes" : "no"); + + p_file_info->flb_config = tail_config; + } + + if(p_file_info->log_type == FLB_WEB_LOG){ + /* Check if a valid web log format configuration is detected */ + char *log_format = appconfig_get(&log_management_config, config_section->name, "log format", LOG_PATH_AUTO); + const char delimiter = ' '; // TODO!!: TO READ FROM CONFIG + collector_info("[%s]: log format = %s", p_file_info->chartname, log_format ? log_format : "NULL!"); + + /* If "log format = auto" or no "log format" config is detected, + * try log format autodetection based on last log file line. + * TODO 1: Add another case in OR where log_format is compared with a valid reg exp. + * TODO 2: Set default log format and delimiter if not found in config? Or auto-detect? */ + if(!log_format || !*log_format || !strcmp(log_format, LOG_PATH_AUTO)){ + collector_info("[%s]: Attempting auto-detection of log format", p_file_info->chartname); + char *line = read_last_line(p_file_info->filename, 0); + if(!line){ + collector_error("[%s]: read_last_line() returned NULL", p_file_info->chartname); + return p_file_info_destroy(p_file_info); + } + p_file_info->parser_config->gen_config = auto_detect_web_log_parser_config(line, delimiter); + freez(line); + } + else{ + p_file_info->parser_config->gen_config = read_web_log_parser_config(log_format, delimiter); + collector_info( "[%s]: Read web log parser config: %s", p_file_info->chartname, + p_file_info->parser_config->gen_config ? "success!" : "failed!"); + } + freez(log_format); + + if(!p_file_info->parser_config->gen_config){ + collector_error("[%s]: No valid web log parser config found", p_file_info->chartname); + return p_file_info_destroy(p_file_info); + } + + /* Check whether metrics verification during parsing is required */ + Web_log_parser_config_t *wblp_config = (Web_log_parser_config_t *) p_file_info->parser_config->gen_config; + wblp_config->verify_parsed_logs = appconfig_get_boolean( &log_management_config, config_section->name, + "verify parsed logs", CONFIG_BOOLEAN_NO); + collector_info("[%s]: verify parsed logs = %d", p_file_info->chartname, wblp_config->verify_parsed_logs); + + wblp_config->skip_timestamp_parsing = p_file_info->use_log_timestamp ? 0 : 1; + collector_info("[%s]: skip_timestamp_parsing = %d", p_file_info->chartname, wblp_config->skip_timestamp_parsing); + + for(int j = 0; j < wblp_config->num_fields; j++){ + if((wblp_config->fields[j] == VHOST_WITH_PORT || wblp_config->fields[j] == VHOST) + && appconfig_get_boolean(&log_management_config, config_section->name, "vhosts chart", CONFIG_BOOLEAN_NO)){ + p_file_info->parser_config->chart_config |= CHART_VHOST; + } + if((wblp_config->fields[j] == VHOST_WITH_PORT || wblp_config->fields[j] == PORT) + && appconfig_get_boolean(&log_management_config, config_section->name, "ports chart", CONFIG_BOOLEAN_NO)){ + p_file_info->parser_config->chart_config |= CHART_PORT; + } + if((wblp_config->fields[j] == REQ_CLIENT) + && appconfig_get_boolean(&log_management_config, config_section->name, "IP versions chart", CONFIG_BOOLEAN_NO)){ + p_file_info->parser_config->chart_config |= CHART_IP_VERSION; + } + if((wblp_config->fields[j] == REQ_CLIENT) + && appconfig_get_boolean(&log_management_config, config_section->name, "unique client IPs - current poll chart", CONFIG_BOOLEAN_NO)){ + p_file_info->parser_config->chart_config |= CHART_REQ_CLIENT_CURRENT; + } + if((wblp_config->fields[j] == REQ_CLIENT) + && appconfig_get_boolean(&log_management_config, config_section->name, "unique client IPs - all-time chart", CONFIG_BOOLEAN_NO)){ + p_file_info->parser_config->chart_config |= CHART_REQ_CLIENT_ALL_TIME; + } + if((wblp_config->fields[j] == REQ || wblp_config->fields[j] == REQ_METHOD) + && appconfig_get_boolean(&log_management_config, config_section->name, "http request methods chart", CONFIG_BOOLEAN_NO)){ + p_file_info->parser_config->chart_config |= CHART_REQ_METHODS; + } + if((wblp_config->fields[j] == REQ || wblp_config->fields[j] == REQ_PROTO) + && appconfig_get_boolean(&log_management_config, config_section->name, "http protocol versions chart", CONFIG_BOOLEAN_NO)){ + p_file_info->parser_config->chart_config |= CHART_REQ_PROTO; + } + if((wblp_config->fields[j] == REQ_SIZE || wblp_config->fields[j] == RESP_SIZE) + && appconfig_get_boolean(&log_management_config, config_section->name, "bandwidth chart", CONFIG_BOOLEAN_NO)){ + p_file_info->parser_config->chart_config |= CHART_BANDWIDTH; + } + if((wblp_config->fields[j] == REQ_PROC_TIME) + && appconfig_get_boolean(&log_management_config, config_section->name, "timings chart", CONFIG_BOOLEAN_NO)){ + p_file_info->parser_config->chart_config |= CHART_REQ_PROC_TIME; + } + if((wblp_config->fields[j] == RESP_CODE) + && appconfig_get_boolean(&log_management_config, config_section->name, "response code families chart", CONFIG_BOOLEAN_NO)){ + p_file_info->parser_config->chart_config |= CHART_RESP_CODE_FAMILY; + } + if((wblp_config->fields[j] == RESP_CODE) + && appconfig_get_boolean(&log_management_config, config_section->name, "response codes chart", CONFIG_BOOLEAN_NO)){ + p_file_info->parser_config->chart_config |= CHART_RESP_CODE; + } + if((wblp_config->fields[j] == RESP_CODE) + && appconfig_get_boolean(&log_management_config, config_section->name, "response code types chart", CONFIG_BOOLEAN_NO)){ + p_file_info->parser_config->chart_config |= CHART_RESP_CODE_TYPE; + } + if((wblp_config->fields[j] == SSL_PROTO) + && appconfig_get_boolean(&log_management_config, config_section->name, "SSL protocols chart", CONFIG_BOOLEAN_NO)){ + p_file_info->parser_config->chart_config |= CHART_SSL_PROTO; + } + if((wblp_config->fields[j] == SSL_CIPHER_SUITE) + && appconfig_get_boolean(&log_management_config, config_section->name, "SSL chipher suites chart", CONFIG_BOOLEAN_NO)){ + p_file_info->parser_config->chart_config |= CHART_SSL_CIPHER; + } + } + } + else if(p_file_info->log_type == FLB_KMSG){ + Flb_kmsg_config_t *kmsg_config = callocz(1, sizeof(Flb_kmsg_config_t)); + + kmsg_config->prio_level = appconfig_get(&log_management_config, config_section->name, "prio level", "8"); + + p_file_info->flb_config = kmsg_config; + + if(appconfig_get_boolean(&log_management_config, config_section->name, "severity chart", CONFIG_BOOLEAN_NO)) { + p_file_info->parser_config->chart_config |= CHART_SYSLOG_SEVER; + } + if(appconfig_get_boolean(&log_management_config, config_section->name, "subsystem chart", CONFIG_BOOLEAN_NO)) { + p_file_info->parser_config->chart_config |= CHART_KMSG_SUBSYSTEM; + } + if(appconfig_get_boolean(&log_management_config, config_section->name, "device chart", CONFIG_BOOLEAN_NO)) { + p_file_info->parser_config->chart_config |= CHART_KMSG_DEVICE; + } + } + else if(p_file_info->log_type == FLB_SYSTEMD || p_file_info->log_type == FLB_SYSLOG){ + if(p_file_info->log_type == FLB_SYSLOG){ + Syslog_parser_config_t *syslog_config = callocz(1, sizeof(Syslog_parser_config_t)); + + /* Read syslog format */ + syslog_config->log_format = appconfig_get( &log_management_config, + config_section->name, + "log format", NULL); + collector_info("[%s]: log format = %s", p_file_info->chartname, + syslog_config->log_format ? syslog_config->log_format : "NULL!"); + if(!syslog_config->log_format || !*syslog_config->log_format || !strcasecmp(syslog_config->log_format, "auto")){ + freez(syslog_config->log_format); + freez(syslog_config); + return p_file_info_destroy(p_file_info); + } + + syslog_config->socket_config = callocz(1, sizeof(Flb_socket_config_t)); + + /* Read syslog socket mode + * see also https://docs.fluentbit.io/manual/pipeline/inputs/syslog#configuration-parameters */ + syslog_config->socket_config->mode = appconfig_get( &log_management_config, + config_section->name, + "mode", "unix_udp"); + collector_info("[%s]: mode = %s", p_file_info->chartname, syslog_config->socket_config->mode); + + /* Check for valid socket path if (mode == unix_udp) or + * (mode == unix_tcp), else read syslog network interface to bind, + * if (mode == udp) or (mode == tcp). */ + if( !strcasecmp(syslog_config->socket_config->mode, "unix_udp") || + !strcasecmp(syslog_config->socket_config->mode, "unix_tcp")){ + if(!p_file_info->filename || !*p_file_info->filename || !strcasecmp(p_file_info->filename, LOG_PATH_AUTO)){ + // freez(syslog_config->socket_config->mode); + freez(syslog_config->socket_config); + freez(syslog_config->log_format); + freez(syslog_config); + return p_file_info_destroy(p_file_info); + } + syslog_config->socket_config->unix_perm = appconfig_get(&log_management_config, + config_section->name, + "unix_perm", "0644"); + collector_info("[%s]: unix_perm = %s", p_file_info->chartname, syslog_config->socket_config->unix_perm); + } else if( !strcasecmp(syslog_config->socket_config->mode, "udp") || + !strcasecmp(syslog_config->socket_config->mode, "tcp")){ + // TODO: Check if listen is in valid format + syslog_config->socket_config->listen = appconfig_get( &log_management_config, + config_section->name, + "listen", "0.0.0.0"); + collector_info("[%s]: listen = %s", p_file_info->chartname, syslog_config->socket_config->listen); + syslog_config->socket_config->port = appconfig_get( &log_management_config, + config_section->name, + "port", "5140"); + collector_info("[%s]: port = %s", p_file_info->chartname, syslog_config->socket_config->port); + } else { + /* Any other modes are invalid */ + // freez(syslog_config->socket_config->mode); + freez(syslog_config->socket_config); + freez(syslog_config->log_format); + freez(syslog_config); + return p_file_info_destroy(p_file_info); + } + + p_file_info->parser_config->gen_config = syslog_config; + } + if(appconfig_get_boolean(&log_management_config, config_section->name, "priority value chart", CONFIG_BOOLEAN_NO)) { + p_file_info->parser_config->chart_config |= CHART_SYSLOG_PRIOR; + } + if(appconfig_get_boolean(&log_management_config, config_section->name, "severity chart", CONFIG_BOOLEAN_NO)) { + p_file_info->parser_config->chart_config |= CHART_SYSLOG_SEVER; + } + if(appconfig_get_boolean(&log_management_config, config_section->name, "facility chart", CONFIG_BOOLEAN_NO)) { + p_file_info->parser_config->chart_config |= CHART_SYSLOG_FACIL; + } + } + else if(p_file_info->log_type == FLB_DOCKER_EV){ + if(appconfig_get_boolean(&log_management_config, config_section->name, "event type chart", CONFIG_BOOLEAN_NO)) { + p_file_info->parser_config->chart_config |= CHART_DOCKER_EV_TYPE; + } + if(appconfig_get_boolean(&log_management_config, config_section->name, "event action chart", CONFIG_BOOLEAN_NO)) { + p_file_info->parser_config->chart_config |= CHART_DOCKER_EV_ACTION; + } + } + else if(p_file_info->log_type == FLB_SERIAL){ + Flb_serial_config_t *serial_config = callocz(1, sizeof(Flb_serial_config_t)); + + serial_config->bitrate = appconfig_get(&log_management_config, config_section->name, "bitrate", "115200"); + serial_config->min_bytes = appconfig_get(&log_management_config, config_section->name, "min bytes", "1"); + serial_config->separator = appconfig_get(&log_management_config, config_section->name, "separator", ""); + serial_config->format = appconfig_get(&log_management_config, config_section->name, "format", ""); + + p_file_info->flb_config = serial_config; + } + else if(p_file_info->log_type == FLB_MQTT){ + Flb_socket_config_t *socket_config = callocz(1, sizeof(Flb_socket_config_t)); + + socket_config->listen = appconfig_get(&log_management_config, config_section->name, "listen", "0.0.0.0"); + socket_config->port = appconfig_get(&log_management_config, config_section->name, "port", "1883"); + + p_file_info->flb_config = socket_config; + + if(appconfig_get_boolean(&log_management_config, config_section->name, "topic chart", CONFIG_BOOLEAN_NO)) { + p_file_info->parser_config->chart_config |= CHART_MQTT_TOPIC; + } + } + + + /* ------------------------------------------------------------------------- + * Allocate p_file_info->parser_metrics memory. + * ------------------------------------------------------------------------- */ + p_file_info->parser_metrics = callocz(1, sizeof(Log_parser_metrics_t)); + switch(p_file_info->log_type){ + case FLB_WEB_LOG:{ + p_file_info->parser_metrics->web_log = callocz(1, sizeof(Web_log_metrics_t)); + break; + } + case FLB_KMSG: { + p_file_info->parser_metrics->kernel = callocz(1, sizeof(Kernel_metrics_t)); + p_file_info->parser_metrics->kernel->subsystem = dictionary_create( DICT_OPTION_SINGLE_THREADED | + DICT_OPTION_NAME_LINK_DONT_CLONE | + DICT_OPTION_DONT_OVERWRITE_VALUE); + dictionary_register_conflict_callback(p_file_info->parser_metrics->kernel->subsystem, metrics_dict_conflict_cb, NULL); + p_file_info->parser_metrics->kernel->device = dictionary_create(DICT_OPTION_SINGLE_THREADED | + DICT_OPTION_NAME_LINK_DONT_CLONE | + DICT_OPTION_DONT_OVERWRITE_VALUE); + dictionary_register_conflict_callback(p_file_info->parser_metrics->kernel->device, metrics_dict_conflict_cb, NULL); + break; + } + case FLB_SYSTEMD: + case FLB_SYSLOG: { + p_file_info->parser_metrics->systemd = callocz(1, sizeof(Systemd_metrics_t)); + break; + } + case FLB_DOCKER_EV: { + p_file_info->parser_metrics->docker_ev = callocz(1, sizeof(Docker_ev_metrics_t)); + break; + } + case FLB_MQTT: { + p_file_info->parser_metrics->mqtt = callocz(1, sizeof(Mqtt_metrics_t)); + p_file_info->parser_metrics->mqtt->topic = dictionary_create( DICT_OPTION_SINGLE_THREADED | + DICT_OPTION_NAME_LINK_DONT_CLONE | + DICT_OPTION_DONT_OVERWRITE_VALUE); + dictionary_register_conflict_callback(p_file_info->parser_metrics->mqtt->topic, metrics_dict_conflict_cb, NULL); + break; + } + default: + break; + } + + + /* ------------------------------------------------------------------------- + * Configure (optional) custom charts. + * ------------------------------------------------------------------------- */ + p_file_info->parser_cus_config = callocz(1, sizeof(Log_parser_cus_config_t *)); + p_file_info->parser_metrics->parser_cus = callocz(1, sizeof(Log_parser_cus_metrics_t *)); + for(int cus_off = 1; cus_off <= MAX_CUS_CHARTS_PER_SOURCE; cus_off++){ + + /* Read chart name config */ + char *cus_chart_k = mallocz(snprintf(NULL, 0, "custom %d chart", MAX_CUS_CHARTS_PER_SOURCE) + 1); + sprintf(cus_chart_k, "custom %d chart", cus_off); + char *cus_chart_v = appconfig_get(&log_management_config, config_section->name, cus_chart_k, NULL); + debug_log( "cus chart: (%s:%s)", cus_chart_k, cus_chart_v ? cus_chart_v : "NULL"); + freez(cus_chart_k); + if(unlikely(!cus_chart_v)){ + collector_error("[%s]: custom %d chart = NULL, custom charts for this log source will be disabled.", + p_file_info->chartname, cus_off); + break; + } + netdata_fix_chart_id(cus_chart_v); + + /* Read regex config */ + char *cus_regex_k = mallocz(snprintf(NULL, 0, "custom %d regex", MAX_CUS_CHARTS_PER_SOURCE) + 1); + sprintf(cus_regex_k, "custom %d regex", cus_off); + char *cus_regex_v = appconfig_get(&log_management_config, config_section->name, cus_regex_k, NULL); + debug_log( "cus regex: (%s:%s)", cus_regex_k, cus_regex_v ? cus_regex_v : "NULL"); + freez(cus_regex_k); + if(unlikely(!cus_regex_v)) { + collector_error("[%s]: custom %d regex = NULL, custom charts for this log source will be disabled.", + p_file_info->chartname, cus_off); + freez(cus_chart_v); + break; + } + + /* Read regex name config */ + char *cus_regex_name_k = mallocz(snprintf(NULL, 0, "custom %d regex name", MAX_CUS_CHARTS_PER_SOURCE) + 1); + sprintf(cus_regex_name_k, "custom %d regex name", cus_off); + char *cus_regex_name_v = appconfig_get( &log_management_config, config_section->name, + cus_regex_name_k, cus_regex_v); + debug_log( "cus regex name: (%s:%s)", cus_regex_name_k, cus_regex_name_v ? cus_regex_name_v : "NULL"); + freez(cus_regex_name_k); + m_assert(cus_regex_name_v, "cus_regex_name_v cannot be NULL, should be cus_regex_v"); + + + /* Escape any backslashes in the regex name, to ensure dimension is displayed correctly in charts */ + int regex_name_bslashes = 0; + char **p_regex_name = &cus_regex_name_v; + for(char *p = *p_regex_name; *p; p++) if(unlikely(*p == '\\')) regex_name_bslashes++; + if(regex_name_bslashes) { + *p_regex_name = reallocz(*p_regex_name, strlen(*p_regex_name) + 1 + regex_name_bslashes); + for(char *p = *p_regex_name; *p; p++){ + if(unlikely(*p == '\\')){ + memmove(p + 1, p, strlen(p) + 1); + *p++ = '\\'; + } + } + } + + /* Read ignore case config */ + char *cus_ignore_case_k = mallocz(snprintf(NULL, 0, "custom %d ignore case", MAX_CUS_CHARTS_PER_SOURCE) + 1); + sprintf(cus_ignore_case_k, "custom %d ignore case", cus_off); + int cus_ignore_case_v = appconfig_get_boolean( &log_management_config, + config_section->name, cus_ignore_case_k, CONFIG_BOOLEAN_YES); + debug_log( "cus case: (%s:%s)", cus_ignore_case_k, cus_ignore_case_v ? "yes" : "no"); + freez(cus_ignore_case_k); + + int regex_flags = cus_ignore_case_v ? REG_EXTENDED | REG_NEWLINE | REG_ICASE : REG_EXTENDED | REG_NEWLINE; + + int rc; + regex_t regex; + if (unlikely((rc = regcomp(®ex, cus_regex_v, regex_flags)))){ + size_t regcomp_err_str_size = regerror(rc, ®ex, 0, 0); + char *regcomp_err_str = mallocz(regcomp_err_str_size); + regerror(rc, ®ex, regcomp_err_str, regcomp_err_str_size); + collector_error("[%s]: could not compile regex for custom %d chart: %s due to error: %s. " + "Custom charts for this log source will be disabled.", + p_file_info->chartname, cus_off, cus_chart_v, regcomp_err_str); + freez(regcomp_err_str); + freez(cus_chart_v); + freez(cus_regex_v); + freez(cus_regex_name_v); + break; + }; + + /* Allocate memory and copy config to p_file_info->parser_cus_config struct */ + p_file_info->parser_cus_config = reallocz( p_file_info->parser_cus_config, + (cus_off + 1) * sizeof(Log_parser_cus_config_t *)); + p_file_info->parser_cus_config[cus_off - 1] = callocz(1, sizeof(Log_parser_cus_config_t)); + + p_file_info->parser_cus_config[cus_off - 1]->chartname = cus_chart_v; + p_file_info->parser_cus_config[cus_off - 1]->regex_str = cus_regex_v; + p_file_info->parser_cus_config[cus_off - 1]->regex_name = cus_regex_name_v; + p_file_info->parser_cus_config[cus_off - 1]->regex = regex; + + /* Initialise custom log parser metrics struct array */ + p_file_info->parser_metrics->parser_cus = reallocz( p_file_info->parser_metrics->parser_cus, + (cus_off + 1) * sizeof(Log_parser_cus_metrics_t *)); + p_file_info->parser_metrics->parser_cus[cus_off - 1] = callocz(1, sizeof(Log_parser_cus_metrics_t)); + + + p_file_info->parser_cus_config[cus_off] = NULL; + p_file_info->parser_metrics->parser_cus[cus_off] = NULL; + } + + + /* ------------------------------------------------------------------------- + * Configure (optional) Fluent Bit outputs. + * ------------------------------------------------------------------------- */ + + Flb_output_config_t **output_next_p = &p_file_info->flb_outputs; + for(int out_off = 1; out_off <= MAX_OUTPUTS_PER_SOURCE; out_off++){ + + /* Read output plugin */ + char *out_plugin_k = callocz(1, snprintf(NULL, 0, "output %d " FLB_OUTPUT_PLUGIN_NAME_KEY, MAX_OUTPUTS_PER_SOURCE) + 1); + sprintf(out_plugin_k, "output %d " FLB_OUTPUT_PLUGIN_NAME_KEY, out_off); + char *out_plugin_v = appconfig_get(&log_management_config, config_section->name, out_plugin_k, NULL); + debug_log( "output %d "FLB_OUTPUT_PLUGIN_NAME_KEY": %s", out_off, out_plugin_v ? out_plugin_v : "NULL"); + freez(out_plugin_k); + if(unlikely(!out_plugin_v)){ + collector_error("[%s]: output %d "FLB_OUTPUT_PLUGIN_NAME_KEY" = NULL, outputs for this log source will be disabled.", + p_file_info->chartname, out_off); + break; + } + + Flb_output_config_t *output = callocz(1, sizeof(Flb_output_config_t)); + output->id = out_off; + output->plugin = out_plugin_v; + + /* Read parameters for this output */ + avl_traverse_lock(&config_section->values_index, flb_output_param_get_cb, output); + + *output_next_p = output; + output_next_p = &output->next; + } + + + /* ------------------------------------------------------------------------- + * Read circular buffer configuration and initialize the buffer. + * ------------------------------------------------------------------------- */ + size_t circular_buffer_max_size = ((size_t)appconfig_get_number(&log_management_config, + config_section->name, + "circular buffer max size MiB", + g_logs_manag_config.circ_buff_max_size_in_mib)) MiB; + if(circular_buffer_max_size > CIRCULAR_BUFF_MAX_SIZE_RANGE_MAX) { + circular_buffer_max_size = CIRCULAR_BUFF_MAX_SIZE_RANGE_MAX; + collector_info( "[%s]: circular buffer max size out of range. Using maximum permitted value (MiB): %zu", + p_file_info->chartname, (size_t) (circular_buffer_max_size / (1 MiB))); + } else if(circular_buffer_max_size < CIRCULAR_BUFF_MAX_SIZE_RANGE_MIN) { + circular_buffer_max_size = CIRCULAR_BUFF_MAX_SIZE_RANGE_MIN; + collector_info( "[%s]: circular buffer max size out of range. Using minimum permitted value (MiB): %zu", + p_file_info->chartname, (size_t) (circular_buffer_max_size / (1 MiB))); + } + collector_info("[%s]: circular buffer max size MiB = %zu", p_file_info->chartname, (size_t) (circular_buffer_max_size / (1 MiB))); + + int circular_buffer_allow_dropped_logs = appconfig_get_boolean( &log_management_config, + config_section->name, + "circular buffer drop logs if full", + g_logs_manag_config.circ_buff_drop_logs); + collector_info("[%s]: circular buffer drop logs if full = %s", p_file_info->chartname, + circular_buffer_allow_dropped_logs ? "yes" : "no"); + + p_file_info->circ_buff = circ_buff_init(p_file_info->buff_flush_to_db_interval, + circular_buffer_max_size, + circular_buffer_allow_dropped_logs); + + + /* ------------------------------------------------------------------------- + * Initialize rrd related structures. + * ------------------------------------------------------------------------- */ + p_file_info->chart_meta = callocz(1, sizeof(struct Chart_meta)); + memcpy(p_file_info->chart_meta, &chart_types[p_file_info->log_type], sizeof(struct Chart_meta)); + p_file_info->chart_meta->base_prio = NETDATA_CHART_PRIO_LOGS_BASE + p_file_infos_arr->count * NETDATA_CHART_PRIO_LOGS_INCR; + netdata_mutex_lock(stdout_mut); + p_file_info->chart_meta->init(p_file_info); + fflush(stdout); + netdata_mutex_unlock(stdout_mut); + + /* ------------------------------------------------------------------------- + * Initialize input plugin for local log sources. + * ------------------------------------------------------------------------- */ + if(p_file_info->log_source == LOG_SOURCE_LOCAL){ + int rc = flb_add_input(p_file_info); + if(unlikely(rc)){ + collector_error("[%s]: flb_add_input() error: %d", p_file_info->chartname, rc); + return p_file_info_destroy(p_file_info); + } + } + + /* flb_complete_item_timer_timeout_cb() is needed for both local and + * non-local sources. */ + p_file_info->flb_tmp_buff_cpy_timer.data = p_file_info; + if(unlikely(0 != uv_mutex_init(&p_file_info->flb_tmp_buff_mut))) + fatal("uv_mutex_init(&p_file_info->flb_tmp_buff_mut) failed"); + + fatal_assert(0 == uv_timer_init( main_loop, + &p_file_info->flb_tmp_buff_cpy_timer)); + + fatal_assert(0 == uv_timer_start( &p_file_info->flb_tmp_buff_cpy_timer, + flb_complete_item_timer_timeout_cb, 0, + p_file_info->update_timeout * MSEC_PER_SEC)); + + + /* ------------------------------------------------------------------------- + * All set up successfully - add p_file_info to list of all p_file_info structs. + * ------------------------------------------------------------------------- */ + p_file_infos_arr->data = reallocz(p_file_infos_arr->data, (++p_file_infos_arr->count) * (sizeof p_file_info)); + p_file_infos_arr->data[p_file_infos_arr->count - 1] = p_file_info; + + __atomic_store_n(&p_file_info->state, LOG_SRC_READY, __ATOMIC_RELAXED); + + collector_info("[%s]: initialization completed", p_file_info->chartname); +} + +void config_file_load( uv_loop_t *main_loop, + Flb_socket_config_t *p_forward_in_config, + flb_srvc_config_t *p_flb_srvc_config, + netdata_mutex_t *stdout_mut){ + + int user_default_conf_found = 0; + + struct section *config_section; + + char tmp_name[FILENAME_MAX + 1]; + snprintfz(tmp_name, FILENAME_MAX, "%s/logsmanagement.d", get_user_config_dir()); + DIR *dir = opendir(tmp_name); + + if(dir){ + struct dirent *de = NULL; + while ((de = readdir(dir))) { + size_t d_name_len = strlen(de->d_name); + if (de->d_type == DT_DIR || d_name_len < 6 || strncmp(&de->d_name[d_name_len - 5], ".conf", sizeof(".conf"))) + continue; + + if(!user_default_conf_found && !strncmp(de->d_name, "default.conf", sizeof("default.conf"))) + user_default_conf_found = 1; + + snprintfz(tmp_name, FILENAME_MAX, "%s/logsmanagement.d/%s", get_user_config_dir(), de->d_name); + collector_info("loading config:%s", tmp_name); + log_management_config = (struct config){ + .first_section = NULL, + .last_section = NULL, + .mutex = NETDATA_MUTEX_INITIALIZER, + .index = { + .avl_tree = { + .root = NULL, + .compar = appconfig_section_compare + }, + .rwlock = AVL_LOCK_INITIALIZER + } + }; + if(!appconfig_load(&log_management_config, tmp_name, 0, NULL)) + continue; + + config_section = log_management_config.first_section; + do { + config_section_init(main_loop, config_section, p_forward_in_config, p_flb_srvc_config, stdout_mut); + config_section = config_section->next; + } while(config_section); + + } + closedir(dir); + } + + if(!user_default_conf_found){ + collector_info("CONFIG: cannot load user config '%s/logsmanagement.d/default.conf'. Will try stock config.", get_user_config_dir()); + snprintfz(tmp_name, FILENAME_MAX, "%s/logsmanagement.d/default.conf", get_stock_config_dir()); + if(!appconfig_load(&log_management_config, tmp_name, 0, NULL)){ + collector_error("CONFIG: cannot load stock config '%s/logsmanagement.d/default.conf'. Logs management will be disabled.", get_stock_config_dir()); + exit(1); + } + + config_section = log_management_config.first_section; + do { + config_section_init(main_loop, config_section, p_forward_in_config, p_flb_srvc_config, stdout_mut); + config_section = config_section->next; + } while(config_section); + } +} |