diff options
Diffstat (limited to 'src/log_format_loader.cc')
| -rw-r--r-- | src/log_format_loader.cc | 218 |
1 files changed, 181 insertions, 37 deletions
diff --git a/src/log_format_loader.cc b/src/log_format_loader.cc index 4e75b0b..c354479 100644 --- a/src/log_format_loader.cc +++ b/src/log_format_loader.cc @@ -34,8 +34,6 @@ #include "log_format_loader.hh" -#include <errno.h> -#include <fcntl.h> #include <glob.h> #include <libgen.h> #include <sys/stat.h> @@ -53,6 +51,7 @@ #include "fmt/format.h" #include "lnav_config.hh" #include "log_format_ext.hh" +#include "sql_execute.hh" #include "sql_util.hh" #include "yajlpp/yajlpp.hh" #include "yajlpp/yajlpp_def.hh" @@ -139,7 +138,10 @@ value_def_provider(const yajlpp_provider_context& ypc, external_log_format* elf) if (iter == elf->elf_value_defs.end()) { retval = std::make_shared<external_log_format::value_def>( - value_name, value_kind_t::VALUE_TEXT, -1, elf); + value_name, + value_kind_t::VALUE_TEXT, + logline_value_meta::external_column{}, + elf); elf->elf_value_defs[value_name] = retval; elf->elf_value_def_order.emplace_back(retval); } else { @@ -169,6 +171,26 @@ format_tag_def_provider(const yajlpp_provider_context& ypc, return retval.get(); } +static format_partition_def* +format_partition_def_provider(const yajlpp_provider_context& ypc, + external_log_format* elf) +{ + const intern_string_t partition_name = ypc.get_substr_i(0); + + auto iter = elf->lf_partition_defs.find(partition_name); + std::shared_ptr<format_partition_def> retval; + + if (iter == elf->lf_partition_defs.end()) { + retval = std::make_shared<format_partition_def>( + partition_name.to_string()); + elf->lf_partition_defs[partition_name] = retval; + } else { + retval = iter->second; + } + + return retval.get(); +} + static scaling_factor* scaling_factor_provider(const yajlpp_provider_context& ypc, external_log_format::value_def* value_def) @@ -281,11 +303,6 @@ read_format_field(yajlpp_parse_context* ypc, } else if (field_name == "module-field") { elf->elf_module_id_field = intern_string::lookup(value); elf->elf_container = true; - } else if (field_name == "mime-types") { - auto value_opt = ypc->ypc_current_handler->to_enum_value(value); - if (value_opt) { - elf->elf_mime_types.insert((file_format_t) *value_opt); - } } return 1; @@ -439,6 +456,8 @@ static const json_path_handler_base::enum_value_t OVERFLOW_ENUM[] = { {"truncate", external_log_format::json_format_element::overflow_t::TRUNCATE}, {"dot-dot", external_log_format::json_format_element::overflow_t::DOTDOT}, + {"last-word", + external_log_format::json_format_element::overflow_t::LASTWORD}, json_path_handler_base::ENUM_TERMINATOR, }; @@ -598,7 +617,8 @@ static const struct json_path_container value_def_handlers = { .with_synopsis("<bool>") .with_description("Indicates whether or not this field should be " "treated as a foreign key for row in another table") - .for_field(&external_log_format::value_def::vd_foreign_key), + .for_field(&external_log_format::value_def::vd_meta, + &logline_value_meta::lvm_foreign_key), yajlpp::property_handler("hidden") .with_synopsis("<bool>") @@ -762,6 +782,35 @@ static const struct json_path_container tag_handlers = { .with_children(format_tag_def_handlers), }; +static const struct json_path_container format_partition_def_handlers = { + yajlpp::property_handler("paths#") + .with_description("Restrict partitioning to the given paths") + .for_field(&format_partition_def::fpd_paths) + .with_children(tag_path_handlers), + yajlpp::property_handler("pattern") + .with_synopsis("<regex>") + .with_description("The regular expression to match against the body of " + "the log message") + .with_example("\\w+ is down") + .for_field(&format_partition_def::fpd_pattern), + yajlpp::property_handler("description") + .with_synopsis("<string>") + .with_description("A description of this partition") + .for_field(&format_partition_def::fpd_description), + json_path_handler("level") + .with_synopsis("<log-level>") + .with_description("Constrain hits to log messages with this level") + .with_enum_values(LEVEL_ENUM) + .for_field(&format_partition_def::fpd_level), +}; + +static const struct json_path_container partition_handlers = { + yajlpp::pattern_property_handler(R"((?<partition_type>[\w:;\._\-]+))") + .with_description("The type of partition to apply") + .with_obj_provider(format_partition_def_provider) + .with_children(format_partition_def_handlers), +}; + static const struct json_path_container highlight_handlers = { yajlpp::pattern_property_handler(R"((?<highlight_name>[^/]+))") .with_description("The definition of a highlight") @@ -820,13 +869,93 @@ static const struct json_path_container search_table_handlers = { .with_children(search_table_def_handlers), }; -static const json_path_handler_base::enum_value_t MIME_TYPE_ENUM[] = { - { - "application/vnd.tcpdump.pcap", - file_format_t::PCAP, - }, +static const struct json_path_container header_expr_handlers = { + yajlpp::pattern_property_handler(R"((?<header_expr_name>\w+))") + .with_description("SQLite expression") + .for_field(&external_log_format::header_exprs::he_exprs), +}; - json_path_handler_base::ENUM_TERMINATOR, +static const struct json_path_container header_handlers = { + yajlpp::property_handler("expr") + .with_description("The expressions used to check if a file header " + "matches this file format") + .for_child(&external_log_format::header::h_exprs) + .with_children(header_expr_handlers), + yajlpp::property_handler("size") + .with_description("The minimum size required for this header type") + .for_field(&external_log_format::header::h_size), +}; + +static const struct json_path_container converter_handlers = { + yajlpp::property_handler("type") + .with_description("The MIME type") + .for_field(&external_log_format::converter::c_type), + yajlpp::property_handler("header") + .with_description("File header detection definitions") + .for_child(&external_log_format::converter::c_header) + .with_children(header_handlers), + yajlpp::property_handler("command") + .with_description("The script used to convert the file") + .with_pattern(R"([\w\.\-]+)") + .for_field(&external_log_format::converter::c_command), +}; + +static const struct json_path_container opid_descriptor_handlers = { + yajlpp::property_handler("field") + .with_synopsis("<name>") + .with_description("The field to include in the operation description") + .for_field(&log_format::opid_descriptor::od_field), + yajlpp::property_handler("extractor") + .with_synopsis("<regex>") + .with_description( + "The regex used to extract content for the operation description") + .for_field(&log_format::opid_descriptor::od_extractor), + yajlpp::property_handler("prefix") + .with_description( + "A string to prepend to this field in the description") + .for_field(&log_format::opid_descriptor::od_prefix), + yajlpp::property_handler("suffix") + .with_description("A string to append to this field in the description") + .for_field(&log_format::opid_descriptor::od_suffix), + yajlpp::property_handler("joiner") + .with_description("A string to insert between instances of this field " + "when the field is found more than once") + .for_field(&log_format::opid_descriptor::od_joiner), +}; + +static const struct json_path_container opid_description_format_handlers = { + yajlpp::property_handler("format#") + .with_description("Defines the elements of this operation description") + .for_field(&log_format::opid_descriptors::od_descriptors) + .with_children(opid_descriptor_handlers), +}; + +static const struct json_path_container opid_description_handlers = { + yajlpp::pattern_property_handler(R"((?<opid_descriptor>[\w\.\-]+))") + .with_description("A type of description for this operation") + .for_field(&log_format::lf_opid_description_def) + .with_children(opid_description_format_handlers), +}; + +static const struct json_path_container subid_description_handlers = { + yajlpp::pattern_property_handler(R"((?<subid_descriptor>[\w\.\-]+))") + .with_description("A type of description for this sub-operation") + .for_field(&log_format::lf_subid_description_def) + .with_children(opid_description_format_handlers), +}; + +static const struct json_path_container opid_handlers = { + yajlpp::property_handler("subid") + .with_description("The field that holds the ID for a sub-operation") + .for_field(&external_log_format::elf_subid_field), + yajlpp::property_handler("description") + .with_description( + "Define how to construct a description of an operation") + .with_children(opid_description_handlers), + yajlpp::property_handler("sub-description") + .with_description( + "Define how to construct a description of a sub-operation") + .with_children(subid_description_handlers), }; const struct json_path_container format_handlers = { @@ -838,14 +967,16 @@ const struct json_path_container format_handlers = { json_path_handler("json", read_format_bool) .with_description( R"(Indicates that log files are JSON-encoded (deprecated, use "file-type": "json"))"), - json_path_handler("convert-to-local-time", read_format_bool) + json_path_handler("convert-to-local-time") .with_description("Indicates that displayed timestamps should " - "automatically be converted to local time"), - json_path_handler("hide-extra", read_format_bool) + "automatically be converted to local time") + .for_field(&external_log_format::lf_date_time, + &date_time_scanner::dts_local_time), + json_path_handler("hide-extra") .with_description( "Specifies whether extra values in JSON logs should be displayed") .for_field(&external_log_format::jlf_hide_extra), - json_path_handler("multiline", read_format_bool) + json_path_handler("multiline") .with_description("Indicates that log messages can span multiple lines") .for_field(&log_format::lf_multiline), json_path_handler("timestamp-divisor", read_format_double) @@ -857,9 +988,11 @@ const struct json_path_container format_handlers = { .with_description("A regular expression that restricts this format to " "log files with a matching name") .for_field(&external_log_format::elf_filename_pcre), - json_path_handler("mime-types#", read_format_field) - .with_description("A list of mime-types this format should be used for") - .with_enum_values(MIME_TYPE_ENUM), + json_path_handler("converter") + .with_description("Describes how the file format can be detected and " + "converted to a log that can be understood by lnav") + .for_child(&external_log_format::elf_converter) + .with_children(converter_handlers), json_path_handler("level-field") .with_description( "The name of the level field in the log message pattern") @@ -896,7 +1029,7 @@ const struct json_path_container format_handlers = { .with_description("A URL with more information about this log format"), json_path_handler("title", read_format_field) .with_description("The human-readable name for this log format"), - json_path_handler("description", read_format_field) + json_path_handler("description") .with_description("A longer description of this log format") .for_field(&external_log_format::lf_description), json_path_handler("timestamp-format#", read_format_field) @@ -904,10 +1037,13 @@ const struct json_path_container format_handlers = { json_path_handler("module-field", read_format_field) .with_description( "The name of the module field in the log message pattern"), - json_path_handler("opid-field", read_format_field) + json_path_handler("opid-field") .with_description( "The name of the operation-id field in the log message pattern") .for_field(&external_log_format::elf_opid_field), + yajlpp::property_handler("opid") + .with_description("Definitions related to operations found in logs") + .with_children(opid_handlers), yajlpp::property_handler("ordered-by-time") .with_synopsis("<bool>") .with_description( @@ -926,6 +1062,11 @@ const struct json_path_container format_handlers = { .with_description("The tags to automatically apply to log messages") .with_children(tag_handlers), + yajlpp::property_handler("partitions") + .with_description( + "The partitions to automatically apply to log messages") + .with_children(partition_handlers), + yajlpp::property_handler("action").with_children(action_handlers), yajlpp::property_handler("sample#") .with_description("An array of sample log messages to be tested " @@ -1447,22 +1588,25 @@ static void find_format_in_path(const ghc::filesystem::path& path, available_scripts& scripts) { - auto format_path = path / "formats/*/*.lnav"; - static_root_mem<glob_t, globfree> gl; + for (auto format_path : + {path / "formats/*/*.lnav", path / "configs/*/*.lnav"}) + { + static_root_mem<glob_t, globfree> gl; - log_debug("Searching for script in path: %s", format_path.c_str()); - if (glob(format_path.c_str(), 0, nullptr, gl.inout()) == 0) { - for (int lpc = 0; lpc < (int) gl->gl_pathc; lpc++) { - const char* filename = basename(gl->gl_pathv[lpc]); - auto script_name = std::string(filename, strlen(filename) - 5); - struct script_metadata meta; + log_debug("Searching for script in path: %s", format_path.c_str()); + if (glob(format_path.c_str(), 0, nullptr, gl.inout()) == 0) { + for (int lpc = 0; lpc < (int) gl->gl_pathc; lpc++) { + const char* filename = basename(gl->gl_pathv[lpc]); + auto script_name = std::string(filename, strlen(filename) - 5); + struct script_metadata meta; - meta.sm_path = gl->gl_pathv[lpc]; - meta.sm_name = script_name; - extract_metadata_from_file(meta); - scripts.as_scripts[script_name].push_back(meta); + meta.sm_path = gl->gl_pathv[lpc]; + meta.sm_name = script_name; + extract_metadata_from_file(meta); + scripts.as_scripts[script_name].push_back(meta); - log_debug(" found script: %s", meta.sm_path.c_str()); + log_debug(" found script: %s", meta.sm_path.c_str()); + } } } } |