diff options
author | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-05-07 04:48:35 +0000 |
---|---|---|
committer | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-05-07 04:48:35 +0000 |
commit | 207df6fc406e81bfeebdff7f404bd242ff3f099f (patch) | |
tree | a1a796b056909dd0a04ffec163db9363a8757808 /src/log_format_impls.cc | |
parent | Releasing progress-linux version 0.11.2-1~progress7.99u1. (diff) | |
download | lnav-207df6fc406e81bfeebdff7f404bd242ff3f099f.tar.xz lnav-207df6fc406e81bfeebdff7f404bd242ff3f099f.zip |
Merging upstream version 0.12.2.
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'src/log_format_impls.cc')
-rw-r--r-- | src/log_format_impls.cc | 458 |
1 files changed, 381 insertions, 77 deletions
diff --git a/src/log_format_impls.cc b/src/log_format_impls.cc index 96d37f5..3bd45ad 100644 --- a/src/log_format_impls.cc +++ b/src/log_format_impls.cc @@ -41,17 +41,89 @@ #include "config.h" #include "formats/logfmt/logfmt.parser.hh" #include "log_vtab_impl.hh" +#include "scn/scn.h" #include "sql_util.hh" #include "yajlpp/yajlpp.hh" +class piper_log_format : public log_format { +public: + const intern_string_t get_name() const override + { + static const intern_string_t RETVAL + = intern_string::lookup("lnav_piper_log"); + + return RETVAL; + } + + scan_result_t scan(logfile& lf, + std::vector<logline>& dst, + const line_info& li, + shared_buffer_ref& sbr, + scan_batch_context& sbc) override + { + if (lf.has_line_metadata() + && lf.get_text_format() == text_format_t::TF_LOG) + { + dst.emplace_back( + li.li_file_range.fr_offset, li.li_timestamp, li.li_level); + return scan_match{100}; + } + + return scan_no_match{""}; + } + + void annotate(uint64_t line_number, + string_attrs_t& sa, + logline_value_vector& values, + bool annotate_module) const override + { + auto lr = line_range{0, 0}; + sa.emplace_back(lr, logline::L_TIMESTAMP.value()); + } + + void get_subline(const logline& ll, + shared_buffer_ref& sbr, + bool full_message) override + { + this->plf_cached_line.resize(23); + sql_strftime(this->plf_cached_line.data(), + this->plf_cached_line.size(), + ll.get_timeval(), + 'T'); + this->plf_cached_line.push_back(' '); + const auto prefix_len = this->plf_cached_line.size(); + this->plf_cached_line.resize(this->plf_cached_line.size() + + sbr.length()); + memcpy( + &this->plf_cached_line[prefix_len], sbr.get_data(), sbr.length()); + + sbr.share(this->plf_share_manager, + this->plf_cached_line.data(), + this->plf_cached_line.size()); + } + + std::shared_ptr<log_format> specialized(int fmt_lock) override + { + auto retval = std::make_shared<piper_log_format>(*this); + + retval->lf_specialized = true; + return retval; + } + +private: + shared_buffer plf_share_manager; + std::vector<char> plf_cached_line; +}; + class generic_log_format : public log_format { - static pcre_format* get_pcre_log_formats() +public: + static const pcre_format* get_pcre_log_formats() { - static pcre_format log_fmt[] = { + static const pcre_format log_fmt[] = { pcre_format( "^(?:\\*\\*\\*\\s+)?(?<timestamp>@[0-9a-zA-Z]{16,24})(.*)"), pcre_format( - "^(?:\\*\\*\\*\\s+)?(?<timestamp>[\\dTZ: +/\\-,\\.-]+)([^:]+)"), + R"(^(?:\*\*\*\s+)?(?<timestamp>(?:\s|\d{4}[\-\/]\d{2}[\-\/]\d{2}|T|\d{1,2}:\d{2}(?::\d{2}(?:[\.,]\d{1,6})?)?|Z|[+\-]\d{2}:?\d{2}|(?!ERR|INFO|WARN)[A-Z]{3,4})+)(?:\s+|[:|])([^:]+))"), pcre_format( "^(?:\\*\\*\\*\\s+)?(?<timestamp>[\\w:+/\\.-]+) \\[\\w (.*)"), pcre_format("^(?:\\*\\*\\*\\s+)?(?<timestamp>[\\w:,/\\.-]+) (.*)"), @@ -88,7 +160,10 @@ class generic_log_format : public log_format { const intern_string_t get_name() const override { - return intern_string::lookup("generic_log"); + static const intern_string_t RETVAL + = intern_string::lookup("generic_log"); + + return RETVAL; } scan_result_t scan(logfile& lf, @@ -103,6 +178,17 @@ class generic_log_format : public log_format { nonstd::optional<string_fragment> level; const char* last_pos; + if (dst.empty()) { + auto file_options = lf.get_file_options(); + + if (file_options) { + this->lf_date_time.dts_default_zone + = file_options->second.fo_default_zone.pp_value; + } else { + this->lf_date_time.dts_default_zone = nullptr; + } + } + if ((last_pos = this->log_scanf(dst.size(), sbr.to_string_fragment(), get_pcre_log_formats(), @@ -126,6 +212,24 @@ class generic_log_format : public log_format { this->check_for_new_year(dst, log_time, log_tv); } + if (!(this->lf_timestamp_flags + & (ETF_MILLIS_SET | ETF_MICROS_SET | ETF_NANOS_SET)) + && !dst.empty() && dst.back().get_time() == log_tv.tv_sec + && dst.back().get_millis() != 0) + { + auto log_ms + = std::chrono::milliseconds(dst.back().get_millis()); + + log_time.et_nsec + = std::chrono::duration_cast<std::chrono::nanoseconds>( + log_ms) + .count(); + log_tv.tv_usec + = std::chrono::duration_cast<std::chrono::microseconds>( + log_ms) + .count(); + } + dst.emplace_back(li.li_file_range.fr_offset, log_tv, level_val); return scan_match{0}; } @@ -140,7 +244,7 @@ class generic_log_format : public log_format { { auto& line = values.lvv_sbr; int pat_index = this->pattern_index_for_line(line_number); - auto& fmt = get_pcre_log_formats()[pat_index]; + const auto& fmt = get_pcre_log_formats()[pat_index]; int prefix_len = 0; auto md = fmt.pcre->create_match_data(); auto match_res = fmt.pcre->capture_from(line.to_string_fragment()) @@ -151,16 +255,25 @@ class generic_log_format : public log_format { return; } - auto lr = to_line_range(md[fmt.pf_timestamp_index].value()); + auto ts_cap = md[fmt.pf_timestamp_index].value(); + auto lr = to_line_range(ts_cap.trim()); sa.emplace_back(lr, logline::L_TIMESTAMP.value()); - prefix_len = lr.lr_end; + values.lvv_values.emplace_back(TS_META, line, lr); + values.lvv_values.back().lv_meta.lvm_format = (log_format*) this; + + prefix_len = ts_cap.sf_end; auto level_cap = md[2]; if (level_cap) { if (string2level(level_cap->data(), level_cap->length(), true) != LEVEL_UNKNOWN) { prefix_len = level_cap->sf_end; + + values.lvv_values.emplace_back( + LEVEL_META, line, to_line_range(level_cap->trim())); + values.lvv_values.back().lv_meta.lvm_format + = (log_format*) this; } } @@ -180,6 +293,41 @@ class generic_log_format : public log_format { retval->lf_specialized = true; return retval; } + + bool hide_field(const intern_string_t field_name, bool val) override + { + if (field_name == TS_META.lvm_name) { + TS_META.lvm_user_hidden = val; + return true; + } else if (field_name == LEVEL_META.lvm_name) { + LEVEL_META.lvm_user_hidden = val; + return true; + } + return false; + } + + std::map<intern_string_t, logline_value_meta> get_field_states() override + { + return { + {TS_META.lvm_name, TS_META}, + {LEVEL_META.lvm_name, LEVEL_META}, + }; + } + +private: + static logline_value_meta TS_META; + static logline_value_meta LEVEL_META; +}; + +logline_value_meta generic_log_format::TS_META{ + intern_string::lookup("log_time"), + value_kind_t::VALUE_TEXT, + logline_value_meta::table_column{2}, +}; +logline_value_meta generic_log_format::LEVEL_META{ + intern_string::lookup("log_level"), + value_kind_t::VALUE_TEXT, + logline_value_meta::table_column{4}, }; std::string @@ -324,22 +472,29 @@ class bro_log_format : public log_format { public: struct field_def { logline_value_meta fd_meta; + logline_value_meta* fd_root_meta; std::string fd_collator; nonstd::optional<size_t> fd_numeric_index; explicit field_def(const intern_string_t name, - int col, + size_t col, log_format* format) - : fd_meta(name, value_kind_t::VALUE_TEXT, col, format) + : fd_meta(name, + value_kind_t::VALUE_TEXT, + logline_value_meta::table_column{col}, + format), + fd_root_meta(&FIELD_META.find(name)->second) { } field_def& with_kind(value_kind_t kind, bool identifier = false, + bool foreign_key = false, const std::string& collator = "") { this->fd_meta.lvm_kind = kind; this->fd_meta.lvm_identifier = identifier; + this->fd_meta.lvm_foreign_key = foreign_key; this->fd_collator = collator; return *this; } @@ -351,10 +506,26 @@ public: } }; + static std::unordered_map<const intern_string_t, logline_value_meta> + FIELD_META; + + static const intern_string_t get_opid_desc() + { + static const intern_string_t RETVAL = intern_string::lookup("std"); + + return RETVAL; + } + bro_log_format() { + this->lf_structured = true; this->lf_is_self_describing = true; this->lf_time_ordered = false; + + auto desc_v = std::make_shared<std::vector<opid_descriptor>>(); + desc_v->emplace({}); + this->lf_opid_description_def->emplace(get_opid_desc(), + opid_descriptors{desc_v}); } const intern_string_t get_name() const override @@ -373,12 +544,15 @@ public: scan_result_t scan_int(std::vector<logline>& dst, const line_info& li, - shared_buffer_ref& sbr) + shared_buffer_ref& sbr, + scan_batch_context& sbc) { static const intern_string_t STATUS_CODE = intern_string::lookup("bro_status_code"); static const intern_string_t TS = intern_string::lookup("bro_ts"); static const intern_string_t UID = intern_string::lookup("bro_uid"); + static const intern_string_t ID_ORIG_H + = intern_string::lookup("bro_id_orig_h"); separated_string ss(sbr.get_data(), sbr.length()); struct timeval tv; @@ -386,6 +560,8 @@ public: bool found_ts = false; log_level_t level = LEVEL_INFO; uint8_t opid = 0; + auto opid_cap = string_fragment::invalid(); + auto host_cap = string_fragment::invalid(); ss.with_separator(this->blf_separator.get()); @@ -410,29 +586,28 @@ public: found_ts = true; } } else if (STATUS_CODE == fd.fd_meta.lvm_name) { - string_fragment sf = *iter; + const auto sf = *iter; if (!sf.empty() && sf[0] >= '4') { level = LEVEL_ERROR; } } else if (UID == fd.fd_meta.lvm_name) { - string_fragment sf = *iter; + opid_cap = *iter; - opid = hash_str(sf.data(), sf.length()); + opid = hash_str(opid_cap.data(), opid_cap.length()); + } else if (ID_ORIG_H == fd.fd_meta.lvm_name) { + host_cap = *iter; } if (fd.fd_numeric_index) { switch (fd.fd_meta.lvm_kind) { case value_kind_t::VALUE_INTEGER: case value_kind_t::VALUE_FLOAT: { - string_fragment sf = *iter; - char field_copy[sf.length() + 1]; - double val; - - if (sscanf(sf.to_string(field_copy), "%lf", &val) == 1) - { + const auto sv = (*iter).to_string_view(); + auto scan_float_res = scn::scan_value<double>(sv); + if (scan_float_res) { this->lf_value_stats[fd.fd_numeric_index.value()] - .add_value(val); + .add_value(scan_float_res.value()); } break; } @@ -448,6 +623,31 @@ public: ll.set_ignore(true); } } + + if (opid_cap.is_valid()) { + auto opid_iter = sbc.sbc_opids.los_opid_ranges.find(opid_cap); + + if (opid_iter == sbc.sbc_opids.los_opid_ranges.end()) { + auto opid_copy = opid_cap.to_owned(sbc.sbc_allocator); + auto otr = opid_time_range{time_range{tv, tv}}; + auto emplace_res + = sbc.sbc_opids.los_opid_ranges.emplace(opid_copy, otr); + opid_iter = emplace_res.first; + } else { + opid_iter->second.otr_range.extend_to(tv); + } + + opid_iter->second.otr_level_stats.update_msg_count(level); + + auto& otr = opid_iter->second; + if (!otr.otr_description.lod_id && host_cap.is_valid() + && otr.otr_description.lod_elements.empty()) + { + otr.otr_description.lod_id = get_opid_desc(); + otr.otr_description.lod_elements.emplace_back( + 0, host_cap.to_string()); + } + } dst.emplace_back(li.li_file_range.fr_offset, tv, level, 0, opid); return scan_match{0}; } @@ -463,8 +663,19 @@ public: static const auto SEP_RE = lnav::pcre2pp::code::from_const(R"(^#separator\s+(.+))"); + if (dst.empty()) { + auto file_options = lf.get_file_options(); + + if (file_options) { + this->lf_date_time.dts_default_zone + = file_options->second.fo_default_zone.pp_value; + } else { + this->lf_date_time.dts_default_zone = nullptr; + } + } + if (!this->blf_format_name.empty()) { - return this->scan_int(dst, li, sbr); + return this->scan_int(dst, li, sbr, sbc); } if (dst.empty() || dst.size() > 20 || sbr.empty() @@ -531,10 +742,18 @@ public: this->blf_format_name = intern_string::lookup(full_name); } else if (directive == "#fields" && this->blf_field_defs.empty()) { do { + auto field_name + = intern_string::lookup("bro_" + sql_safe_ident(*iter)); + auto common_iter = FIELD_META.find(field_name); + if (common_iter == FIELD_META.end()) { + FIELD_META.emplace(field_name, + logline_value_meta{ + field_name, + value_kind_t::VALUE_TEXT, + }); + } this->blf_field_defs.emplace_back( - intern_string::lookup("bro_" + sql_safe_ident(*iter)), - this->blf_field_defs.size(), - this); + field_name, this->blf_field_defs.size(), this); ++iter; } while (iter != ss.end()); } else if (directive == "#types") { @@ -551,12 +770,14 @@ public: "bro_referrer", "bro_resp_fuids", "bro_service", - "bro_status_code", "bro_uid", "bro_uri", "bro_user_agent", "bro_username", }; + static const char* KNOWN_FOREIGN[] = { + "bro_status_code", + }; int numeric_count = 0; @@ -575,7 +796,12 @@ public: bool ident = std::binary_search(std::begin(KNOWN_IDS), std::end(KNOWN_IDS), fd.fd_meta.lvm_name); - fd.with_kind(value_kind_t::VALUE_INTEGER, ident) + bool foreign + = std::binary_search(std::begin(KNOWN_FOREIGN), + std::end(KNOWN_FOREIGN), + fd.fd_meta.lvm_name); + fd.with_kind( + value_kind_t::VALUE_INTEGER, ident, foreign) .with_numeric_index(numeric_count); numeric_count += 1; } else if (field_type == "bool") { @@ -601,8 +827,7 @@ public: if (!this->blf_format_name.empty() && !this->blf_separator.empty() && !this->blf_field_defs.empty()) { - dst.clear(); - return this->scan_int(dst, li, sbr); + return this->scan_int(dst, li, sbr, sbc); } this->blf_format_name.clear(); @@ -651,6 +876,8 @@ public: } else { values.lvv_values.emplace_back(fd.fd_meta); } + values.lvv_values.back().lv_meta.lvm_user_hidden + = fd.fd_root_meta->lvm_user_hidden; } } @@ -675,20 +902,27 @@ public: bool hide_field(const intern_string_t field_name, bool val) override { - auto fd_iter - = std::find_if(this->blf_field_defs.begin(), - this->blf_field_defs.end(), - [field_name](const field_def& elem) { - return elem.fd_meta.lvm_name == field_name; - }); - if (fd_iter == this->blf_field_defs.end()) { + auto fd_iter = FIELD_META.find(field_name); + if (fd_iter == FIELD_META.end()) { return false; } - fd_iter->fd_meta.lvm_user_hidden = val; + fd_iter->second.lvm_user_hidden = val; + return true; } + std::map<intern_string_t, logline_value_meta> get_field_states() override + { + std::map<intern_string_t, logline_value_meta> retval; + + for (const auto& fd : FIELD_META) { + retval.emplace(fd.first, fd.second); + } + + return retval; + } + std::shared_ptr<log_format> specialized(int fmt_lock = -1) override { auto retval = std::make_shared<bro_log_format>(*this); @@ -707,9 +941,8 @@ public: void get_columns(std::vector<vtab_column>& cols) const override { for (const auto& fd : this->blt_format.blf_field_defs) { - std::pair<int, unsigned int> type_pair - = log_vtab_impl::logline_value_to_sqlite_type( - fd.fd_meta.lvm_kind); + auto type_pair = log_vtab_impl::logline_value_to_sqlite_type( + fd.fd_meta.lvm_kind); cols.emplace_back(fd.fd_meta.lvm_name.to_string(), type_pair.first, @@ -726,7 +959,7 @@ public: this->log_vtab_impl::get_foreign_keys(keys_inout); for (const auto& fd : this->blt_format.blf_field_defs) { - if (fd.fd_meta.lvm_identifier) { + if (fd.fd_meta.lvm_identifier || fd.fd_meta.lvm_foreign_key) { keys_inout.push_back(fd.fd_meta.lvm_name.to_string()); } } @@ -775,6 +1008,9 @@ public: std::vector<field_def> blf_field_defs; }; +std::unordered_map<const intern_string_t, logline_value_meta> + bro_log_format::FIELD_META; + struct ws_separated_string { const char* ss_str; size_t ss_len; @@ -877,6 +1113,7 @@ public: struct field_def { const intern_string_t fd_name; logline_value_meta fd_meta; + logline_value_meta* fd_root_meta{nullptr}; std::string fd_collator; nonstd::optional<size_t> fd_numeric_index; @@ -892,19 +1129,21 @@ public: { } - field_def(int col, + field_def(size_t col, const char* name, value_kind_t kind, bool ident = false, + bool foreign_key = false, std::string coll = "") : fd_name(intern_string::lookup(name)), fd_meta( intern_string::lookup(sql_safe_ident(string_fragment(name))), kind, - col), + logline_value_meta::table_column{col}), fd_collator(std::move(coll)) { this->fd_meta.lvm_identifier = ident; + this->fd_meta.lvm_foreign_key = foreign_key; } field_def& with_kind(value_kind_t kind, @@ -924,6 +1163,9 @@ public: } }; + static std::unordered_map<const intern_string_t, logline_value_meta> + FIELD_META; + struct field_to_struct_t { field_to_struct_t(const char* prefix, const char* struct_name) : fs_prefix(prefix), @@ -942,6 +1184,7 @@ public: { this->lf_is_self_describing = true; this->lf_time_ordered = false; + this->lf_structured = true; } const intern_string_t get_name() const override @@ -990,7 +1233,7 @@ public: break; } - const field_def& fd = this->wlf_field_defs[iter.index()]; + const auto& fd = this->wlf_field_defs[iter.index()]; string_fragment sf = *iter; if (sf.startswith("#")) { @@ -1051,13 +1294,12 @@ public: switch (fd.fd_meta.lvm_kind) { case value_kind_t::VALUE_INTEGER: case value_kind_t::VALUE_FLOAT: { - char field_copy[sf.length() + 1]; - double val; + auto scan_float_res + = scn::scan_value<double>(sf.to_string_view()); - if (sscanf(sf.to_string(field_copy), "%lf", &val) == 1) - { + if (scan_float_res) { this->lf_value_stats[fd.fd_numeric_index.value()] - .add_value(val); + .add_value(scan_float_res.value()); } break; } @@ -1079,9 +1321,7 @@ public: tm.et_tm.tm_yday = date_tm.et_tm.tm_yday; } - tv.tv_sec = tm2sec(&tm.et_tm); - tv.tv_usec = tm.et_nsec / 1000; - + tv = tm.to_timeval(); if (!this->lf_specialized) { for (auto& ll : dst) { ll.set_ignore(true); @@ -1108,6 +1348,17 @@ public: return scan_incomplete{}; } + if (dst.empty()) { + auto file_options = lf.get_file_options(); + + if (file_options) { + this->lf_date_time.dts_default_zone + = file_options->second.fo_default_zone.pp_value; + } else { + this->lf_date_time.dts_default_zone = nullptr; + } + } + if (!this->wlf_format_name.empty()) { return this->scan_int(dst, li, sbr); } @@ -1131,8 +1382,7 @@ public: auto line = next_read_result.unwrap(); ws_separated_string ss(line.get_data(), line.length()); auto iter = ss.begin(); - - string_fragment directive = *iter; + const auto directive = *iter; if (directive.empty() || directive[0] != '#') { continue; @@ -1170,9 +1420,25 @@ public: [&sf](auto elem) { return sf == elem.fd_name; }); if (field_iter != end(KNOWN_FIELDS)) { this->wlf_field_defs.emplace_back(*field_iter); + auto& fd = this->wlf_field_defs.back(); + auto common_iter = FIELD_META.find(fd.fd_meta.lvm_name); + if (common_iter == FIELD_META.end()) { + auto emp_res = FIELD_META.emplace( + fd.fd_meta.lvm_name, fd.fd_meta); + common_iter = emp_res.first; + } + fd.fd_root_meta = &common_iter->second; } else if (sf == "date" || sf == "time") { this->wlf_field_defs.emplace_back( intern_string::lookup(sf)); + auto& fd = this->wlf_field_defs.back(); + auto common_iter = FIELD_META.find(fd.fd_meta.lvm_name); + if (common_iter == FIELD_META.end()) { + auto emp_res = FIELD_META.emplace( + fd.fd_meta.lvm_name, fd.fd_meta); + common_iter = emp_res.first; + } + fd.fd_root_meta = &common_iter->second; } else { const auto fs_iter = std::find_if( begin(KNOWN_STRUCT_FIELDS), @@ -1181,27 +1447,30 @@ public: return sf.startswith(elem.fs_prefix); }); if (fs_iter != end(KNOWN_STRUCT_FIELDS)) { - auto field_name + const intern_string_t field_name = intern_string::lookup(sf.substr(3)); this->wlf_field_defs.emplace_back( field_name, logline_value_meta( field_name, value_kind_t::VALUE_TEXT, - KNOWN_FIELDS.size() + 1 + logline_value_meta::table_column{ + KNOWN_FIELDS.size() + 1 + std::distance( begin(KNOWN_STRUCT_FIELDS), - fs_iter), + fs_iter)}, this) .with_struct_name(fs_iter->fs_struct_name)); } else { - auto field_name = intern_string::lookup(sf); + const intern_string_t field_name + = intern_string::lookup(sf); this->wlf_field_defs.emplace_back( field_name, logline_value_meta( field_name, value_kind_t::VALUE_TEXT, - KNOWN_FIELDS.size() + X_FIELDS_IDX, + logline_value_meta::table_column{ + KNOWN_FIELDS.size() + X_FIELDS_IDX}, this) .with_struct_name(X_FIELDS_NAME)); } @@ -1253,7 +1522,7 @@ public: return; } - const field_def& fd = this->wlf_field_defs[iter.index()]; + const auto& fd = this->wlf_field_defs[iter.index()]; if (sf == "-") { sf.invalidate(); @@ -1275,6 +1544,10 @@ public: } else { values.lvv_values.emplace_back(fd.fd_meta); } + if (fd.fd_root_meta != nullptr) { + values.lvv_values.back().lv_meta.lvm_user_hidden + = fd.fd_root_meta->lvm_user_hidden; + } } } @@ -1299,20 +1572,27 @@ public: bool hide_field(const intern_string_t field_name, bool val) override { - auto fd_iter - = std::find_if(this->wlf_field_defs.begin(), - this->wlf_field_defs.end(), - [field_name](const field_def& elem) { - return elem.fd_meta.lvm_name == field_name; - }); - if (fd_iter == this->wlf_field_defs.end()) { + auto fd_iter = FIELD_META.find(field_name); + if (fd_iter == FIELD_META.end()) { return false; } - fd_iter->fd_meta.lvm_user_hidden = val; + fd_iter->second.lvm_user_hidden = val; + return true; } + std::map<intern_string_t, logline_value_meta> get_field_states() override + { + std::map<intern_string_t, logline_value_meta> retval; + + for (const auto& fd : FIELD_META) { + retval.emplace(fd.first, fd.second); + } + + return retval; + } + std::shared_ptr<log_format> specialized(int fmt_lock = -1) override { auto retval = std::make_shared<w3c_log_format>(*this); @@ -1356,7 +1636,7 @@ public: this->log_vtab_impl::get_foreign_keys(keys_inout); for (const auto& fd : KNOWN_FIELDS) { - if (fd.fd_meta.lvm_identifier) { + if (fd.fd_meta.lvm_identifier || fd.fd_meta.lvm_foreign_key) { keys_inout.push_back(fd.fd_meta.lvm_name.to_string()); } } @@ -1402,7 +1682,10 @@ public: std::vector<field_def> wlf_field_defs; }; -static int KNOWN_FIELD_INDEX = 0; +std::unordered_map<const intern_string_t, logline_value_meta> + w3c_log_format::FIELD_META; + +static size_t KNOWN_FIELD_INDEX = 0; const std::vector<w3c_log_format::field_def> w3c_log_format::KNOWN_FIELDS = { { KNOWN_FIELD_INDEX++, @@ -1415,6 +1698,7 @@ const std::vector<w3c_log_format::field_def> w3c_log_format::KNOWN_FIELDS = { "c-ip", value_kind_t::VALUE_TEXT, true, + false, "ipaddress", }, { @@ -1434,6 +1718,7 @@ const std::vector<w3c_log_format::field_def> w3c_log_format::KNOWN_FIELDS = { "cs-uri-stem", value_kind_t::VALUE_TEXT, true, + false, "naturalnocase", }, { @@ -1459,6 +1744,7 @@ const std::vector<w3c_log_format::field_def> w3c_log_format::KNOWN_FIELDS = { "s-ip", value_kind_t::VALUE_TEXT, true, + false, "ipaddress", }, { @@ -1490,6 +1776,7 @@ const std::vector<w3c_log_format::field_def> w3c_log_format::KNOWN_FIELDS = { "sc-status", value_kind_t::VALUE_INTEGER, false, + true, }, { KNOWN_FIELD_INDEX++, @@ -1551,7 +1838,7 @@ class logfmt_format : public log_format { public: const intern_string_t get_name() const override { - const static auto NAME = intern_string::lookup("logfmt_log"); + const static intern_string_t NAME = intern_string::lookup("logfmt_log"); return NAME; } @@ -1589,6 +1876,17 @@ public: bool done = false; logfmt_pair_handler lph(this->lf_date_time); + if (dst.empty()) { + auto file_options = lf.get_file_options(); + + if (file_options) { + this->lf_date_time.dts_default_zone + = file_options->second.fo_default_zone.pp_value; + } else { + this->lf_date_time.dts_default_zone = nullptr; + } + } + while (!done) { auto parse_result = p.step(); @@ -1685,7 +1983,8 @@ public: kvp.first), value_kind_t:: VALUE_INTEGER, - 0, + logline_value_meta:: + table_column{0}, (log_format*) this} .with_struct_name(FIELDS_NAME); values.lvv_values.emplace_back(lvm, bv.bv_value); @@ -1698,7 +1997,8 @@ public: kvp.first), value_kind_t:: VALUE_INTEGER, - 0, + logline_value_meta:: + table_column{0}, (log_format*) this} .with_struct_name(FIELDS_NAME); values.lvv_values.emplace_back(lvm, iv.iv_value); @@ -1711,7 +2011,8 @@ public: kvp.first), value_kind_t:: VALUE_INTEGER, - 0, + logline_value_meta:: + table_column{0}, (log_format*) this} .with_struct_name(FIELDS_NAME); values.lvv_values.emplace_back(lvm, fv.fv_value); @@ -1732,8 +2033,9 @@ public: } else if (kvp.first == "level") { } else if (kvp.first == "msg") { sa.emplace_back(value_lr, SA_BODY.value()); - } else if (!kvp.second.is<logfmt::parser::int_value>() - && !kvp.second.is<logfmt::parser::bool_value>()) + } else if (kvp.second.is<logfmt::parser::quoted_value>() + || kvp.second + .is<logfmt::parser::unquoted_value>()) { auto lvm = logline_value_meta{intern_string::lookup( @@ -1741,7 +2043,8 @@ public: value_frag.startswith("\"") ? value_kind_t::VALUE_JSON : value_kind_t::VALUE_TEXT, - 0, + logline_value_meta:: + table_column{0}, (log_format*) this} .with_struct_name(FIELDS_NAME); values.lvv_values.emplace_back(lvm, value_frag); @@ -1772,4 +2075,5 @@ static auto format_binder = injector::bind_multiple<log_format>() .add<logfmt_format>() .add<bro_log_format>() .add<w3c_log_format>() - .add<generic_log_format>(); + .add<generic_log_format>() + .add<piper_log_format>(); |