Merging upstream version 0.12.2.

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

1 files changed, 381 insertions, 77 deletions

diff --git a/src/log_format_impls.cc b/src/log_format_impls.cc
index 96d37f5..3bd45ad 100644
--- a/src/log_format_impls.cc
+++ b/src/log_format_impls.cc
@@ -41,17 +41,89 @@
 #include "config.h"
 #include "formats/logfmt/logfmt.parser.hh"
 #include "log_vtab_impl.hh"
+#include "scn/scn.h"
 #include "sql_util.hh"
 #include "yajlpp/yajlpp.hh"
 
+class piper_log_format : public log_format {
+public:
+    const intern_string_t get_name() const override
+    {
+        static const intern_string_t RETVAL
+            = intern_string::lookup("lnav_piper_log");
+
+        return RETVAL;
+    }
+
+    scan_result_t scan(logfile& lf,
+                       std::vector<logline>& dst,
+                       const line_info& li,
+                       shared_buffer_ref& sbr,
+                       scan_batch_context& sbc) override
+    {
+        if (lf.has_line_metadata()
+            && lf.get_text_format() == text_format_t::TF_LOG)
+        {
+            dst.emplace_back(
+                li.li_file_range.fr_offset, li.li_timestamp, li.li_level);
+            return scan_match{100};
+        }
+
+        return scan_no_match{""};
+    }
+
+    void annotate(uint64_t line_number,
+                  string_attrs_t& sa,
+                  logline_value_vector& values,
+                  bool annotate_module) const override
+    {
+        auto lr = line_range{0, 0};
+        sa.emplace_back(lr, logline::L_TIMESTAMP.value());
+    }
+
+    void get_subline(const logline& ll,
+                     shared_buffer_ref& sbr,
+                     bool full_message) override
+    {
+        this->plf_cached_line.resize(23);
+        sql_strftime(this->plf_cached_line.data(),
+                     this->plf_cached_line.size(),
+                     ll.get_timeval(),
+                     'T');
+        this->plf_cached_line.push_back(' ');
+        const auto prefix_len = this->plf_cached_line.size();
+        this->plf_cached_line.resize(this->plf_cached_line.size()
+                                     + sbr.length());
+        memcpy(
+            &this->plf_cached_line[prefix_len], sbr.get_data(), sbr.length());
+
+        sbr.share(this->plf_share_manager,
+                  this->plf_cached_line.data(),
+                  this->plf_cached_line.size());
+    }
+
+    std::shared_ptr<log_format> specialized(int fmt_lock) override
+    {
+        auto retval = std::make_shared<piper_log_format>(*this);
+
+        retval->lf_specialized = true;
+        return retval;
+    }
+
+private:
+    shared_buffer plf_share_manager;
+    std::vector<char> plf_cached_line;
+};
+
 class generic_log_format : public log_format {
-    static pcre_format* get_pcre_log_formats()
+public:
+    static const pcre_format* get_pcre_log_formats()
     {
-        static pcre_format log_fmt[] = {
+        static const pcre_format log_fmt[] = {
             pcre_format(
                 "^(?:\\*\\*\\*\\s+)?(?<timestamp>@[0-9a-zA-Z]{16,24})(.*)"),
             pcre_format(
-                "^(?:\\*\\*\\*\\s+)?(?<timestamp>[\\dTZ: +/\\-,\\.-]+)([^:]+)"),
+                R"(^(?:\*\*\*\s+)?(?<timestamp>(?:\s|\d{4}[\-\/]\d{2}[\-\/]\d{2}|T|\d{1,2}:\d{2}(?::\d{2}(?:[\.,]\d{1,6})?)?|Z|[+\-]\d{2}:?\d{2}|(?!ERR|INFO|WARN)[A-Z]{3,4})+)(?:\s+|[:|])([^:]+))"),
             pcre_format(
                 "^(?:\\*\\*\\*\\s+)?(?<timestamp>[\\w:+/\\.-]+) \\[\\w (.*)"),
             pcre_format("^(?:\\*\\*\\*\\s+)?(?<timestamp>[\\w:,/\\.-]+) (.*)"),
@@ -88,7 +160,10 @@ class generic_log_format : public log_format {
 
     const intern_string_t get_name() const override
     {
-        return intern_string::lookup("generic_log");
+        static const intern_string_t RETVAL
+            = intern_string::lookup("generic_log");
+
+        return RETVAL;
     }
 
     scan_result_t scan(logfile& lf,
@@ -103,6 +178,17 @@ class generic_log_format : public log_format {
         nonstd::optional<string_fragment> level;
         const char* last_pos;
 
+        if (dst.empty()) {
+            auto file_options = lf.get_file_options();
+
+            if (file_options) {
+                this->lf_date_time.dts_default_zone
+                    = file_options->second.fo_default_zone.pp_value;
+            } else {
+                this->lf_date_time.dts_default_zone = nullptr;
+            }
+        }
+
         if ((last_pos = this->log_scanf(dst.size(),
                                         sbr.to_string_fragment(),
                                         get_pcre_log_formats(),
@@ -126,6 +212,24 @@ class generic_log_format : public log_format {
                 this->check_for_new_year(dst, log_time, log_tv);
             }
 
+            if (!(this->lf_timestamp_flags
+                  & (ETF_MILLIS_SET | ETF_MICROS_SET | ETF_NANOS_SET))
+                && !dst.empty() && dst.back().get_time() == log_tv.tv_sec
+                && dst.back().get_millis() != 0)
+            {
+                auto log_ms
+                    = std::chrono::milliseconds(dst.back().get_millis());
+
+                log_time.et_nsec
+                    = std::chrono::duration_cast<std::chrono::nanoseconds>(
+                          log_ms)
+                          .count();
+                log_tv.tv_usec
+                    = std::chrono::duration_cast<std::chrono::microseconds>(
+                          log_ms)
+                          .count();
+            }
+
             dst.emplace_back(li.li_file_range.fr_offset, log_tv, level_val);
             return scan_match{0};
         }
@@ -140,7 +244,7 @@ class generic_log_format : public log_format {
     {
         auto& line = values.lvv_sbr;
         int pat_index = this->pattern_index_for_line(line_number);
-        auto& fmt = get_pcre_log_formats()[pat_index];
+        const auto& fmt = get_pcre_log_formats()[pat_index];
         int prefix_len = 0;
         auto md = fmt.pcre->create_match_data();
         auto match_res = fmt.pcre->capture_from(line.to_string_fragment())
@@ -151,16 +255,25 @@ class generic_log_format : public log_format {
             return;
         }
 
-        auto lr = to_line_range(md[fmt.pf_timestamp_index].value());
+        auto ts_cap = md[fmt.pf_timestamp_index].value();
+        auto lr = to_line_range(ts_cap.trim());
         sa.emplace_back(lr, logline::L_TIMESTAMP.value());
 
-        prefix_len = lr.lr_end;
+        values.lvv_values.emplace_back(TS_META, line, lr);
+        values.lvv_values.back().lv_meta.lvm_format = (log_format*) this;
+
+        prefix_len = ts_cap.sf_end;
         auto level_cap = md[2];
         if (level_cap) {
             if (string2level(level_cap->data(), level_cap->length(), true)
                 != LEVEL_UNKNOWN)
             {
                 prefix_len = level_cap->sf_end;
+
+                values.lvv_values.emplace_back(
+                    LEVEL_META, line, to_line_range(level_cap->trim()));
+                values.lvv_values.back().lv_meta.lvm_format
+                    = (log_format*) this;
             }
         }
 
@@ -180,6 +293,41 @@ class generic_log_format : public log_format {
         retval->lf_specialized = true;
         return retval;
     }
+
+    bool hide_field(const intern_string_t field_name, bool val) override
+    {
+        if (field_name == TS_META.lvm_name) {
+            TS_META.lvm_user_hidden = val;
+            return true;
+        } else if (field_name == LEVEL_META.lvm_name) {
+            LEVEL_META.lvm_user_hidden = val;
+            return true;
+        }
+        return false;
+    }
+
+    std::map<intern_string_t, logline_value_meta> get_field_states() override
+    {
+        return {
+            {TS_META.lvm_name, TS_META},
+            {LEVEL_META.lvm_name, LEVEL_META},
+        };
+    }
+
+private:
+    static logline_value_meta TS_META;
+    static logline_value_meta LEVEL_META;
+};
+
+logline_value_meta generic_log_format::TS_META{
+    intern_string::lookup("log_time"),
+    value_kind_t::VALUE_TEXT,
+    logline_value_meta::table_column{2},
+};
+logline_value_meta generic_log_format::LEVEL_META{
+    intern_string::lookup("log_level"),
+    value_kind_t::VALUE_TEXT,
+    logline_value_meta::table_column{4},
 };
 
 std::string
@@ -324,22 +472,29 @@ class bro_log_format : public log_format {
 public:
     struct field_def {
         logline_value_meta fd_meta;
+        logline_value_meta* fd_root_meta;
         std::string fd_collator;
         nonstd::optional<size_t> fd_numeric_index;
 
         explicit field_def(const intern_string_t name,
-                           int col,
+                           size_t col,
                            log_format* format)
-            : fd_meta(name, value_kind_t::VALUE_TEXT, col, format)
+            : fd_meta(name,
+                      value_kind_t::VALUE_TEXT,
+                      logline_value_meta::table_column{col},
+                      format),
+              fd_root_meta(&FIELD_META.find(name)->second)
         {
         }
 
         field_def& with_kind(value_kind_t kind,
                              bool identifier = false,
+                             bool foreign_key = false,
                              const std::string& collator = "")
         {
             this->fd_meta.lvm_kind = kind;
             this->fd_meta.lvm_identifier = identifier;
+            this->fd_meta.lvm_foreign_key = foreign_key;
             this->fd_collator = collator;
             return *this;
         }
@@ -351,10 +506,26 @@ public:
         }
     };
 
+    static std::unordered_map<const intern_string_t, logline_value_meta>
+        FIELD_META;
+
+    static const intern_string_t get_opid_desc()
+    {
+        static const intern_string_t RETVAL = intern_string::lookup("std");
+
+        return RETVAL;
+    }
+
     bro_log_format()
     {
+        this->lf_structured = true;
         this->lf_is_self_describing = true;
         this->lf_time_ordered = false;
+
+        auto desc_v = std::make_shared<std::vector<opid_descriptor>>();
+        desc_v->emplace({});
+        this->lf_opid_description_def->emplace(get_opid_desc(),
+                                               opid_descriptors{desc_v});
     }
 
     const intern_string_t get_name() const override
@@ -373,12 +544,15 @@ public:
 
     scan_result_t scan_int(std::vector<logline>& dst,
                            const line_info& li,
-                           shared_buffer_ref& sbr)
+                           shared_buffer_ref& sbr,
+                           scan_batch_context& sbc)
     {
         static const intern_string_t STATUS_CODE
             = intern_string::lookup("bro_status_code");
         static const intern_string_t TS = intern_string::lookup("bro_ts");
         static const intern_string_t UID = intern_string::lookup("bro_uid");
+        static const intern_string_t ID_ORIG_H
+            = intern_string::lookup("bro_id_orig_h");
 
         separated_string ss(sbr.get_data(), sbr.length());
         struct timeval tv;
@@ -386,6 +560,8 @@ public:
         bool found_ts = false;
         log_level_t level = LEVEL_INFO;
         uint8_t opid = 0;
+        auto opid_cap = string_fragment::invalid();
+        auto host_cap = string_fragment::invalid();
 
         ss.with_separator(this->blf_separator.get());
 
@@ -410,29 +586,28 @@ public:
                     found_ts = true;
                 }
             } else if (STATUS_CODE == fd.fd_meta.lvm_name) {
-                string_fragment sf = *iter;
+                const auto sf = *iter;
 
                 if (!sf.empty() && sf[0] >= '4') {
                     level = LEVEL_ERROR;
                 }
             } else if (UID == fd.fd_meta.lvm_name) {
-                string_fragment sf = *iter;
+                opid_cap = *iter;
 
-                opid = hash_str(sf.data(), sf.length());
+                opid = hash_str(opid_cap.data(), opid_cap.length());
+            } else if (ID_ORIG_H == fd.fd_meta.lvm_name) {
+                host_cap = *iter;
             }
 
             if (fd.fd_numeric_index) {
                 switch (fd.fd_meta.lvm_kind) {
                     case value_kind_t::VALUE_INTEGER:
                     case value_kind_t::VALUE_FLOAT: {
-                        string_fragment sf = *iter;
-                        char field_copy[sf.length() + 1];
-                        double val;
-
-                        if (sscanf(sf.to_string(field_copy), "%lf", &val) == 1)
-                        {
+                        const auto sv = (*iter).to_string_view();
+                        auto scan_float_res = scn::scan_value<double>(sv);
+                        if (scan_float_res) {
                             this->lf_value_stats[fd.fd_numeric_index.value()]
-                                .add_value(val);
+                                .add_value(scan_float_res.value());
                         }
                         break;
                     }
@@ -448,6 +623,31 @@ public:
                     ll.set_ignore(true);
                 }
             }
+
+            if (opid_cap.is_valid()) {
+                auto opid_iter = sbc.sbc_opids.los_opid_ranges.find(opid_cap);
+
+                if (opid_iter == sbc.sbc_opids.los_opid_ranges.end()) {
+                    auto opid_copy = opid_cap.to_owned(sbc.sbc_allocator);
+                    auto otr = opid_time_range{time_range{tv, tv}};
+                    auto emplace_res
+                        = sbc.sbc_opids.los_opid_ranges.emplace(opid_copy, otr);
+                    opid_iter = emplace_res.first;
+                } else {
+                    opid_iter->second.otr_range.extend_to(tv);
+                }
+
+                opid_iter->second.otr_level_stats.update_msg_count(level);
+
+                auto& otr = opid_iter->second;
+                if (!otr.otr_description.lod_id && host_cap.is_valid()
+                    && otr.otr_description.lod_elements.empty())
+                {
+                    otr.otr_description.lod_id = get_opid_desc();
+                    otr.otr_description.lod_elements.emplace_back(
+                        0, host_cap.to_string());
+                }
+            }
             dst.emplace_back(li.li_file_range.fr_offset, tv, level, 0, opid);
             return scan_match{0};
         }
@@ -463,8 +663,19 @@ public:
         static const auto SEP_RE
             = lnav::pcre2pp::code::from_const(R"(^#separator\s+(.+))");
 
+        if (dst.empty()) {
+            auto file_options = lf.get_file_options();
+
+            if (file_options) {
+                this->lf_date_time.dts_default_zone
+                    = file_options->second.fo_default_zone.pp_value;
+            } else {
+                this->lf_date_time.dts_default_zone = nullptr;
+            }
+        }
+
         if (!this->blf_format_name.empty()) {
-            return this->scan_int(dst, li, sbr);
+            return this->scan_int(dst, li, sbr, sbc);
         }
 
         if (dst.empty() || dst.size() > 20 || sbr.empty()
@@ -531,10 +742,18 @@ public:
                 this->blf_format_name = intern_string::lookup(full_name);
             } else if (directive == "#fields" && this->blf_field_defs.empty()) {
                 do {
+                    auto field_name
+                        = intern_string::lookup("bro_" + sql_safe_ident(*iter));
+                    auto common_iter = FIELD_META.find(field_name);
+                    if (common_iter == FIELD_META.end()) {
+                        FIELD_META.emplace(field_name,
+                                           logline_value_meta{
+                                               field_name,
+                                               value_kind_t::VALUE_TEXT,
+                                           });
+                    }
                     this->blf_field_defs.emplace_back(
-                        intern_string::lookup("bro_" + sql_safe_ident(*iter)),
-                        this->blf_field_defs.size(),
-                        this);
+                        field_name, this->blf_field_defs.size(), this);
                     ++iter;
                 } while (iter != ss.end());
             } else if (directive == "#types") {
@@ -551,12 +770,14 @@ public:
                     "bro_referrer",
                     "bro_resp_fuids",
                     "bro_service",
-                    "bro_status_code",
                     "bro_uid",
                     "bro_uri",
                     "bro_user_agent",
                     "bro_username",
                 };
+                static const char* KNOWN_FOREIGN[] = {
+                    "bro_status_code",
+                };
 
                 int numeric_count = 0;
 
@@ -575,7 +796,12 @@ public:
                         bool ident = std::binary_search(std::begin(KNOWN_IDS),
                                                         std::end(KNOWN_IDS),
                                                         fd.fd_meta.lvm_name);
-                        fd.with_kind(value_kind_t::VALUE_INTEGER, ident)
+                        bool foreign
+                            = std::binary_search(std::begin(KNOWN_FOREIGN),
+                                                 std::end(KNOWN_FOREIGN),
+                                                 fd.fd_meta.lvm_name);
+                        fd.with_kind(
+                              value_kind_t::VALUE_INTEGER, ident, foreign)
                             .with_numeric_index(numeric_count);
                         numeric_count += 1;
                     } else if (field_type == "bool") {
@@ -601,8 +827,7 @@ public:
         if (!this->blf_format_name.empty() && !this->blf_separator.empty()
             && !this->blf_field_defs.empty())
         {
-            dst.clear();
-            return this->scan_int(dst, li, sbr);
+            return this->scan_int(dst, li, sbr, sbc);
         }
 
         this->blf_format_name.clear();
@@ -651,6 +876,8 @@ public:
             } else {
                 values.lvv_values.emplace_back(fd.fd_meta);
             }
+            values.lvv_values.back().lv_meta.lvm_user_hidden
+                = fd.fd_root_meta->lvm_user_hidden;
         }
     }
 
@@ -675,20 +902,27 @@ public:
 
     bool hide_field(const intern_string_t field_name, bool val) override
     {
-        auto fd_iter
-            = std::find_if(this->blf_field_defs.begin(),
-                           this->blf_field_defs.end(),
-                           [field_name](const field_def& elem) {
-                               return elem.fd_meta.lvm_name == field_name;
-                           });
-        if (fd_iter == this->blf_field_defs.end()) {
+        auto fd_iter = FIELD_META.find(field_name);
+        if (fd_iter == FIELD_META.end()) {
             return false;
         }
 
-        fd_iter->fd_meta.lvm_user_hidden = val;
+        fd_iter->second.lvm_user_hidden = val;
+
         return true;
     }
 
+    std::map<intern_string_t, logline_value_meta> get_field_states() override
+    {
+        std::map<intern_string_t, logline_value_meta> retval;
+
+        for (const auto& fd : FIELD_META) {
+            retval.emplace(fd.first, fd.second);
+        }
+
+        return retval;
+    }
+
     std::shared_ptr<log_format> specialized(int fmt_lock = -1) override
     {
         auto retval = std::make_shared<bro_log_format>(*this);
@@ -707,9 +941,8 @@ public:
         void get_columns(std::vector<vtab_column>& cols) const override
         {
             for (const auto& fd : this->blt_format.blf_field_defs) {
-                std::pair<int, unsigned int> type_pair
-                    = log_vtab_impl::logline_value_to_sqlite_type(
-                        fd.fd_meta.lvm_kind);
+                auto type_pair = log_vtab_impl::logline_value_to_sqlite_type(
+                    fd.fd_meta.lvm_kind);
 
                 cols.emplace_back(fd.fd_meta.lvm_name.to_string(),
                                   type_pair.first,
@@ -726,7 +959,7 @@ public:
             this->log_vtab_impl::get_foreign_keys(keys_inout);
 
             for (const auto& fd : this->blt_format.blf_field_defs) {
-                if (fd.fd_meta.lvm_identifier) {
+                if (fd.fd_meta.lvm_identifier || fd.fd_meta.lvm_foreign_key) {
                     keys_inout.push_back(fd.fd_meta.lvm_name.to_string());
                 }
             }
@@ -775,6 +1008,9 @@ public:
     std::vector<field_def> blf_field_defs;
 };
 
+std::unordered_map<const intern_string_t, logline_value_meta>
+    bro_log_format::FIELD_META;
+
 struct ws_separated_string {
     const char* ss_str;
     size_t ss_len;
@@ -877,6 +1113,7 @@ public:
     struct field_def {
         const intern_string_t fd_name;
         logline_value_meta fd_meta;
+        logline_value_meta* fd_root_meta{nullptr};
         std::string fd_collator;
         nonstd::optional<size_t> fd_numeric_index;
 
@@ -892,19 +1129,21 @@ public:
         {
         }
 
-        field_def(int col,
+        field_def(size_t col,
                   const char* name,
                   value_kind_t kind,
                   bool ident = false,
+                  bool foreign_key = false,
                   std::string coll = "")
             : fd_name(intern_string::lookup(name)),
               fd_meta(
                   intern_string::lookup(sql_safe_ident(string_fragment(name))),
                   kind,
-                  col),
+                  logline_value_meta::table_column{col}),
               fd_collator(std::move(coll))
         {
             this->fd_meta.lvm_identifier = ident;
+            this->fd_meta.lvm_foreign_key = foreign_key;
         }
 
         field_def& with_kind(value_kind_t kind,
@@ -924,6 +1163,9 @@ public:
         }
     };
 
+    static std::unordered_map<const intern_string_t, logline_value_meta>
+        FIELD_META;
+
     struct field_to_struct_t {
         field_to_struct_t(const char* prefix, const char* struct_name)
             : fs_prefix(prefix),
@@ -942,6 +1184,7 @@ public:
     {
         this->lf_is_self_describing = true;
         this->lf_time_ordered = false;
+        this->lf_structured = true;
     }
 
     const intern_string_t get_name() const override
@@ -990,7 +1233,7 @@ public:
                 break;
             }
 
-            const field_def& fd = this->wlf_field_defs[iter.index()];
+            const auto& fd = this->wlf_field_defs[iter.index()];
             string_fragment sf = *iter;
 
             if (sf.startswith("#")) {
@@ -1051,13 +1294,12 @@ public:
                 switch (fd.fd_meta.lvm_kind) {
                     case value_kind_t::VALUE_INTEGER:
                     case value_kind_t::VALUE_FLOAT: {
-                        char field_copy[sf.length() + 1];
-                        double val;
+                        auto scan_float_res
+                            = scn::scan_value<double>(sf.to_string_view());
 
-                        if (sscanf(sf.to_string(field_copy), "%lf", &val) == 1)
-                        {
+                        if (scan_float_res) {
                             this->lf_value_stats[fd.fd_numeric_index.value()]
-                                .add_value(val);
+                                .add_value(scan_float_res.value());
                         }
                         break;
                     }
@@ -1079,9 +1321,7 @@ public:
                 tm.et_tm.tm_yday = date_tm.et_tm.tm_yday;
             }
 
-            tv.tv_sec = tm2sec(&tm.et_tm);
-            tv.tv_usec = tm.et_nsec / 1000;
-
+            tv = tm.to_timeval();
             if (!this->lf_specialized) {
                 for (auto& ll : dst) {
                     ll.set_ignore(true);
@@ -1108,6 +1348,17 @@ public:
             return scan_incomplete{};
         }
 
+        if (dst.empty()) {
+            auto file_options = lf.get_file_options();
+
+            if (file_options) {
+                this->lf_date_time.dts_default_zone
+                    = file_options->second.fo_default_zone.pp_value;
+            } else {
+                this->lf_date_time.dts_default_zone = nullptr;
+            }
+        }
+
         if (!this->wlf_format_name.empty()) {
             return this->scan_int(dst, li, sbr);
         }
@@ -1131,8 +1382,7 @@ public:
             auto line = next_read_result.unwrap();
             ws_separated_string ss(line.get_data(), line.length());
             auto iter = ss.begin();
-
-            string_fragment directive = *iter;
+            const auto directive = *iter;
 
             if (directive.empty() || directive[0] != '#') {
                 continue;
@@ -1170,9 +1420,25 @@ public:
                         [&sf](auto elem) { return sf == elem.fd_name; });
                     if (field_iter != end(KNOWN_FIELDS)) {
                         this->wlf_field_defs.emplace_back(*field_iter);
+                        auto& fd = this->wlf_field_defs.back();
+                        auto common_iter = FIELD_META.find(fd.fd_meta.lvm_name);
+                        if (common_iter == FIELD_META.end()) {
+                            auto emp_res = FIELD_META.emplace(
+                                fd.fd_meta.lvm_name, fd.fd_meta);
+                            common_iter = emp_res.first;
+                        }
+                        fd.fd_root_meta = &common_iter->second;
                     } else if (sf == "date" || sf == "time") {
                         this->wlf_field_defs.emplace_back(
                             intern_string::lookup(sf));
+                        auto& fd = this->wlf_field_defs.back();
+                        auto common_iter = FIELD_META.find(fd.fd_meta.lvm_name);
+                        if (common_iter == FIELD_META.end()) {
+                            auto emp_res = FIELD_META.emplace(
+                                fd.fd_meta.lvm_name, fd.fd_meta);
+                            common_iter = emp_res.first;
+                        }
+                        fd.fd_root_meta = &common_iter->second;
                     } else {
                         const auto fs_iter = std::find_if(
                             begin(KNOWN_STRUCT_FIELDS),
@@ -1181,27 +1447,30 @@ public:
                                 return sf.startswith(elem.fs_prefix);
                             });
                         if (fs_iter != end(KNOWN_STRUCT_FIELDS)) {
-                            auto field_name
+                            const intern_string_t field_name
                                 = intern_string::lookup(sf.substr(3));
                             this->wlf_field_defs.emplace_back(
                                 field_name,
                                 logline_value_meta(
                                     field_name,
                                     value_kind_t::VALUE_TEXT,
-                                    KNOWN_FIELDS.size() + 1
+                                    logline_value_meta::table_column{
+                                        KNOWN_FIELDS.size() + 1
                                         + std::distance(
                                             begin(KNOWN_STRUCT_FIELDS),
-                                            fs_iter),
+                                            fs_iter)},
                                     this)
                                     .with_struct_name(fs_iter->fs_struct_name));
                         } else {
-                            auto field_name = intern_string::lookup(sf);
+                            const intern_string_t field_name
+                                = intern_string::lookup(sf);
                             this->wlf_field_defs.emplace_back(
                                 field_name,
                                 logline_value_meta(
                                     field_name,
                                     value_kind_t::VALUE_TEXT,
-                                    KNOWN_FIELDS.size() + X_FIELDS_IDX,
+                                    logline_value_meta::table_column{
+                                        KNOWN_FIELDS.size() + X_FIELDS_IDX},
                                     this)
                                     .with_struct_name(X_FIELDS_NAME));
                         }
@@ -1253,7 +1522,7 @@ public:
                 return;
             }
 
-            const field_def& fd = this->wlf_field_defs[iter.index()];
+            const auto& fd = this->wlf_field_defs[iter.index()];
 
             if (sf == "-") {
                 sf.invalidate();
@@ -1275,6 +1544,10 @@ public:
             } else {
                 values.lvv_values.emplace_back(fd.fd_meta);
             }
+            if (fd.fd_root_meta != nullptr) {
+                values.lvv_values.back().lv_meta.lvm_user_hidden
+                    = fd.fd_root_meta->lvm_user_hidden;
+            }
         }
     }
 
@@ -1299,20 +1572,27 @@ public:
 
     bool hide_field(const intern_string_t field_name, bool val) override
     {
-        auto fd_iter
-            = std::find_if(this->wlf_field_defs.begin(),
-                           this->wlf_field_defs.end(),
-                           [field_name](const field_def& elem) {
-                               return elem.fd_meta.lvm_name == field_name;
-                           });
-        if (fd_iter == this->wlf_field_defs.end()) {
+        auto fd_iter = FIELD_META.find(field_name);
+        if (fd_iter == FIELD_META.end()) {
             return false;
         }
 
-        fd_iter->fd_meta.lvm_user_hidden = val;
+        fd_iter->second.lvm_user_hidden = val;
+
         return true;
     }
 
+    std::map<intern_string_t, logline_value_meta> get_field_states() override
+    {
+        std::map<intern_string_t, logline_value_meta> retval;
+
+        for (const auto& fd : FIELD_META) {
+            retval.emplace(fd.first, fd.second);
+        }
+
+        return retval;
+    }
+
     std::shared_ptr<log_format> specialized(int fmt_lock = -1) override
     {
         auto retval = std::make_shared<w3c_log_format>(*this);
@@ -1356,7 +1636,7 @@ public:
             this->log_vtab_impl::get_foreign_keys(keys_inout);
 
             for (const auto& fd : KNOWN_FIELDS) {
-                if (fd.fd_meta.lvm_identifier) {
+                if (fd.fd_meta.lvm_identifier || fd.fd_meta.lvm_foreign_key) {
                     keys_inout.push_back(fd.fd_meta.lvm_name.to_string());
                 }
             }
@@ -1402,7 +1682,10 @@ public:
     std::vector<field_def> wlf_field_defs;
 };
 
-static int KNOWN_FIELD_INDEX = 0;
+std::unordered_map<const intern_string_t, logline_value_meta>
+    w3c_log_format::FIELD_META;
+
+static size_t KNOWN_FIELD_INDEX = 0;
 const std::vector<w3c_log_format::field_def> w3c_log_format::KNOWN_FIELDS = {
     {
         KNOWN_FIELD_INDEX++,
@@ -1415,6 +1698,7 @@ const std::vector<w3c_log_format::field_def> w3c_log_format::KNOWN_FIELDS = {
         "c-ip",
         value_kind_t::VALUE_TEXT,
         true,
+        false,
         "ipaddress",
     },
     {
@@ -1434,6 +1718,7 @@ const std::vector<w3c_log_format::field_def> w3c_log_format::KNOWN_FIELDS = {
         "cs-uri-stem",
         value_kind_t::VALUE_TEXT,
         true,
+        false,
         "naturalnocase",
     },
     {
@@ -1459,6 +1744,7 @@ const std::vector<w3c_log_format::field_def> w3c_log_format::KNOWN_FIELDS = {
         "s-ip",
         value_kind_t::VALUE_TEXT,
         true,
+        false,
         "ipaddress",
     },
     {
@@ -1490,6 +1776,7 @@ const std::vector<w3c_log_format::field_def> w3c_log_format::KNOWN_FIELDS = {
         "sc-status",
         value_kind_t::VALUE_INTEGER,
         false,
+        true,
     },
     {
         KNOWN_FIELD_INDEX++,
@@ -1551,7 +1838,7 @@ class logfmt_format : public log_format {
 public:
     const intern_string_t get_name() const override
     {
-        const static auto NAME = intern_string::lookup("logfmt_log");
+        const static intern_string_t NAME = intern_string::lookup("logfmt_log");
 
         return NAME;
     }
@@ -1589,6 +1876,17 @@ public:
         bool done = false;
         logfmt_pair_handler lph(this->lf_date_time);
 
+        if (dst.empty()) {
+            auto file_options = lf.get_file_options();
+
+            if (file_options) {
+                this->lf_date_time.dts_default_zone
+                    = file_options->second.fo_default_zone.pp_value;
+            } else {
+                this->lf_date_time.dts_default_zone = nullptr;
+            }
+        }
+
         while (!done) {
             auto parse_result = p.step();
 
@@ -1685,7 +1983,8 @@ public:
                                                               kvp.first),
                                                           value_kind_t::
                                                               VALUE_INTEGER,
-                                                          0,
+                                                          logline_value_meta::
+                                                              table_column{0},
                                                           (log_format*) this}
                                            .with_struct_name(FIELDS_NAME);
                             values.lvv_values.emplace_back(lvm, bv.bv_value);
@@ -1698,7 +1997,8 @@ public:
                                                               kvp.first),
                                                           value_kind_t::
                                                               VALUE_INTEGER,
-                                                          0,
+                                                          logline_value_meta::
+                                                              table_column{0},
                                                           (log_format*) this}
                                            .with_struct_name(FIELDS_NAME);
                             values.lvv_values.emplace_back(lvm, iv.iv_value);
@@ -1711,7 +2011,8 @@ public:
                                                               kvp.first),
                                                           value_kind_t::
                                                               VALUE_INTEGER,
-                                                          0,
+                                                          logline_value_meta::
+                                                              table_column{0},
                                                           (log_format*) this}
                                            .with_struct_name(FIELDS_NAME);
                             values.lvv_values.emplace_back(lvm, fv.fv_value);
@@ -1732,8 +2033,9 @@ public:
                     } else if (kvp.first == "level") {
                     } else if (kvp.first == "msg") {
                         sa.emplace_back(value_lr, SA_BODY.value());
-                    } else if (!kvp.second.is<logfmt::parser::int_value>()
-                               && !kvp.second.is<logfmt::parser::bool_value>())
+                    } else if (kvp.second.is<logfmt::parser::quoted_value>()
+                               || kvp.second
+                                      .is<logfmt::parser::unquoted_value>())
                     {
                         auto lvm
                             = logline_value_meta{intern_string::lookup(
@@ -1741,7 +2043,8 @@ public:
                                                  value_frag.startswith("\"")
                                                      ? value_kind_t::VALUE_JSON
                                                      : value_kind_t::VALUE_TEXT,
-                                                 0,
+                                                 logline_value_meta::
+                                                     table_column{0},
                                                  (log_format*) this}
                                   .with_struct_name(FIELDS_NAME);
                         values.lvv_values.emplace_back(lvm, value_frag);
@@ -1772,4 +2075,5 @@ static auto format_binder = injector::bind_multiple<log_format>()
                                 .add<logfmt_format>()
                                 .add<bro_log_format>()
                                 .add<w3c_log_format>()
-                                .add<generic_log_format>();
+                                .add<generic_log_format>()
+                                .add<piper_log_format>();