Adding upstream version 0.11.1.upstream/0.11.1upstream

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

1 files changed, 427 insertions, 0 deletions

diff --git a/src/log_format_ext.hh b/src/log_format_ext.hh
new file mode 100644
index 0000000..e966b27
--- /dev/null
+++ b/src/log_format_ext.hh
@@ -0,0 +1,427 @@
+/**
+ * Copyright (c) 2020, Timothy Stack
+ *
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * * Neither the name of Timothy Stack nor the names of its contributors
+ * may be used to endorse or promote products derived from this software
+ * without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ''AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * @file log_format_ext.hh
+ */
+
+#ifndef lnav_log_format_ext_hh
+#define lnav_log_format_ext_hh
+
+#include <unordered_map>
+
+#include "log_format.hh"
+#include "log_search_table_fwd.hh"
+#include "yajlpp/yajlpp.hh"
+
+class module_format;
+
+class external_log_format : public log_format {
+public:
+    struct sample {
+        positioned_property<std::string> s_line;
+        std::string s_description;
+        log_level_t s_level{LEVEL_UNKNOWN};
+        std::set<std::string> s_matched_regexes;
+    };
+
+    struct value_def {
+        value_def(intern_string_t name,
+                  value_kind_t kind,
+                  int col,
+                  log_format* format)
+            : vd_meta(name, kind, col, format)
+        {
+        }
+
+        void set_rewrite_src_name()
+        {
+            this->vd_rewrite_src_name = intern_string::lookup(
+                fmt::format(FMT_STRING("{}:{}"),
+                            this->vd_meta.lvm_format.value()->get_name(),
+                            this->vd_meta.lvm_name));
+        }
+
+        logline_value_meta vd_meta;
+        std::string vd_collate;
+        bool vd_foreign_key{false};
+        intern_string_t vd_unit_field;
+        std::map<const intern_string_t, scaling_factor> vd_unit_scaling;
+        ssize_t vd_values_index{-1};
+        bool vd_internal{false};
+        std::vector<std::string> vd_action_list;
+        std::string vd_rewriter;
+        std::string vd_description;
+        intern_string_t vd_rewrite_src_name;
+    };
+
+    struct indexed_value_def {
+        indexed_value_def(int index = -1,
+                          int unit_index = -1,
+                          std::shared_ptr<value_def> vd = nullptr)
+            : ivd_index(index), ivd_unit_field_index(unit_index),
+              ivd_value_def(std::move(vd))
+        {
+        }
+
+        int ivd_index;
+        int ivd_unit_field_index;
+        std::shared_ptr<value_def> ivd_value_def;
+
+        bool operator<(const indexed_value_def& rhs) const
+        {
+            return this->ivd_index < rhs.ivd_index;
+        }
+    };
+
+    struct pattern {
+        intern_string_t p_name;
+        std::string p_config_path;
+        factory_container<lnav::pcre2pp::code,
+                          int>::with_default_args<PCRE2_DOTALL>
+            p_pcre;
+        std::vector<indexed_value_def> p_value_by_index;
+        std::vector<int> p_numeric_value_indexes;
+        int p_timestamp_field_index{-1};
+        int p_time_field_index{-1};
+        int p_level_field_index{-1};
+        int p_module_field_index{-1};
+        int p_opid_field_index{-1};
+        int p_body_field_index{-1};
+        int p_timestamp_end{-1};
+        bool p_module_format{false};
+        std::set<size_t> p_matched_samples;
+    };
+
+    struct level_pattern {
+        factory_container<lnav::pcre2pp::code> lp_pcre;
+    };
+
+    struct yajl_handle_deleter {
+        void operator()(yajl_handle handle) const
+        {
+            if (handle != nullptr) {
+                yajl_free(handle);
+            }
+        }
+    };
+
+    external_log_format(const intern_string_t name)
+        : elf_level_field(intern_string::lookup("level", -1)),
+          elf_body_field(intern_string::lookup("body", -1)),
+          jlf_yajl_handle(nullptr, yajl_handle_deleter()), elf_name(name)
+    {
+        this->jlf_line_offsets.reserve(128);
+    }
+
+    const intern_string_t get_name() const { return this->elf_name; }
+
+    bool match_name(const std::string& filename);
+
+    bool match_mime_type(const file_format_t ff) const;
+
+    scan_result_t scan(logfile& lf,
+                       std::vector<logline>& dst,
+                       const line_info& offset,
+                       shared_buffer_ref& sbr,
+                       scan_batch_context& sbc);
+
+    bool scan_for_partial(shared_buffer_ref& sbr, size_t& len_out) const;
+
+    void annotate(uint64_t line_number,
+                  string_attrs_t& sa,
+                  logline_value_vector& values,
+                  bool annotate_module = true) const;
+
+    void rewrite(exec_context& ec,
+                 shared_buffer_ref& line,
+                 string_attrs_t& sa,
+                 std::string& value_out);
+
+    void build(std::vector<lnav::console::user_message>& errors);
+
+    void register_vtabs(log_vtab_manager* vtab_manager,
+                        std::vector<lnav::console::user_message>& errors);
+
+    bool match_samples(const std::vector<sample>& samples) const;
+
+    bool hide_field(const intern_string_t field_name, bool val)
+    {
+        auto vd_iter = this->elf_value_defs.find(field_name);
+
+        if (vd_iter == this->elf_value_defs.end()) {
+            return false;
+        }
+
+        vd_iter->second->vd_meta.lvm_user_hidden = val;
+        return true;
+    }
+
+    std::shared_ptr<log_format> specialized(int fmt_lock);
+
+    const logline_value_stats* stats_for_value(
+        const intern_string_t& name) const
+    {
+        const logline_value_stats* retval = nullptr;
+
+        for (size_t lpc = 0; lpc < this->elf_numeric_value_defs.size(); lpc++) {
+            value_def& vd = *this->elf_numeric_value_defs[lpc];
+
+            if (vd.vd_meta.lvm_name == name) {
+                retval = &this->lf_value_stats[lpc];
+                break;
+            }
+        }
+
+        return retval;
+    }
+
+    void get_subline(const logline& ll,
+                     shared_buffer_ref& sbr,
+                     bool full_message);
+
+    std::shared_ptr<log_vtab_impl> get_vtab_impl() const;
+
+    const std::vector<std::string>* get_actions(const logline_value& lv) const
+    {
+        const std::vector<std::string>* retval = nullptr;
+
+        const auto iter = this->elf_value_defs.find(lv.lv_meta.lvm_name);
+        if (iter != this->elf_value_defs.end()) {
+            retval = &iter->second->vd_action_list;
+        }
+
+        return retval;
+    }
+
+    std::set<std::string> get_source_path() const
+    {
+        return this->elf_source_path;
+    }
+
+    std::vector<logline_value_meta> get_value_metadata() const;
+
+    enum class json_log_field {
+        CONSTANT,
+        VARIABLE
+    };
+
+    struct json_format_element {
+        enum class align_t {
+            LEFT,
+            RIGHT,
+        };
+
+        enum class overflow_t {
+            ABBREV,
+            TRUNCATE,
+            DOTDOT,
+        };
+
+        enum class transform_t {
+            NONE,
+            UPPERCASE,
+            LOWERCASE,
+            CAPITALIZE,
+        };
+
+        json_log_field jfe_type{json_log_field::CONSTANT};
+        positioned_property<intern_string_t> jfe_value;
+        std::string jfe_default_value{"-"};
+        long long jfe_min_width{0};
+        long long jfe_max_width{LLONG_MAX};
+        align_t jfe_align{align_t::LEFT};
+        overflow_t jfe_overflow{overflow_t::ABBREV};
+        transform_t jfe_text_transform{transform_t::NONE};
+        std::string jfe_ts_format;
+    };
+
+    struct json_field_cmp {
+        json_field_cmp(json_log_field type, const intern_string_t name)
+            : jfc_type(type), jfc_field_name(name)
+        {
+        }
+
+        bool operator()(const json_format_element& jfe) const
+        {
+            return (this->jfc_type == jfe.jfe_type
+                    && this->jfc_field_name == jfe.jfe_value.pp_value);
+        }
+
+        json_log_field jfc_type;
+        const intern_string_t jfc_field_name;
+    };
+
+    struct highlighter_def {
+        factory_container<lnav::pcre2pp::code> hd_pattern;
+        positioned_property<std::string> hd_color;
+        positioned_property<std::string> hd_background_color;
+        bool hd_underline{false};
+        bool hd_blink{false};
+    };
+
+    long value_line_count(const intern_string_t ist,
+                          bool top_level,
+                          const unsigned char* str = nullptr,
+                          ssize_t len = -1) const;
+
+    bool has_value_def(const intern_string_t ist) const
+    {
+        const auto iter = this->elf_value_defs.find(ist);
+
+        return iter != this->elf_value_defs.end();
+    }
+
+    std::string get_pattern_path(uint64_t line_number) const
+    {
+        if (this->elf_type != elf_type_t::ELF_TYPE_TEXT) {
+            return "structured";
+        }
+        int pat_index = this->pattern_index_for_line(line_number);
+        return this->elf_pattern_order[pat_index]->p_config_path;
+    }
+
+    intern_string_t get_pattern_name(uint64_t line_number) const;
+
+    std::string get_pattern_regex(uint64_t line_number) const
+    {
+        if (this->elf_type != elf_type_t::ELF_TYPE_TEXT) {
+            return "";
+        }
+        int pat_index = this->pattern_index_for_line(line_number);
+        return this->elf_pattern_order[pat_index]
+            ->p_pcre.pp_value->get_pattern();
+    }
+
+    log_level_t convert_level(string_fragment str,
+                              scan_batch_context* sbc) const;
+
+    using mod_map_t = std::map<intern_string_t, module_format>;
+    static mod_map_t MODULE_FORMATS;
+    static std::vector<std::shared_ptr<external_log_format>>
+        GRAPH_ORDERED_FORMATS;
+
+    std::set<std::string> elf_source_path;
+    std::vector<ghc::filesystem::path> elf_format_source_order;
+    std::map<intern_string_t, int> elf_format_sources;
+    std::list<intern_string_t> elf_collision;
+    std::set<file_format_t> elf_mime_types;
+    factory_container<lnav::pcre2pp::code> elf_filename_pcre;
+    std::map<std::string, std::shared_ptr<pattern>> elf_patterns;
+    std::vector<std::shared_ptr<pattern>> elf_pattern_order;
+    std::vector<sample> elf_samples;
+    std::unordered_map<const intern_string_t, std::shared_ptr<value_def>>
+        elf_value_defs;
+    std::vector<std::shared_ptr<value_def>> elf_value_def_order;
+    std::vector<std::shared_ptr<value_def>> elf_numeric_value_defs;
+    int elf_column_count{0};
+    double elf_timestamp_divisor{1.0};
+    intern_string_t elf_level_field;
+    factory_container<lnav::pcre2pp::code> elf_level_pointer;
+    intern_string_t elf_body_field;
+    intern_string_t elf_module_id_field;
+    intern_string_t elf_opid_field;
+    std::map<log_level_t, level_pattern> elf_level_patterns;
+    std::vector<std::pair<int64_t, log_level_t>> elf_level_pairs;
+    bool elf_container{false};
+    bool elf_has_module_format{false};
+    bool elf_builtin_format{false};
+
+    using search_table_pcre2pp
+        = factory_container<lnav::pcre2pp::code, int>::with_default_args<
+            log_search_table_ns::PATTERN_OPTIONS>;
+
+    struct search_table_def {
+        search_table_pcre2pp std_pattern;
+        std::string std_glob;
+        log_level_t std_level{LEVEL_UNKNOWN};
+    };
+
+    std::map<intern_string_t, search_table_def> elf_search_tables;
+    std::map<const intern_string_t, highlighter_def> elf_highlighter_patterns;
+
+    enum class elf_type_t {
+        ELF_TYPE_TEXT,
+        ELF_TYPE_JSON,
+        ELF_TYPE_CSV,
+    };
+
+    elf_type_t elf_type{elf_type_t::ELF_TYPE_TEXT};
+
+    void json_append_to_cache(const char* value, ssize_t len)
+    {
+        size_t old_size = this->jlf_cached_line.size();
+        if (len == -1) {
+            len = strlen(value);
+        }
+        this->jlf_cached_line.resize(old_size + len);
+        memcpy(&(this->jlf_cached_line[old_size]), value, len);
+    }
+
+    void json_append_to_cache(ssize_t len)
+    {
+        size_t old_size = this->jlf_cached_line.size();
+        this->jlf_cached_line.resize(old_size + len);
+        memset(&this->jlf_cached_line[old_size], ' ', len);
+    }
+
+    void json_append(const json_format_element& jfe,
+                     const char* value,
+                     ssize_t len);
+
+    logline_value_meta get_value_meta(intern_string_t field_name,
+                                      value_kind_t kind);
+
+    std::vector<lnav::console::snippet> get_snippets() const;
+
+    bool jlf_hide_extra{false};
+    std::vector<json_format_element> jlf_line_format;
+    int jlf_line_format_init_count{0};
+    shared_buffer jlf_share_manager;
+    logline_value_vector jlf_line_values;
+
+    off_t jlf_cached_offset{-1};
+    bool jlf_cached_full{false};
+    std::vector<off_t> jlf_line_offsets;
+    std::vector<char> jlf_cached_line;
+    string_attrs_t jlf_line_attrs;
+    std::shared_ptr<yajlpp_parse_context> jlf_parse_context;
+    std::shared_ptr<yajl_handle_t> jlf_yajl_handle;
+
+private:
+    const intern_string_t elf_name;
+
+    static uint8_t module_scan(string_fragment body_cap,
+                               const intern_string_t& mod_name);
+};
+
+class module_format {
+public:
+    std::shared_ptr<log_format> mf_mod_format;
+};
+
+#endif