Merging upstream version 0.12.2.

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

1 files changed, 245 insertions, 110 deletions

diff --git a/src/field_overlay_source.cc b/src/field_overlay_source.cc
index ad4c312..821a8b9 100644
--- a/src/field_overlay_source.cc
+++ b/src/field_overlay_source.cc
@@ -29,27 +29,33 @@
 
 #include "field_overlay_source.hh"
 
-#include "base/ansi_scrubber.hh"
 #include "base/humanize.time.hh"
 #include "base/snippet_highlighters.hh"
+#include "command_executor.hh"
 #include "config.h"
+#include "log.annotate.hh"
 #include "log_format_ext.hh"
 #include "log_vtab_impl.hh"
 #include "md2attr_line.hh"
+#include "ptimec.hh"
 #include "readline_highlighters.hh"
-#include "relative_time.hh"
 #include "vtab_module.hh"
 #include "vtab_module_json.hh"
 
+using namespace md4cpp::literals;
+using namespace lnav::roles::literals;
+
 json_string extract(const char* str);
 
 void
-field_overlay_source::build_field_lines(const listview_curses& lv)
+field_overlay_source::build_field_lines(const listview_curses& lv,
+                                        vis_line_t row)
 {
     auto& lss = this->fos_lss;
     auto& vc = view_colors::singleton();
 
     this->fos_lines.clear();
+    this->fos_row_to_field_meta.clear();
 
     if (lss.text_line_count() == 0) {
         this->fos_log_helper.clear();
@@ -57,7 +63,7 @@ field_overlay_source::build_field_lines(const listview_curses& lv)
         return;
     }
 
-    content_line_t cl = lss.at(lv.get_selection());
+    content_line_t cl = lss.at(row);
     std::shared_ptr<logfile> file = lss.find(cl);
     auto ll = file->begin() + cl;
     auto format = file->get_format();
@@ -72,13 +78,11 @@ field_overlay_source::build_field_lines(const listview_curses& lv)
         display = display || this->fos_contexts.top().c_show;
     }
 
-    this->build_meta_line(lv, this->fos_lines, lv.get_top());
-
     if (!display) {
         return;
     }
 
-    if (!this->fos_log_helper.parse_line(lv.get_selection())) {
+    if (!this->fos_log_helper.parse_line(row)) {
         return;
     }
 
@@ -132,7 +136,7 @@ field_overlay_source::build_field_lines(const listview_curses& lv)
     time_lr.lr_end = time_str.length();
     time_line.with_attr(
         string_attr(time_lr, VC_STYLE.value(text_attrs{A_BOLD})));
-    time_str.append(" -- ");
+    time_str.append(" \u2014 ");
     time_lr.lr_start = time_str.length();
     time_str.append(humanize::time::point::from_tv(ll->get_timeval())
                         .with_convert_to_local(true)
@@ -155,14 +159,13 @@ field_overlay_source::build_field_lines(const listview_curses& lv)
 
         dts.set_base_time(format->lf_date_time.dts_base_time,
                           format->lf_date_time.dts_base_tm.et_tm);
+        dts.dts_zoned_to_local = format->lf_date_time.dts_zoned_to_local;
         if (format->lf_date_time.scan(time_src,
                                       time_range.length(),
                                       format->get_timestamp_formats(),
                                       &tm,
-                                      actual_tv,
-                                      false)
-            || dts.scan(
-                time_src, time_range.length(), nullptr, &tm, actual_tv, false))
+                                      actual_tv)
+            || dts.scan(time_src, time_range.length(), nullptr, &tm, actual_tv))
         {
             sql_strftime(
                 orig_timestamp, sizeof(orig_timestamp), actual_tv, 'T');
@@ -210,10 +213,15 @@ field_overlay_source::build_field_lines(const listview_curses& lv)
         time_line.append("  Format: ")
             .append(lnav::roles::symbol(
                 ts_formats[format->lf_date_time.dts_fmt_lock]));
+        if (format->lf_date_time.dts_default_zone != nullptr) {
+            time_line.append("  Default Zone: ")
+                .append(lnav::roles::symbol(
+                    format->lf_date_time.dts_default_zone->name()));
+        }
     }
 
     if ((!this->fos_contexts.empty() && this->fos_contexts.top().c_show)
-        || diff_tv.tv_sec > 0)
+        || diff_tv.tv_sec > 0 || ll->is_time_skewed())
     {
         this->fos_lines.emplace_back(time_line);
     }
@@ -228,11 +236,16 @@ field_overlay_source::build_field_lines(const listview_curses& lv)
     }
     this->fos_unknown_key_size = 0;
 
-    for (auto& ldh_line_value : this->fos_log_helper.ldh_line_values.lvv_values)
+    for (const auto& ldh_line_value :
+         this->fos_log_helper.ldh_line_values.lvv_values)
     {
         auto& meta = ldh_line_value.lv_meta;
         int this_key_size = meta.lvm_name.size();
 
+        if (!meta.lvm_column.is<logline_value_meta::table_column>()) {
+            continue;
+        }
+
         if (!this->fos_contexts.empty()) {
             this_key_size += this->fos_contexts.top().c_prefix.length();
         }
@@ -280,16 +293,21 @@ field_overlay_source::build_field_lines(const listview_curses& lv)
 
     const log_format* last_format = nullptr;
 
-    for (auto& lv : this->fos_log_helper.ldh_line_values.lvv_values) {
-        if (!lv.lv_meta.lvm_format) {
+    for (const auto& lv : this->fos_log_helper.ldh_line_values.lvv_values) {
+        const auto& meta = lv.lv_meta;
+        if (!meta.lvm_format) {
+            continue;
+        }
+
+        if (!meta.lvm_column.is<logline_value_meta::table_column>()) {
             continue;
         }
 
-        auto* curr_format = lv.lv_meta.lvm_format.value();
+        auto* curr_format = meta.lvm_format.value();
         auto* curr_elf = dynamic_cast<external_log_format*>(curr_format);
         const auto format_name = curr_format->get_name().to_string();
         attr_line_t al;
-        std::string str, value_str = lv.to_string();
+        std::string value_str = lv.to_string();
 
         if (curr_format != last_format) {
             this->fos_lines.emplace_back(" Known message fields for table "
@@ -302,86 +320,106 @@ field_overlay_source::build_field_lines(const listview_curses& lv)
         }
 
         std::string field_name, orig_field_name;
-        if (lv.lv_meta.lvm_struct_name.empty()) {
-            if (curr_elf && curr_elf->elf_body_field == lv.lv_meta.lvm_name) {
+        line_range hl_range;
+        al.append(" ").append("|", VC_GRAPHIC.value(ACS_LTEE)).append(" ");
+        if (meta.lvm_struct_name.empty()) {
+            if (curr_elf && curr_elf->elf_body_field == meta.lvm_name) {
                 field_name = LOG_BODY;
             } else if (curr_elf
-                       && curr_elf->lf_timestamp_field == lv.lv_meta.lvm_name)
+                       && curr_elf->lf_timestamp_field == meta.lvm_name)
             {
                 field_name = LOG_TIME;
             } else {
-                field_name = lv.lv_meta.lvm_name.to_string();
+                field_name = meta.lvm_name.to_string();
             }
             orig_field_name = field_name;
             if (!this->fos_contexts.empty()) {
                 field_name = this->fos_contexts.top().c_prefix + field_name;
             }
-            str = "   " + field_name;
-        } else {
-            auto_mem<char, sqlite3_free> jgetter;
+            if (meta.is_hidden()) {
+                al.append("\u25c7"_comment);
+            } else {
+                al.append("\u25c6"_ok);
+            }
+            al.append(" ");
+
+            switch (meta.to_chart_type()) {
+                case chart_type_t::none:
+                    al.append("   ");
+                    break;
+                case chart_type_t::hist:
+                case chart_type_t::spectro:
+                    al.append(":bar_chart:"_emoji).append(" ");
+                    break;
+            }
+            auto prefix_len = al.utf8_length_or_length();
+            hl_range.lr_start = al.get_string().length();
+            al.append(field_name);
+            hl_range.lr_end = al.get_string().length();
+            al.pad_to(prefix_len + this->fos_known_key_size);
 
-            jgetter = sqlite3_mprintf("   jget(%s, '/%q')",
-                                      lv.lv_meta.lvm_struct_name.get(),
-                                      lv.lv_meta.lvm_name.get());
-            str = jgetter;
-        }
-        str.append(this->fos_known_key_size - (str.length() - 3), ' ');
-        str += " = " + value_str;
-
-        al.with_string(str);
-        if (lv.lv_meta.lvm_struct_name.empty()) {
-            auto prefix_len = field_name.length() - orig_field_name.length();
-            al.with_attr(string_attr(
-                line_range(3 + prefix_len, 3 + prefix_len + field_name.size()),
-                VC_STYLE.value(vc.attrs_for_ident(orig_field_name))));
+            this->fos_row_to_field_meta.emplace(this->fos_lines.size(), meta);
         } else {
-            al.with_attr(string_attr(
-                line_range(8, 8 + lv.lv_meta.lvm_struct_name.size()),
-                VC_STYLE.value(
-                    vc.attrs_for_ident(lv.lv_meta.lvm_struct_name))));
+            auto jget_str = lnav::sql::mprintf("jget(%s, '/%q')",
+                                               meta.lvm_struct_name.get(),
+                                               meta.lvm_name.get());
+            hl_range.lr_start = al.get_string().length();
+            al.append(jget_str.in());
+            hl_range.lr_end = al.get_string().length();
         }
+        readline_sqlite_highlighter_int(al, -1, hl_range);
+
+        al.append(" = ").append(scrub_ws(value_str.c_str()));
 
         this->fos_lines.emplace_back(al);
-        this->add_key_line_attrs(this->fos_known_key_size);
 
-        if (lv.lv_meta.lvm_kind == value_kind_t::VALUE_STRUCT) {
+        if (meta.lvm_kind == value_kind_t::VALUE_STRUCT) {
             json_string js = extract(value_str.c_str());
 
             al.clear()
                 .append("   extract(")
-                .append(lv.lv_meta.lvm_name.get(),
-                        VC_STYLE.value(vc.attrs_for_ident(lv.lv_meta.lvm_name)))
+                .append(meta.lvm_name.get(),
+                        VC_STYLE.value(vc.attrs_for_ident(meta.lvm_name)))
                 .append(")")
-                .append(this->fos_known_key_size - lv.lv_meta.lvm_name.size()
-                            - 9 + 3,
+                .append(this->fos_known_key_size - meta.lvm_name.size() - 9 + 3,
                         ' ')
                 .append(" = ")
-                .append(
-                    string_fragment::from_bytes(js.js_content.in(), js.js_len));
+                .append(scrub_ws(string_fragment::from_bytes(js.js_content.in(),
+                                                             js.js_len)));
             this->fos_lines.emplace_back(al);
             this->add_key_line_attrs(this->fos_known_key_size);
         }
     }
 
-    std::map<const intern_string_t, json_ptr_walk::walk_list_t>::iterator
-        json_iter;
-
-    if (!this->fos_log_helper.ldh_json_pairs.empty()) {
+    if (!this->fos_log_helper.ldh_extra_json.empty()
+        || !this->fos_log_helper.ldh_json_pairs.empty())
+    {
         this->fos_lines.emplace_back(" JSON fields:");
     }
 
-    for (json_iter = this->fos_log_helper.ldh_json_pairs.begin();
-         json_iter != this->fos_log_helper.ldh_json_pairs.end();
-         ++json_iter)
-    {
-        json_ptr_walk::walk_list_t& jpairs = json_iter->second;
+    for (const auto& extra_pair : this->fos_log_helper.ldh_extra_json) {
+        auto qname = lnav::sql::mprintf("%Q", extra_pair.first.c_str());
+        auto key_line = attr_line_t("   jget(log_raw_text, ")
+                            .append(qname.in())
+                            .append(")");
+        readline_sqlite_highlighter(key_line, 0);
+        auto key_size = key_line.length();
+        key_line.append(" = ").append(scrub_ws(extra_pair.second));
+        this->fos_lines.emplace_back(key_line);
+        this->add_key_line_attrs(key_size - 3);
+    }
+
+    for (const auto& jpairs_map : this->fos_log_helper.ldh_json_pairs) {
+        const auto& jpairs = jpairs_map.second;
 
         for (size_t lpc = 0; lpc < jpairs.size(); lpc++) {
-            this->fos_lines.emplace_back(
-                "   "
-                + this->fos_log_helper.format_json_getter(json_iter->first, lpc)
-                + " = " + jpairs[lpc].wt_value);
-            this->add_key_line_attrs(0);
+            auto key_line = attr_line_t("   ").append(
+                this->fos_log_helper.format_json_getter(jpairs_map.first, lpc));
+            readline_sqlite_highlighter(key_line, 0);
+            auto key_size = key_line.length();
+            key_line.append(" = ").append(scrub_ws(jpairs[lpc].wt_value));
+            this->fos_lines.emplace_back(key_line);
+            this->add_key_line_attrs(key_size - 3);
         }
     }
 
@@ -390,28 +428,26 @@ field_overlay_source::build_field_lines(const listview_curses& lv)
     }
 
     for (const auto& xml_pair : this->fos_log_helper.ldh_xml_pairs) {
-        auto_mem<char, sqlite3_free> qname;
-        auto_mem<char, sqlite3_free> xp_call;
-
-        qname = sql_quote_ident(xml_pair.first.first.get());
-        xp_call = sqlite3_mprintf(
-            "xpath(%Q, %s)", xml_pair.first.second.c_str(), qname.in());
-        this->fos_lines.emplace_back(fmt::format(
-            FMT_STRING("   {} = {}"), xp_call.in(), xml_pair.second));
-        this->add_key_line_attrs(0);
-    }
-
-    if (!this->fos_contexts.empty()
-        && !this->fos_contexts.top().c_show_discovered)
-    {
-        return;
+        auto qname = sql_quote_ident(xml_pair.first.first.get());
+        auto xp_call = lnav::sql::mprintf(
+            "xpath(%Q, %s.%s)",
+            xml_pair.first.second.c_str(),
+            this->fos_log_helper.ldh_file->get_format()->get_name().c_str(),
+            qname.in());
+        auto key_line = attr_line_t("   ").append(xp_call.in());
+        readline_sqlite_highlighter(key_line, 0);
+        auto key_size = key_line.length();
+        key_line.append(" = ").append(scrub_ws(xml_pair.second));
+        this->fos_lines.emplace_back(key_line);
+        this->add_key_line_attrs(key_size - 3);
     }
 
     if (this->fos_log_helper.ldh_parser->dp_pairs.empty()) {
         this->fos_lines.emplace_back(" No discovered message fields");
     } else {
         this->fos_lines.emplace_back(
-            " Discovered fields for logline table from message format: ");
+            " Discovered fields for logline table from message "
+            "format: ");
         this->fos_lines.back().with_attr(
             string_attr(line_range(23, 23 + 7),
                         VC_STYLE.value(vc.attrs_for_ident("logline"))));
@@ -450,13 +486,43 @@ field_overlay_source::build_meta_line(const listview_curses& lv,
 {
     auto line_meta_opt = this->fos_lss.find_bookmark_metadata(row);
 
+    if (!this->fos_contexts.empty()
+        && this->fos_contexts.top().c_show_applicable_annotations)
+    {
+        auto file_and_line = this->fos_lss.find_line_with_file(row);
+
+        if (file_and_line && !file_and_line->second->is_continued()) {
+            auto applicable_anno = lnav::log::annotate::applicable(row);
+            if (!applicable_anno.empty()
+                && (!line_meta_opt
+                    || line_meta_opt.value()->bm_annotations.la_pairs.empty()))
+            {
+                auto anno_msg
+                    = attr_line_t(" ")
+                          .append(":memo:"_emoji)
+                          .append(" Annotations available, ")
+                          .append(lv.get_selection() == row
+                                      ? "use "
+                                      : "focus on this line and use ")
+                          .append(":annotate"_quoted_code)
+                          .append(" to apply them")
+                          .append(lv.get_selection() == row ? " to this line"
+                                                            : "")
+                          .with_attr_for_all(
+                              VC_ROLE.value(role_t::VCR_COMMENT));
+
+                dst.emplace_back(anno_msg);
+            }
+        }
+    }
+
     if (!line_meta_opt) {
         return;
     }
+    const auto* tc = dynamic_cast<const textview_curses*>(&lv);
     auto& vc = view_colors::singleton();
     const auto& line_meta = *(line_meta_opt.value());
     size_t filename_width = this->fos_lss.get_filename_offset();
-    const auto* tc = dynamic_cast<const textview_curses*>(&lv);
 
     if (!line_meta.bm_comment.empty()) {
         const auto* lead = line_meta.bm_tags.empty() ? " \u2514 " : " \u251c ";
@@ -474,6 +540,9 @@ field_overlay_source::build_meta_line(const listview_curses& lv,
         }
 
         auto comment_lines = al.rtrim().split_lines();
+        if (comment_lines.back().empty()) {
+            comment_lines.pop_back();
+        }
         for (size_t lpc = 0; lpc < comment_lines.size(); lpc++) {
             auto& comment_line = comment_lines[lpc];
 
@@ -524,12 +593,63 @@ field_overlay_source::build_meta_line(const listview_curses& lv,
         }
         dst.emplace_back(al);
     }
+    if (!line_meta.bm_annotations.la_pairs.empty()) {
+        for (const auto& anno_pair : line_meta.bm_annotations.la_pairs) {
+            attr_line_t al;
+            md2attr_line mdal;
+
+            dst.push_back(
+                attr_line_t()
+                    .append(filename_width, ' ')
+                    .appendf(FMT_STRING(" \u251c {}:"), anno_pair.first)
+                    .with_attr_for_all(VC_ROLE.value(role_t::VCR_COMMENT)));
+
+            auto parse_res = md4cpp::parse(anno_pair.second, mdal);
+            if (parse_res.isOk()) {
+                al.append(parse_res.unwrap());
+            } else {
+                log_error("%d: cannot convert annotation to markdown: %s",
+                          (int) row,
+                          parse_res.unwrapErr().c_str());
+                al.append(anno_pair.second);
+            }
+
+            auto anno_lines = al.rtrim().split_lines();
+            if (anno_lines.back().empty()) {
+                anno_lines.pop_back();
+            }
+            for (size_t lpc = 0; lpc < anno_lines.size(); lpc++) {
+                auto& anno_line = anno_lines[lpc];
+
+                if (lpc == 0 && anno_line.empty()) {
+                    continue;
+                }
+                // anno_line.with_attr_for_all(VC_ROLE.value(role_t::VCR_COMMENT));
+                anno_line.insert(0,
+                                 lpc == anno_lines.size() - 1
+                                     ? " \u2570 "_comment
+                                     : " \u2502 "_comment);
+                anno_line.insert(0, filename_width, ' ');
+                if (tc != nullptr) {
+                    auto hl = tc->get_highlights();
+                    auto hl_iter
+                        = hl.find({highlight_source_t::PREVIEW, "search"});
+
+                    if (hl_iter != hl.end()) {
+                        hl_iter->second.annotate(anno_line, filename_width);
+                    }
+                }
+
+                dst.emplace_back(anno_line);
+            }
+        }
+    }
 }
 
 void
 field_overlay_source::add_key_line_attrs(int key_size, bool last_line)
 {
-    string_attrs_t& sa = this->fos_lines.back().get_attrs();
+    auto& sa = this->fos_lines.back().get_attrs();
     struct line_range lr(1, 2);
     int64_t graphic = (int64_t) (last_line ? ACS_LLCORNER : ACS_LTEE);
     sa.emplace_back(lr, VC_GRAPHIC.value(graphic));
@@ -539,36 +659,51 @@ field_overlay_source::add_key_line_attrs(int key_size, bool last_line)
     sa.emplace_back(lr, VC_STYLE.value(text_attrs{A_BOLD}));
 }
 
-bool
-field_overlay_source::list_value_for_overlay(const listview_curses& lv,
-                                             int y,
-                                             int bottom,
-                                             vis_line_t row,
-                                             attr_line_t& value_out)
+void
+field_overlay_source::list_value_for_overlay(
+    const listview_curses& lv,
+    vis_line_t row,
+    std::vector<attr_line_t>& value_out)
 {
-    if (y == 0) {
-        this->build_field_lines(lv);
-        return false;
-    }
-
-    if (1 <= y && y <= (int) this->fos_lines.size()) {
-        value_out = this->fos_lines[y - 1];
-        return true;
-    }
-
-    if (!this->fos_meta_lines.empty() && this->fos_meta_lines_row == row - 1_vl)
-    {
-        value_out = this->fos_meta_lines.front();
-        this->fos_meta_lines.erase(this->fos_meta_lines.begin());
-
-        return true;
+    if (row == lv.get_selection()) {
+        this->build_field_lines(lv, row);
+        value_out = this->fos_lines;
     }
+    this->build_meta_line(lv, value_out, row);
+}
 
-    if (row < lv.get_inner_height()) {
-        this->fos_meta_lines.clear();
-        this->build_meta_line(lv, this->fos_meta_lines, row);
-        this->fos_meta_lines_row = row;
+nonstd::optional<attr_line_t>
+field_overlay_source::list_header_for_overlay(const listview_curses& lv,
+                                              vis_line_t vl)
+{
+    attr_line_t retval;
+
+    retval.append(this->fos_lss.get_filename_offset(), ' ');
+    if (this->fos_contexts.top().c_show) {
+        retval
+            .appendf(FMT_STRING("\u258C Line {:L} parser details.  "
+                                "Press "),
+                     (int) vl)
+            .append("p"_hotkey)
+            .append(" to hide this panel.");
+    } else {
+        retval.append("\u258C Line ")
+            .append(
+                lnav::roles::number(fmt::format(FMT_STRING("{:L}"), (int) vl)))
+            .append(" metadata");
+    }
+
+    if (lv.get_overlay_selection()) {
+        retval.append("  ")
+            .append("SPC"_hotkey)
+            .append(": hide/show field  ")
+            .append("Esc"_hotkey)
+            .append(": exit this panel");
+    } else {
+        retval.append("  Press ")
+            .append("CTRL-]"_hotkey)
+            .append(" to focus on this panel");
     }
 
-    return false;
+    return retval;
 }