Merging upstream version 0.12.2.

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

1 files changed, 744 insertions, 134 deletions

diff --git a/src/logfile_sub_source.cc b/src/logfile_sub_source.cc
index aa356f9..b78b00c 100644
--- a/src/logfile_sub_source.cc
+++ b/src/logfile_sub_source.cc
@@ -35,23 +35,26 @@
 #include <sqlite3.h>
 
 #include "base/ansi_scrubber.hh"
-#include "base/humanize.time.hh"
-#include "base/injector.hh"
+#include "base/ansi_vars.hh"
+#include "base/fs_util.hh"
 #include "base/itertools.hh"
 #include "base/string_util.hh"
-#include "bound_tags.hh"
+#include "bookmarks.json.hh"
 #include "command_executor.hh"
 #include "config.h"
+#include "field_overlay_source.hh"
 #include "k_merge_tree.h"
+#include "lnav_util.hh"
 #include "log_accel.hh"
-#include "logfile_sub_source.cfg.hh"
 #include "md2attr_line.hh"
-#include "readline_highlighters.hh"
-#include "relative_time.hh"
+#include "ptimec.hh"
+#include "shlex.hh"
 #include "sql_util.hh"
 #include "vtab_module.hh"
 #include "yajlpp/yajlpp.hh"
 
+using namespace lnav::roles::literals;
+
 const bookmark_type_t logfile_sub_source::BM_ERRORS("error");
 const bookmark_type_t logfile_sub_source::BM_WARNINGS("warning");
 const bookmark_type_t logfile_sub_source::BM_FILES("file");
@@ -63,7 +66,7 @@ pretty_sql_callback(exec_context& ec, sqlite3_stmt* stmt)
         return 0;
     }
 
-    int ncols = sqlite3_column_count(stmt);
+    const auto ncols = sqlite3_column_count(stmt);
 
     for (int lpc = 0; lpc < ncols; lpc++) {
         if (!ec.ec_accumulator->empty()) {
@@ -78,6 +81,44 @@ pretty_sql_callback(exec_context& ec, sqlite3_stmt* stmt)
         ec.ec_accumulator->append(res);
     }
 
+    for (int lpc = 0; lpc < ncols; lpc++) {
+        const auto* colname = sqlite3_column_name(stmt, lpc);
+        auto* raw_value = sqlite3_column_value(stmt, lpc);
+        auto value_type = sqlite3_value_type(raw_value);
+        scoped_value_t value;
+
+        switch (value_type) {
+            case SQLITE_INTEGER:
+                value = (int64_t) sqlite3_value_int64(raw_value);
+                break;
+            case SQLITE_FLOAT:
+                value = sqlite3_value_double(raw_value);
+                break;
+            case SQLITE_NULL:
+                value = null_value_t{};
+                break;
+            default:
+                value = string_fragment::from_bytes(
+                    sqlite3_value_text(raw_value),
+                    sqlite3_value_bytes(raw_value));
+                break;
+        }
+        if (!ec.ec_local_vars.empty() && !ec.ec_dry_run) {
+            if (sql_ident_needs_quote(colname)) {
+                continue;
+            }
+            auto& vars = ec.ec_local_vars.top();
+
+            if (vars.find(colname) != vars.end()) {
+                continue;
+            }
+
+            if (value.is<string_fragment>()) {
+                value = value.get<string_fragment>().to_string();
+            }
+            vars[colname] = value;
+        }
+    }
     return 0;
 }
 
@@ -140,13 +181,46 @@ logfile_sub_source::find(const char* fn, content_line_t& line_base)
     return retval;
 }
 
+struct filtered_logline_cmp {
+    filtered_logline_cmp(const logfile_sub_source& lc) : llss_controller(lc) {}
+
+    bool operator()(const uint32_t& lhs, const uint32_t& rhs) const
+    {
+        content_line_t cl_lhs = (content_line_t) llss_controller.lss_index[lhs];
+        content_line_t cl_rhs = (content_line_t) llss_controller.lss_index[rhs];
+        logline* ll_lhs = this->llss_controller.find_line(cl_lhs);
+        logline* ll_rhs = this->llss_controller.find_line(cl_rhs);
+
+        if (ll_lhs == nullptr) {
+            return true;
+        }
+        if (ll_rhs == nullptr) {
+            return false;
+        }
+        return (*ll_lhs) < (*ll_rhs);
+    }
+
+    bool operator()(const uint32_t& lhs, const struct timeval& rhs) const
+    {
+        content_line_t cl_lhs = (content_line_t) llss_controller.lss_index[lhs];
+        logline* ll_lhs = this->llss_controller.find_line(cl_lhs);
+
+        if (ll_lhs == nullptr) {
+            return true;
+        }
+        return (*ll_lhs) < rhs;
+    }
+
+    const logfile_sub_source& llss_controller;
+};
+
 nonstd::optional<vis_line_t>
 logfile_sub_source::find_from_time(const struct timeval& start) const
 {
-    auto lb = lower_bound(this->lss_filtered_index.begin(),
-                          this->lss_filtered_index.end(),
-                          start,
-                          filtered_logline_cmp(*this));
+    auto lb = std::lower_bound(this->lss_filtered_index.begin(),
+                               this->lss_filtered_index.end(),
+                               start,
+                               filtered_logline_cmp(*this));
     if (lb != this->lss_filtered_index.end()) {
         return vis_line_t(lb - this->lss_filtered_index.begin());
     }
@@ -160,6 +234,11 @@ logfile_sub_source::text_value_for_line(textview_curses& tc,
                                         std::string& value_out,
                                         line_flags_t flags)
 {
+    if (this->lss_indexing_in_progress) {
+        value_out = "";
+        return;
+    }
+
     content_line_t line(0);
 
     require_ge(row, 0);
@@ -224,10 +303,12 @@ logfile_sub_source::text_value_for_line(textview_curses& tc,
         exec_context ec(
             &this->lss_token_values, pretty_sql_callback, pretty_pipe_callback);
         std::string rewritten_line;
+        db_label_source rewrite_label_source;
 
         ec.with_perms(exec_context::perm_t::READ_ONLY);
         ec.ec_local_vars.push(std::map<std::string, scoped_value_t>());
         ec.ec_top_line = vis_line_t(row);
+        ec.ec_label_source_stack.push_back(&rewrite_label_source);
         add_ansi_vars(ec.ec_global_vars);
         add_global_vars(ec);
         format->rewrite(ec, sbr, this->lss_token_attrs, rewritten_line);
@@ -235,10 +316,19 @@ logfile_sub_source::text_value_for_line(textview_curses& tc,
         value_out = this->lss_token_value;
     }
 
-    if ((this->lss_token_file->is_time_adjusted()
-         || format->lf_timestamp_flags & ETF_MACHINE_ORIENTED
-         || !(format->lf_timestamp_flags & ETF_DAY_SET)
-         || !(format->lf_timestamp_flags & ETF_MONTH_SET))
+    {
+        auto lr = line_range{0, (int) this->lss_token_value.length()};
+        this->lss_token_attrs.emplace_back(lr, SA_ORIGINAL_LINE.value());
+    }
+
+    if (!this->lss_token_line->is_continued() && !format->lf_formatted_lines
+        && (this->lss_token_file->is_time_adjusted()
+            || ((format->lf_timestamp_flags & ETF_ZONE_SET
+                 || format->lf_date_time.dts_default_zone != nullptr)
+                && format->lf_date_time.dts_zoned_to_local)
+            || format->lf_timestamp_flags & ETF_MACHINE_ORIENTED
+            || !(format->lf_timestamp_flags & ETF_DAY_SET)
+            || !(format->lf_timestamp_flags & ETF_MONTH_SET))
         && format->lf_date_time.dts_fmt_lock != -1)
     {
         auto time_attr
@@ -256,8 +346,10 @@ logfile_sub_source::text_value_for_line(textview_curses& tc,
                 || !(format->lf_timestamp_flags & ETF_MONTH_SET))
             {
                 adjusted_time = this->lss_token_line->get_timeval();
-                fmt = "%Y-%m-%d %H:%M:%S.%f";
-                if (format->lf_timestamp_flags & ETF_MICROS_SET) {
+                if (format->lf_timestamp_flags
+                    & (ETF_MICROS_SET | ETF_NANOS_SET))
+                {
+                    fmt = "%Y-%m-%d %H:%M:%S.%f";
                     struct timeval actual_tv;
                     struct exttm tm;
                     if (format->lf_date_time.scan(
@@ -270,6 +362,10 @@ logfile_sub_source::text_value_for_line(textview_curses& tc,
                     {
                         adjusted_time.tv_usec = actual_tv.tv_usec;
                     }
+                } else if (format->lf_timestamp_flags & ETF_MILLIS_SET) {
+                    fmt = "%Y-%m-%d %H:%M:%S.%L";
+                } else {
+                    fmt = "%Y-%m-%d %H:%M:%S";
                 }
                 gmtime_r(&adjusted_time.tv_sec, &adjusted_tm.et_tm);
                 adjusted_tm.et_nsec
@@ -284,6 +380,14 @@ logfile_sub_source::text_value_for_line(textview_curses& tc,
                     = std::chrono::duration_cast<std::chrono::nanoseconds>(
                           std::chrono::microseconds{adjusted_time.tv_usec})
                           .count();
+                adjusted_tm.et_flags = format->lf_timestamp_flags;
+                if (format->lf_timestamp_flags & ETF_ZONE_SET
+                    && format->lf_date_time.dts_zoned_to_local)
+                {
+                    adjusted_tm.et_flags &= ~ETF_Z_IS_UTC;
+                }
+                adjusted_tm.et_gmtoff
+                    = format->lf_date_time.dts_local_offset_cache;
                 len = format->lf_date_time.ftime(
                     buffer,
                     sizeof(buffer),
@@ -303,14 +407,14 @@ logfile_sub_source::text_value_for_line(textview_curses& tc,
         std::string name;
         if (this->lss_flags & F_FILENAME) {
             file_offset_end = this->lss_filename_width;
-            name = this->lss_token_file->get_filename();
+            name = fmt::to_string(this->lss_token_file->get_filename());
             if (file_offset_end < name.size()) {
                 file_offset_end = name.size();
                 this->lss_filename_width = name.size();
             }
         } else {
             file_offset_end = this->lss_basename_width;
-            name = this->lss_token_file->get_unique_path();
+            name = fmt::to_string(this->lss_token_file->get_unique_path());
             if (file_offset_end < name.size()) {
                 file_offset_end = name.size();
                 this->lss_basename_width = name.size();
@@ -324,35 +428,12 @@ logfile_sub_source::text_value_for_line(textview_curses& tc,
         value_out.insert(0, 1, ' ');
     }
 
-    if (this->lss_flags & F_TIME_OFFSET) {
-        auto curr_tv = this->lss_token_line->get_timeval();
-        struct timeval diff_tv;
+    if (this->tas_display_time_offset) {
         auto row_vl = vis_line_t(row);
-
-        auto prev_umark
-            = tc.get_bookmarks()[&textview_curses::BM_USER].prev(row_vl);
-        auto next_umark
-            = tc.get_bookmarks()[&textview_curses::BM_USER].next(row_vl);
-        auto prev_emark
-            = tc.get_bookmarks()[&textview_curses::BM_USER_EXPR].prev(row_vl);
-        auto next_emark
-            = tc.get_bookmarks()[&textview_curses::BM_USER_EXPR].next(row_vl);
-        if (!prev_umark && !prev_emark && (next_umark || next_emark)) {
-            auto next_line = this->find_line(this->at(
-                std::max(next_umark.value_or(0), next_emark.value_or(0))));
-
-            diff_tv = curr_tv - next_line->get_timeval();
-        } else {
-            auto prev_row
-                = std::max(prev_umark.value_or(0), prev_emark.value_or(0));
-            auto first_line = this->find_line(this->at(prev_row));
-            auto start_tv = first_line->get_timeval();
-            diff_tv = curr_tv - start_tv;
-        }
-
-        auto relstr = humanize::time::duration::from_tv(diff_tv).to_string();
+        auto relstr = this->get_time_offset_for_line(tc, row_vl);
         value_out = fmt::format(FMT_STRING("{: >12}|{}"), relstr, value_out);
     }
+
     this->lss_in_value_for_line = false;
 }
 
@@ -361,6 +442,10 @@ logfile_sub_source::text_attrs_for_line(textview_curses& lv,
                                         int row,
                                         string_attrs_t& value_out)
 {
+    if (this->lss_indexing_in_progress) {
+        return;
+    }
+
     view_colors& vc = view_colors::singleton();
     logline* next_line = nullptr;
     struct line_range lr;
@@ -383,8 +468,7 @@ logfile_sub_source::text_attrs_for_line(textview_curses& lv,
     const auto& line_values = this->lss_token_values;
 
     lr.lr_start = 0;
-    lr.lr_end = this->lss_token_value.length();
-    value_out.emplace_back(lr, SA_ORIGINAL_LINE.value());
+    lr.lr_end = -1;
     value_out.emplace_back(
         lr, SA_LEVEL.value(this->lss_token_line->get_msg_level()));
 
@@ -459,8 +543,7 @@ logfile_sub_source::text_attrs_for_line(textview_curses& lv,
         value_out.emplace_back(lr, VC_GRAPHIC.value(graph));
 
         if (!(this->lss_token_flags & RF_FULL)) {
-            bookmark_vector<vis_line_t>& bv_search
-                = bm[&textview_curses::BM_SEARCH];
+            auto& bv_search = bm[&textview_curses::BM_SEARCH];
 
             if (binary_search(std::begin(bv_search),
                               std::end(bv_search),
@@ -492,7 +575,7 @@ logfile_sub_source::text_attrs_for_line(textview_curses& lv,
                                    this->lss_token_file->get_filename())));
     }
 
-    if (this->lss_flags & F_TIME_OFFSET) {
+    if (this->tas_display_time_offset) {
         time_offset_end = 13;
         lr.lr_start = 0;
         lr.lr_end = time_offset_end;
@@ -526,20 +609,15 @@ logfile_sub_source::text_attrs_for_line(textview_curses& lv,
         lr, SA_FORMAT.value(this->lss_token_file->get_format()->get_name()));
 
     {
-        const auto& bv = lv.get_bookmarks()[&textview_curses::BM_META];
-        bookmark_vector<vis_line_t>::const_iterator bv_iter;
-
-        bv_iter = lower_bound(bv.begin(), bv.end(), vis_line_t(row + 1));
-        if (bv_iter != bv.begin()) {
-            --bv_iter;
-            auto line_meta_opt = this->find_bookmark_metadata(*bv_iter);
-
-            if (line_meta_opt && !line_meta_opt.value()->bm_name.empty()) {
-                lr.lr_start = 0;
-                lr.lr_end = -1;
-                value_out.emplace_back(
-                    lr, logline::L_PARTITION.value(line_meta_opt.value()));
-            }
+        auto line_meta_context = this->get_bookmark_metadata_context(
+            vis_line_t(row + 1), bookmark_metadata::categories::partition);
+        if (line_meta_context.bmc_current_metadata) {
+            lr.lr_start = 0;
+            lr.lr_end = -1;
+            value_out.emplace_back(
+                lr,
+                logline::L_PARTITION.value(
+                    line_meta_context.bmc_current_metadata.value()));
         }
 
         auto line_meta_opt = this->find_bookmark_metadata(vis_line_t(row));
@@ -620,6 +698,53 @@ logfile_sub_source::text_attrs_for_line(textview_curses& lv,
     }
 }
 
+struct logline_cmp {
+    logline_cmp(logfile_sub_source& lc) : llss_controller(lc) {}
+
+    bool operator()(const content_line_t& lhs, const content_line_t& rhs) const
+    {
+        logline* ll_lhs = this->llss_controller.find_line(lhs);
+        logline* ll_rhs = this->llss_controller.find_line(rhs);
+
+        return (*ll_lhs) < (*ll_rhs);
+    }
+
+    bool operator()(const uint32_t& lhs, const uint32_t& rhs) const
+    {
+        content_line_t cl_lhs = (content_line_t) llss_controller.lss_index[lhs];
+        content_line_t cl_rhs = (content_line_t) llss_controller.lss_index[rhs];
+        logline* ll_lhs = this->llss_controller.find_line(cl_lhs);
+        logline* ll_rhs = this->llss_controller.find_line(cl_rhs);
+
+        return (*ll_lhs) < (*ll_rhs);
+    }
+#if 0
+        bool operator()(const indexed_content &lhs, const indexed_content &rhs)
+        {
+            logline *ll_lhs = this->llss_controller.find_line(lhs.ic_value);
+            logline *ll_rhs = this->llss_controller.find_line(rhs.ic_value);
+
+            return (*ll_lhs) < (*ll_rhs);
+        }
+#endif
+
+    bool operator()(const content_line_t& lhs, const time_t& rhs) const
+    {
+        logline* ll_lhs = this->llss_controller.find_line(lhs);
+
+        return *ll_lhs < rhs;
+    }
+
+    bool operator()(const content_line_t& lhs, const struct timeval& rhs) const
+    {
+        logline* ll_lhs = this->llss_controller.find_line(lhs);
+
+        return *ll_lhs < rhs;
+    }
+
+    logfile_sub_source& llss_controller;
+};
+
 logfile_sub_source::rebuild_result
 logfile_sub_source::rebuild_index(
     nonstd::optional<ui_clock::time_point> deadline)
@@ -628,6 +753,9 @@ logfile_sub_source::rebuild_index(
         return rebuild_result::rr_no_change;
     }
 
+    this->lss_indexing_in_progress = true;
+    auto fin = finally([this]() { this->lss_indexing_in_progress = false; });
+
     iterator iter;
     size_t total_lines = 0;
     bool full_sort = false;
@@ -641,6 +769,7 @@ logfile_sub_source::rebuild_index(
     if (force) {
         log_debug("forced to full rebuild");
         retval = rebuild_result::rr_full_rebuild;
+        full_sort = true;
     }
 
     std::vector<size_t> file_order(this->lss_files.size());
@@ -678,6 +807,7 @@ logfile_sub_source::rebuild_index(
                           ld.ld_file_index);
                 force = true;
                 retval = rebuild_result::rr_full_rebuild;
+                full_sort = true;
             }
         } else {
             if (time_left && deadline && ui_clock::now() > deadline.value()) {
@@ -733,6 +863,11 @@ logfile_sub_source::rebuild_index(
                                     {
                                         lowest_tv = new_file_line.get_timeval();
                                     }
+                                } else {
+                                    log_debug(
+                                        "already doing full rebuild, doing "
+                                        "full_sort as well");
+                                    full_sort = true;
                                 }
                             }
                         }
@@ -757,8 +892,12 @@ logfile_sub_source::rebuild_index(
     }
 
     if (this->lss_index.reserve(total_lines)) {
+        // The index array was reallocated, just do a full sort/rebuild since
+        // it's been cleared out.
+        log_debug("expanding index capacity %zu", this->lss_index.ba_capacity);
         force = true;
         retval = rebuild_result::rr_full_rebuild;
+        full_sort = true;
     }
 
     auto& vis_bm = this->tss_view->get_bookmarks();
@@ -854,12 +993,13 @@ logfile_sub_source::rebuild_index(
             if (lf == nullptr) {
                 continue;
             }
-            this->lss_longest_line = std::max(this->lss_longest_line,
-                                              lf->get_longest_line_length());
-            this->lss_basename_width = std::max(this->lss_basename_width,
-                                                lf->get_unique_path().size());
-            this->lss_filename_width
-                = std::max(this->lss_filename_width, lf->get_filename().size());
+            this->lss_longest_line = std::max(
+                this->lss_longest_line, lf->get_longest_line_length() + 1);
+            this->lss_basename_width
+                = std::max(this->lss_basename_width,
+                           lf->get_unique_path().native().size());
+            this->lss_filename_width = std::max(
+                this->lss_filename_width, lf->get_filename().native().size());
         }
 
         if (full_sort) {
@@ -873,13 +1013,41 @@ logfile_sub_source::rebuild_index(
                 for (size_t line_index = 0; line_index < lf->size();
                      line_index++)
                 {
-                    if ((*lf)[line_index].is_ignored()) {
+                    const auto lf_iter
+                        = ld->get_file_ptr()->begin() + line_index;
+                    if (lf_iter->is_ignored()) {
                         continue;
                     }
 
                     content_line_t con_line(
                         ld->ld_file_index * MAX_LINES_PER_FILE + line_index);
 
+                    if (lf_iter->is_meta_marked()) {
+                        auto start_iter = lf_iter;
+                        while (start_iter->is_continued()) {
+                            --start_iter;
+                        }
+                        int start_index
+                            = start_iter - ld->get_file_ptr()->begin();
+                        content_line_t start_con_line(ld->ld_file_index
+                                                          * MAX_LINES_PER_FILE
+                                                      + start_index);
+
+                        auto& line_meta
+                            = ld->get_file_ptr()
+                                  ->get_bookmark_metadata()[start_index];
+                        if (line_meta.has(bookmark_metadata::categories::notes))
+                        {
+                            this->lss_user_marks[&textview_curses::BM_META]
+                                .insert_once(start_con_line);
+                        }
+                        if (line_meta.has(
+                                bookmark_metadata::categories::partition))
+                        {
+                            this->lss_user_marks[&textview_curses::BM_PARTITION]
+                                .insert_once(start_con_line);
+                        }
+                    }
                     this->lss_index.push_back(con_line);
                 }
             }
@@ -932,7 +1100,7 @@ logfile_sub_source::rebuild_index(
                     content_line_t con_line(file_index * MAX_LINES_PER_FILE
                                             + line_index);
 
-                    if (lf_iter->is_marked()) {
+                    if (lf_iter->is_meta_marked()) {
                         auto start_iter = lf_iter;
                         while (start_iter->is_continued()) {
                             --start_iter;
@@ -942,9 +1110,20 @@ logfile_sub_source::rebuild_index(
                         content_line_t start_con_line(
                             file_index * MAX_LINES_PER_FILE + start_index);
 
-                        this->lss_user_marks[&textview_curses::BM_META]
-                            .insert_once(start_con_line);
-                        lf_iter->set_mark(false);
+                        auto& line_meta
+                            = ld->get_file_ptr()
+                                  ->get_bookmark_metadata()[start_index];
+                        if (line_meta.has(bookmark_metadata::categories::notes))
+                        {
+                            this->lss_user_marks[&textview_curses::BM_META]
+                                .insert_once(start_con_line);
+                        }
+                        if (line_meta.has(
+                                bookmark_metadata::categories::partition))
+                        {
+                            this->lss_user_marks[&textview_curses::BM_PARTITION]
+                                .insert_once(start_con_line);
+                        }
                     }
                     this->lss_index.push_back(con_line);
                 }
@@ -985,7 +1164,7 @@ logfile_sub_source::rebuild_index(
              index_index < this->lss_index.size();
              index_index++)
         {
-            content_line_t cl = (content_line_t) this->lss_index[index_index];
+            const auto cl = (content_line_t) this->lss_index[index_index];
             uint64_t line_number;
             auto ld = this->find_data(cl, line_number);
 
@@ -1028,6 +1207,8 @@ logfile_sub_source::rebuild_index(
             }
         }
 
+        this->lss_indexing_in_progress = false;
+
         if (this->lss_index_delegate != nullptr) {
             this->lss_index_delegate->index_complete(*this);
         }
@@ -1060,7 +1241,7 @@ logfile_sub_source::rebuild_index(
 void
 logfile_sub_source::text_update_marks(vis_bookmarks& bm)
 {
-    std::shared_ptr<logfile> last_file;
+    logfile* last_file = nullptr;
     vis_line_t vl;
 
     bm[&BM_WARNINGS].clear();
@@ -1074,7 +1255,7 @@ logfile_sub_source::text_update_marks(vis_bookmarks& bm)
     for (; vl < (int) this->lss_filtered_index.size(); ++vl) {
         const content_line_t orig_cl = this->at(vl);
         content_line_t cl = orig_cl;
-        auto lf = this->find(cl);
+        auto lf = this->find_file_ptr(cl);
 
         for (auto& lss_user_mark : this->lss_user_marks) {
             if (binary_search(lss_user_mark.second.begin(),
@@ -1117,29 +1298,6 @@ logfile_sub_source::text_update_marks(vis_bookmarks& bm)
     }
 }
 
-log_accel::direction_t
-logfile_sub_source::get_line_accel_direction(vis_line_t vl)
-{
-    log_accel la;
-
-    while (vl >= 0) {
-        logline* curr_line = this->find_line(this->at(vl));
-
-        if (!curr_line->is_message()) {
-            --vl;
-            continue;
-        }
-
-        if (!la.add_point(curr_line->get_time_in_millis())) {
-            break;
-        }
-
-        --vl;
-    }
-
-    return la.get_direction();
-}
-
 void
 logfile_sub_source::text_filters_changed()
 {
@@ -1226,6 +1384,68 @@ bool
 logfile_sub_source::list_input_handle_key(listview_curses& lv, int ch)
 {
     switch (ch) {
+        case ' ': {
+            auto ov_vl = lv.get_overlay_selection();
+            if (ov_vl) {
+                auto* fos = dynamic_cast<field_overlay_source*>(
+                    lv.get_overlay_source());
+                auto iter = fos->fos_row_to_field_meta.find(ov_vl.value());
+                if (iter != fos->fos_row_to_field_meta.end()) {
+                    auto find_res = this->find_line_with_file(lv.get_top());
+                    if (find_res) {
+                        auto file_and_line = find_res.value();
+                        auto* format = file_and_line.first->get_format_ptr();
+                        auto fstates = format->get_field_states();
+                        auto state_iter = fstates.find(iter->second.lvm_name);
+                        if (state_iter != fstates.end()) {
+                            format->hide_field(iter->second.lvm_name,
+                                               !state_iter->second.is_hidden());
+                            lv.set_needs_update();
+                        }
+                    }
+                }
+                return true;
+            }
+            return false;
+        }
+        case '#': {
+            auto ov_vl = lv.get_overlay_selection();
+            if (ov_vl) {
+                auto* fos = dynamic_cast<field_overlay_source*>(
+                    lv.get_overlay_source());
+                auto iter = fos->fos_row_to_field_meta.find(ov_vl.value());
+                if (iter != fos->fos_row_to_field_meta.end()) {
+                    const auto& meta = iter->second;
+                    std::string cmd;
+
+                    switch (meta.to_chart_type()) {
+                        case chart_type_t::none:
+                            break;
+                        case chart_type_t::hist: {
+                            auto prql = fmt::format(
+                                FMT_STRING(
+                                    "from {} | stats.hist {} slice:'1h'"),
+                                meta.lvm_format.value()->get_name(),
+                                meta.lvm_name);
+                            cmd = fmt::format(FMT_STRING(":prompt sql ; '{}'"),
+                                              shlex::escape(prql));
+                            break;
+                        }
+                        case chart_type_t::spectro:
+                            cmd = fmt::format(FMT_STRING(":spectrogram {}"),
+                                              meta.lvm_name);
+                            break;
+                    }
+                    if (!cmd.empty()) {
+                        this->lss_exec_context
+                            ->with_provenance(exec_context::mouse_input{})
+                            ->execute(cmd);
+                    }
+                }
+                return true;
+            }
+            return false;
+        }
         case 'h':
         case 'H':
         case KEY_SLEFT:
@@ -1258,6 +1478,24 @@ logfile_sub_source::get_grepper()
         (grep_proc_sink<vis_line_t>*) &this->lss_meta_grepper);
 }
 
+/**
+ * Functor for comparing the ld_file field of the logfile_data struct.
+ */
+struct logfile_data_eq {
+    explicit logfile_data_eq(std::shared_ptr<logfile> lf)
+        : lde_file(std::move(lf))
+    {
+    }
+
+    bool operator()(
+        const std::unique_ptr<logfile_sub_source::logfile_data>& ld) const
+    {
+        return this->lde_file == ld->get_file();
+    }
+
+    std::shared_ptr<logfile> lde_file;
+};
+
 bool
 logfile_sub_source::insert_file(const std::shared_ptr<logfile>& lf)
 {
@@ -1483,6 +1721,26 @@ logfile_sub_source::eval_sql_filter(sqlite3_stmt* stmt,
             }
             continue;
         }
+        if (strcmp(name, ":log_annotations") == 0) {
+            const auto& bm = lf->get_bookmark_metadata();
+            auto line_number
+                = static_cast<uint32_t>(std::distance(lf->cbegin(), ll));
+            auto bm_iter = bm.find(line_number);
+            if (bm_iter != bm.end()
+                && !bm_iter->second.bm_annotations.la_pairs.empty())
+            {
+                const auto& meta = bm_iter->second;
+                auto anno_str = logmsg_annotations_handlers.to_string(
+                    meta.bm_annotations);
+
+                sqlite3_bind_text(stmt,
+                                  lpc + 1,
+                                  anno_str.c_str(),
+                                  anno_str.length(),
+                                  SQLITE_TRANSIENT);
+            }
+            continue;
+        }
         if (strcmp(name, ":log_tags") == 0) {
             const auto& bm = lf->get_bookmark_metadata();
             auto line_number
@@ -1529,7 +1787,7 @@ logfile_sub_source::eval_sql_filter(sqlite3_stmt* stmt,
             sqlite3_bind_text(stmt,
                               lpc + 1,
                               filename.c_str(),
-                              filename.length(),
+                              filename.native().length(),
                               SQLITE_STATIC);
             continue;
         }
@@ -1538,7 +1796,7 @@ logfile_sub_source::eval_sql_filter(sqlite3_stmt* stmt,
             sqlite3_bind_text(stmt,
                               lpc + 1,
                               filename.c_str(),
-                              filename.length(),
+                              filename.native().length(),
                               SQLITE_STATIC);
             continue;
         }
@@ -1565,14 +1823,11 @@ logfile_sub_source::eval_sql_filter(sqlite3_stmt* stmt,
         }
         if (strcmp(name, ":log_opid") == 0) {
             auto opid_attr_opt = get_string_attr(sa, logline::L_OPID);
-            if (opid_attr_opt) {
-                const auto& sar
-                    = opid_attr_opt.value().saw_string_attr->sa_range;
-
+            if (values.lvv_opid_value) {
                 sqlite3_bind_text(stmt,
                                   lpc + 1,
-                                  sbr.get_data_at(sar.lr_start),
-                                  sar.length(),
+                                  values.lvv_opid_value->c_str(),
+                                  values.lvv_opid_value->length(),
                                   SQLITE_STATIC);
             } else {
                 sqlite3_bind_null(stmt, lpc + 1);
@@ -1714,11 +1969,6 @@ logfile_sub_source::text_clear_marks(const bookmark_type_t* bm)
         for (iter = this->lss_user_marks[bm].begin();
              iter != this->lss_user_marks[bm].end();)
         {
-            auto line_meta_opt = this->find_bookmark_metadata(*iter);
-            if (line_meta_opt) {
-                ++iter;
-                continue;
-            }
             this->find_line(*iter)->set_mark(false);
             iter = this->lss_user_marks[bm].erase(iter);
         }
@@ -1891,25 +2141,30 @@ log_location_history::loc_history_forward(vis_line_t current_top)
 }
 
 bool
-sql_filter::matches(const logfile& lf,
-                    logfile::const_iterator ll,
+sql_filter::matches(nonstd::optional<line_source> ls_opt,
                     const shared_buffer_ref& line)
 {
-    if (!ll->is_message()) {
+    if (!ls_opt) {
+        return false;
+    }
+
+    auto ls = ls_opt;
+
+    if (!ls->ls_line->is_message()) {
         return false;
     }
     if (this->sf_filter_stmt == nullptr) {
         return false;
     }
 
-    auto lfp = lf.shared_from_this();
+    auto lfp = ls->ls_file.shared_from_this();
     auto ld = this->sf_log_source.find_data_i(lfp);
     if (ld == this->sf_log_source.end()) {
         return false;
     }
 
-    auto eval_res
-        = this->sf_log_source.eval_sql_filter(this->sf_filter_stmt, ld, ll);
+    auto eval_res = this->sf_log_source.eval_sql_filter(
+        this->sf_filter_stmt, ld, ls->ls_line);
     if (eval_res.unwrapOr(true)) {
         return false;
     }
@@ -1949,6 +2204,21 @@ logfile_sub_source::meta_grepper::grep_value_for_line(vis_line_t line,
             value_out.append(tag);
             value_out.append("\x1c");
         }
+        value_out.append("\x1c");
+        for (const auto& pair : bm.bm_annotations.la_pairs) {
+            value_out.append(pair.first);
+            value_out.append("\x1c");
+
+            md2attr_line mdal;
+
+            auto parse_res = md4cpp::parse(pair.second, mdal);
+            if (parse_res.isOk()) {
+                value_out.append(parse_res.unwrap().get_string());
+            } else {
+                value_out.append(pair.second);
+            }
+            value_out.append("\x1c");
+        }
     }
 
     return !this->lmg_done;
@@ -1958,8 +2228,8 @@ vis_line_t
 logfile_sub_source::meta_grepper::grep_initial_line(vis_line_t start,
                                                     vis_line_t highest)
 {
-    vis_bookmarks& bm = this->lmg_source.tss_view->get_bookmarks();
-    bookmark_vector<vis_line_t>& bv = bm[&textview_curses::BM_META];
+    auto& bm = this->lmg_source.tss_view->get_bookmarks();
+    auto& bv = bm[&textview_curses::BM_META];
 
     if (bv.empty()) {
         return -1_vl;
@@ -1970,8 +2240,8 @@ logfile_sub_source::meta_grepper::grep_initial_line(vis_line_t start,
 void
 logfile_sub_source::meta_grepper::grep_next_line(vis_line_t& line)
 {
-    vis_bookmarks& bm = this->lmg_source.tss_view->get_bookmarks();
-    bookmark_vector<vis_line_t>& bv = bm[&textview_curses::BM_META];
+    auto& bm = this->lmg_source.tss_view->get_bookmarks();
+    auto& bv = bm[&textview_curses::BM_META];
 
     auto line_opt = bv.next(vis_line_t(line));
     if (!line_opt) {
@@ -2132,6 +2402,21 @@ timestamp_poss()
     return retval;
 }
 
+static attr_line_t
+to_display(const std::shared_ptr<logfile>& lf)
+{
+    attr_line_t retval;
+
+    if (lf->get_open_options().loo_piper) {
+        if (!lf->get_open_options().loo_piper->is_finished()) {
+            retval.append("\u21bb "_list_glyph);
+        }
+    }
+    retval.append(lf->get_unique_path());
+
+    return retval;
+}
+
 void
 logfile_sub_source::text_crumbs_for_line(int line,
                                          std::vector<breadcrumb::crumb>& crumbs)
@@ -2142,7 +2427,43 @@ logfile_sub_source::text_crumbs_for_line(int line,
         return;
     }
 
-    auto line_pair_opt = this->find_line_with_file(vis_line_t(line));
+    auto vl = vis_line_t(line);
+    auto bmc = this->get_bookmark_metadata_context(
+        vl, bookmark_metadata::categories::partition);
+    if (bmc.bmc_current_metadata) {
+        const auto& name = bmc.bmc_current_metadata.value()->bm_name;
+        auto key = text_anchors::to_anchor_string(name);
+        auto display = attr_line_t()
+                           .append("\u2291 "_symbol)
+                           .append(lnav::roles::variable(name));
+        crumbs.emplace_back(
+            key,
+            display,
+            [this]() -> std::vector<breadcrumb::possibility> {
+                auto& vb = this->tss_view->get_bookmarks();
+                const auto& bv = vb[&textview_curses::BM_PARTITION];
+                std::vector<breadcrumb::possibility> retval;
+
+                for (const auto& vl : bv) {
+                    auto meta_opt = this->find_bookmark_metadata(vl);
+                    if (!meta_opt || meta_opt.value()->bm_name.empty()) {
+                        continue;
+                    }
+
+                    const auto& name = meta_opt.value()->bm_name;
+                    retval.emplace_back(text_anchors::to_anchor_string(name),
+                                        name);
+                }
+
+                return retval;
+            },
+            [ec = this->lss_exec_context](const auto& part) {
+                ec->execute(fmt::format(FMT_STRING(":goto {}"),
+                                        part.template get<std::string>()));
+            });
+    }
+
+    auto line_pair_opt = this->find_line_with_file(vl);
     if (!line_pair_opt) {
         return;
     }
@@ -2180,7 +2501,8 @@ logfile_sub_source::text_crumbs_for_line(int line,
                        return breadcrumb::possibility{
                            elem.to_string(),
                        };
-                   });
+                   })
+                | lnav::itertools::to_vector();
         },
         [ec = this->lss_exec_context](const auto& format_name) {
             static const std::string MOVE_STMT = R"(;UPDATE lnav_views
@@ -2198,9 +2520,7 @@ logfile_sub_source::text_crumbs_for_line(int line,
     auto file_line_number = std::distance(lf->begin(), msg_start_iter);
     crumbs.emplace_back(
         lf->get_unique_path(),
-        attr_line_t()
-            .append(lf->get_unique_path())
-            .appendf(FMT_STRING("[{:L}]"), file_line_number),
+        to_display(lf).appendf(FMT_STRING("[{:L}]"), file_line_number),
         [this]() -> std::vector<breadcrumb::possibility> {
             return this->lss_files
                 | lnav::itertools::filter_in([](const auto& file_data) {
@@ -2209,8 +2529,7 @@ logfile_sub_source::text_crumbs_for_line(int line,
                 | lnav::itertools::map([](const auto& file_data) {
                        return breadcrumb::possibility{
                            file_data->get_file_ptr()->get_unique_path(),
-                           attr_line_t(
-                               file_data->get_file_ptr()->get_unique_path()),
+                           to_display(file_data->get_file()),
                        };
                    });
         },
@@ -2254,10 +2573,10 @@ logfile_sub_source::text_crumbs_for_line(int line,
                     if (file_data->get_file_ptr() == nullptr) {
                         continue;
                     }
-                    safe::ReadAccess<logfile::safe_opid_map> r_opid_map(
+                    safe::ReadAccess<logfile::safe_opid_state> r_opid_map(
                         file_data->get_file_ptr()->get_opids());
 
-                    for (const auto& pair : *r_opid_map) {
+                    for (const auto& pair : r_opid_map->los_opid_ranges) {
                         retval.emplace_back(pair.first.to_string());
                     }
                 }
@@ -2424,8 +2743,62 @@ logfile_sub_source::get_bookmark_metadata(content_line_t cl)
     return line_pair.first->get_bookmark_metadata()[line_number];
 }
 
+logfile_sub_source::bookmark_metadata_context
+logfile_sub_source::get_bookmark_metadata_context(
+    vis_line_t vl, bookmark_metadata::categories desired) const
+{
+    const auto& vb = this->tss_view->get_bookmarks();
+    const auto bv_iter
+        = vb.find(desired == bookmark_metadata::categories::partition
+                      ? &textview_curses::BM_PARTITION
+                      : &textview_curses::BM_META);
+    if (bv_iter == vb.end()) {
+        return bookmark_metadata_context{};
+    }
+
+    const auto& bv = bv_iter->second;
+    auto vl_iter = std::lower_bound(bv.begin(), bv.end(), vl + 1_vl);
+
+    nonstd::optional<vis_line_t> next_line;
+    for (auto next_vl_iter = vl_iter; next_vl_iter != bv.end(); ++next_vl_iter)
+    {
+        auto bm_opt = this->find_bookmark_metadata(*next_vl_iter);
+        if (!bm_opt) {
+            continue;
+        }
+
+        if (bm_opt.value()->has(desired)) {
+            next_line = *next_vl_iter;
+            break;
+        }
+    }
+    if (vl_iter == bv.begin()) {
+        return bookmark_metadata_context{
+            nonstd::nullopt, nonstd::nullopt, next_line};
+    }
+
+    --vl_iter;
+    while (true) {
+        auto bm_opt = this->find_bookmark_metadata(*vl_iter);
+        if (bm_opt) {
+            if (bm_opt.value()->has(desired)) {
+                return bookmark_metadata_context{
+                    *vl_iter, bm_opt.value(), next_line};
+            }
+        }
+
+        if (vl_iter == bv.begin()) {
+            return bookmark_metadata_context{
+                nonstd::nullopt, nonstd::nullopt, next_line};
+        }
+        --vl_iter;
+    }
+    return bookmark_metadata_context{
+        nonstd::nullopt, nonstd::nullopt, next_line};
+}
+
 nonstd::optional<bookmark_metadata*>
-logfile_sub_source::find_bookmark_metadata(content_line_t cl)
+logfile_sub_source::find_bookmark_metadata(content_line_t cl) const
 {
     auto line_pair = this->find_line_with_file(cl).value();
     auto line_number = static_cast<uint32_t>(
@@ -2465,3 +2838,240 @@ logfile_sub_source::clear_bookmark_metadata()
         ld->get_file_ptr()->get_bookmark_metadata().clear();
     }
 }
+
+void
+logfile_sub_source::increase_line_context()
+{
+    auto old_flags = this->lss_flags;
+
+    if (this->lss_flags & F_FILENAME) {
+        // Nothing to do
+    } else if (this->lss_flags & F_BASENAME) {
+        this->lss_flags &= ~F_NAME_MASK;
+        this->lss_flags |= F_FILENAME;
+    } else {
+        this->lss_flags |= F_BASENAME;
+    }
+    if (old_flags != this->lss_flags) {
+        this->clear_line_size_cache();
+    }
+}
+
+bool
+logfile_sub_source::decrease_line_context()
+{
+    auto old_flags = this->lss_flags;
+
+    if (this->lss_flags & F_FILENAME) {
+        this->lss_flags &= ~F_NAME_MASK;
+        this->lss_flags |= F_BASENAME;
+    } else if (this->lss_flags & F_BASENAME) {
+        this->lss_flags &= ~F_NAME_MASK;
+    }
+    if (old_flags != this->lss_flags) {
+        this->clear_line_size_cache();
+
+        return true;
+    }
+
+    return false;
+}
+
+size_t
+logfile_sub_source::get_filename_offset() const
+{
+    if (this->lss_flags & F_FILENAME) {
+        return this->lss_filename_width;
+    } else if (this->lss_flags & F_BASENAME) {
+        return this->lss_basename_width;
+    }
+
+    return 0;
+}
+
+void
+logfile_sub_source::clear_min_max_log_times()
+{
+    if (this->lss_min_log_time.tv_sec != 0
+        || this->lss_min_log_time.tv_usec != 0
+        || this->lss_max_log_time.tv_sec != std::numeric_limits<time_t>::max()
+        || this->lss_max_log_time.tv_usec != 0)
+    {
+        memset(&this->lss_min_log_time, 0, sizeof(this->lss_min_log_time));
+        this->lss_max_log_time.tv_sec = std::numeric_limits<time_t>::max();
+        this->lss_max_log_time.tv_usec = 0;
+        this->text_filters_changed();
+    }
+}
+
+size_t
+logfile_sub_source::file_count() const
+{
+    size_t retval = 0;
+    const_iterator iter;
+
+    for (iter = this->cbegin(); iter != this->cend(); ++iter) {
+        if (*iter != nullptr && (*iter)->get_file() != nullptr) {
+            retval += 1;
+        }
+    }
+
+    return retval;
+}
+
+size_t
+logfile_sub_source::text_size_for_line(textview_curses& tc,
+                                       int row,
+                                       text_sub_source::line_flags_t flags)
+{
+    size_t index = row % LINE_SIZE_CACHE_SIZE;
+
+    if (this->lss_line_size_cache[index].first != row) {
+        std::string value;
+
+        this->text_value_for_line(tc, row, value, flags);
+        scrub_ansi_string(value, nullptr);
+        this->lss_line_size_cache[index].second
+            = string_fragment::from_str(value).column_width();
+        this->lss_line_size_cache[index].first = row;
+    }
+    return this->lss_line_size_cache[index].second;
+}
+
+int
+logfile_sub_source::get_filtered_count_for(size_t filter_index) const
+{
+    int retval = 0;
+
+    for (const auto& ld : this->lss_files) {
+        retval += ld->ld_filter_state.lfo_filter_state
+                      .tfs_filter_hits[filter_index];
+    }
+
+    return retval;
+}
+
+nonstd::optional<vis_line_t>
+logfile_sub_source::row_for(const row_info& ri)
+{
+    auto lb = std::lower_bound(this->lss_filtered_index.begin(),
+                               this->lss_filtered_index.end(),
+                               ri.ri_time,
+                               filtered_logline_cmp(*this));
+    if (lb != this->lss_filtered_index.end()) {
+        auto first_lb = lb;
+        while (true) {
+            auto cl = this->lss_index[*lb];
+            if (content_line_t(ri.ri_id) == cl) {
+                first_lb = lb;
+                break;
+            }
+            auto ll = this->find_line(cl);
+            if (ll->get_timeval() != ri.ri_time) {
+                break;
+            }
+            ++lb;
+        }
+
+        return vis_line_t(first_lb - this->lss_filtered_index.begin());
+    }
+
+    return nonstd::nullopt;
+}
+
+nonstd::optional<vis_line_t>
+logfile_sub_source::row_for_anchor(const std::string& id)
+{
+    auto& vb = this->tss_view->get_bookmarks();
+    const auto& bv = vb[&textview_curses::BM_PARTITION];
+
+    for (const auto& vl : bv) {
+        auto meta_opt = this->find_bookmark_metadata(vl);
+        if (!meta_opt || meta_opt.value()->bm_name.empty()) {
+            continue;
+        }
+
+        const auto& name = meta_opt.value()->bm_name;
+        if (id == text_anchors::to_anchor_string(name)) {
+            return vl;
+        }
+    }
+
+    return nonstd::nullopt;
+}
+
+nonstd::optional<vis_line_t>
+logfile_sub_source::adjacent_anchor(vis_line_t vl, text_anchors::direction dir)
+{
+    auto bmc = this->get_bookmark_metadata_context(
+        vl, bookmark_metadata::categories::partition);
+    switch (dir) {
+        case text_anchors::direction::prev: {
+            if (bmc.bmc_current && bmc.bmc_current.value() != vl) {
+                return bmc.bmc_current;
+            }
+            if (!bmc.bmc_current || bmc.bmc_current.value() == 0_vl) {
+                return 0_vl;
+            }
+            auto prev_bmc = this->get_bookmark_metadata_context(
+                bmc.bmc_current.value() - 1_vl,
+                bookmark_metadata::categories::partition);
+            if (!prev_bmc.bmc_current) {
+                return 0_vl;
+            }
+            return prev_bmc.bmc_current;
+        }
+        case text_anchors::direction::next:
+            return bmc.bmc_next_line;
+    }
+    return nonstd::nullopt;
+}
+
+nonstd::optional<std::string>
+logfile_sub_source::anchor_for_row(vis_line_t vl)
+{
+    auto line_meta = this->get_bookmark_metadata_context(
+        vl, bookmark_metadata::categories::partition);
+    if (!line_meta.bmc_current_metadata) {
+        return nonstd::nullopt;
+    }
+
+    return text_anchors::to_anchor_string(
+        line_meta.bmc_current_metadata.value()->bm_name);
+}
+
+std::unordered_set<std::string>
+logfile_sub_source::get_anchors()
+{
+    auto& vb = this->tss_view->get_bookmarks();
+    const auto& bv = vb[&textview_curses::BM_PARTITION];
+    std::unordered_set<std::string> retval;
+
+    for (const auto& vl : bv) {
+        auto meta_opt = this->find_bookmark_metadata(vl);
+        if (!meta_opt || meta_opt.value()->bm_name.empty()) {
+            continue;
+        }
+
+        const auto& name = meta_opt.value()->bm_name;
+        retval.emplace(text_anchors::to_anchor_string(name));
+    }
+
+    return retval;
+}
+
+bool
+logfile_sub_source::text_handle_mouse(
+    textview_curses& tc,
+    const listview_curses::display_line_content_t& mouse_line,
+    mouse_event& me)
+{
+    if (tc.get_overlay_selection()) {
+        if (me.is_click_in(mouse_button_t::BUTTON_LEFT, 2, 4)) {
+            this->list_input_handle_key(tc, ' ');
+        } else if (me.is_click_in(mouse_button_t::BUTTON_LEFT, 5, 6)) {
+            this->list_input_handle_key(tc, '#');
+        }
+    }
+    return true;
+}