diff options
Diffstat (limited to '')
| -rw-r--r-- | src/logfile_sub_source.cc | 878 |
1 files changed, 744 insertions, 134 deletions
diff --git a/src/logfile_sub_source.cc b/src/logfile_sub_source.cc index aa356f9..b78b00c 100644 --- a/src/logfile_sub_source.cc +++ b/src/logfile_sub_source.cc @@ -35,23 +35,26 @@ #include <sqlite3.h> #include "base/ansi_scrubber.hh" -#include "base/humanize.time.hh" -#include "base/injector.hh" +#include "base/ansi_vars.hh" +#include "base/fs_util.hh" #include "base/itertools.hh" #include "base/string_util.hh" -#include "bound_tags.hh" +#include "bookmarks.json.hh" #include "command_executor.hh" #include "config.h" +#include "field_overlay_source.hh" #include "k_merge_tree.h" +#include "lnav_util.hh" #include "log_accel.hh" -#include "logfile_sub_source.cfg.hh" #include "md2attr_line.hh" -#include "readline_highlighters.hh" -#include "relative_time.hh" +#include "ptimec.hh" +#include "shlex.hh" #include "sql_util.hh" #include "vtab_module.hh" #include "yajlpp/yajlpp.hh" +using namespace lnav::roles::literals; + const bookmark_type_t logfile_sub_source::BM_ERRORS("error"); const bookmark_type_t logfile_sub_source::BM_WARNINGS("warning"); const bookmark_type_t logfile_sub_source::BM_FILES("file"); @@ -63,7 +66,7 @@ pretty_sql_callback(exec_context& ec, sqlite3_stmt* stmt) return 0; } - int ncols = sqlite3_column_count(stmt); + const auto ncols = sqlite3_column_count(stmt); for (int lpc = 0; lpc < ncols; lpc++) { if (!ec.ec_accumulator->empty()) { @@ -78,6 +81,44 @@ pretty_sql_callback(exec_context& ec, sqlite3_stmt* stmt) ec.ec_accumulator->append(res); } + for (int lpc = 0; lpc < ncols; lpc++) { + const auto* colname = sqlite3_column_name(stmt, lpc); + auto* raw_value = sqlite3_column_value(stmt, lpc); + auto value_type = sqlite3_value_type(raw_value); + scoped_value_t value; + + switch (value_type) { + case SQLITE_INTEGER: + value = (int64_t) sqlite3_value_int64(raw_value); + break; + case SQLITE_FLOAT: + value = sqlite3_value_double(raw_value); + break; + case SQLITE_NULL: + value = null_value_t{}; + break; + default: + value = string_fragment::from_bytes( + sqlite3_value_text(raw_value), + sqlite3_value_bytes(raw_value)); + break; + } + if (!ec.ec_local_vars.empty() && !ec.ec_dry_run) { + if (sql_ident_needs_quote(colname)) { + continue; + } + auto& vars = ec.ec_local_vars.top(); + + if (vars.find(colname) != vars.end()) { + continue; + } + + if (value.is<string_fragment>()) { + value = value.get<string_fragment>().to_string(); + } + vars[colname] = value; + } + } return 0; } @@ -140,13 +181,46 @@ logfile_sub_source::find(const char* fn, content_line_t& line_base) return retval; } +struct filtered_logline_cmp { + filtered_logline_cmp(const logfile_sub_source& lc) : llss_controller(lc) {} + + bool operator()(const uint32_t& lhs, const uint32_t& rhs) const + { + content_line_t cl_lhs = (content_line_t) llss_controller.lss_index[lhs]; + content_line_t cl_rhs = (content_line_t) llss_controller.lss_index[rhs]; + logline* ll_lhs = this->llss_controller.find_line(cl_lhs); + logline* ll_rhs = this->llss_controller.find_line(cl_rhs); + + if (ll_lhs == nullptr) { + return true; + } + if (ll_rhs == nullptr) { + return false; + } + return (*ll_lhs) < (*ll_rhs); + } + + bool operator()(const uint32_t& lhs, const struct timeval& rhs) const + { + content_line_t cl_lhs = (content_line_t) llss_controller.lss_index[lhs]; + logline* ll_lhs = this->llss_controller.find_line(cl_lhs); + + if (ll_lhs == nullptr) { + return true; + } + return (*ll_lhs) < rhs; + } + + const logfile_sub_source& llss_controller; +}; + nonstd::optional<vis_line_t> logfile_sub_source::find_from_time(const struct timeval& start) const { - auto lb = lower_bound(this->lss_filtered_index.begin(), - this->lss_filtered_index.end(), - start, - filtered_logline_cmp(*this)); + auto lb = std::lower_bound(this->lss_filtered_index.begin(), + this->lss_filtered_index.end(), + start, + filtered_logline_cmp(*this)); if (lb != this->lss_filtered_index.end()) { return vis_line_t(lb - this->lss_filtered_index.begin()); } @@ -160,6 +234,11 @@ logfile_sub_source::text_value_for_line(textview_curses& tc, std::string& value_out, line_flags_t flags) { + if (this->lss_indexing_in_progress) { + value_out = ""; + return; + } + content_line_t line(0); require_ge(row, 0); @@ -224,10 +303,12 @@ logfile_sub_source::text_value_for_line(textview_curses& tc, exec_context ec( &this->lss_token_values, pretty_sql_callback, pretty_pipe_callback); std::string rewritten_line; + db_label_source rewrite_label_source; ec.with_perms(exec_context::perm_t::READ_ONLY); ec.ec_local_vars.push(std::map<std::string, scoped_value_t>()); ec.ec_top_line = vis_line_t(row); + ec.ec_label_source_stack.push_back(&rewrite_label_source); add_ansi_vars(ec.ec_global_vars); add_global_vars(ec); format->rewrite(ec, sbr, this->lss_token_attrs, rewritten_line); @@ -235,10 +316,19 @@ logfile_sub_source::text_value_for_line(textview_curses& tc, value_out = this->lss_token_value; } - if ((this->lss_token_file->is_time_adjusted() - || format->lf_timestamp_flags & ETF_MACHINE_ORIENTED - || !(format->lf_timestamp_flags & ETF_DAY_SET) - || !(format->lf_timestamp_flags & ETF_MONTH_SET)) + { + auto lr = line_range{0, (int) this->lss_token_value.length()}; + this->lss_token_attrs.emplace_back(lr, SA_ORIGINAL_LINE.value()); + } + + if (!this->lss_token_line->is_continued() && !format->lf_formatted_lines + && (this->lss_token_file->is_time_adjusted() + || ((format->lf_timestamp_flags & ETF_ZONE_SET + || format->lf_date_time.dts_default_zone != nullptr) + && format->lf_date_time.dts_zoned_to_local) + || format->lf_timestamp_flags & ETF_MACHINE_ORIENTED + || !(format->lf_timestamp_flags & ETF_DAY_SET) + || !(format->lf_timestamp_flags & ETF_MONTH_SET)) && format->lf_date_time.dts_fmt_lock != -1) { auto time_attr @@ -256,8 +346,10 @@ logfile_sub_source::text_value_for_line(textview_curses& tc, || !(format->lf_timestamp_flags & ETF_MONTH_SET)) { adjusted_time = this->lss_token_line->get_timeval(); - fmt = "%Y-%m-%d %H:%M:%S.%f"; - if (format->lf_timestamp_flags & ETF_MICROS_SET) { + if (format->lf_timestamp_flags + & (ETF_MICROS_SET | ETF_NANOS_SET)) + { + fmt = "%Y-%m-%d %H:%M:%S.%f"; struct timeval actual_tv; struct exttm tm; if (format->lf_date_time.scan( @@ -270,6 +362,10 @@ logfile_sub_source::text_value_for_line(textview_curses& tc, { adjusted_time.tv_usec = actual_tv.tv_usec; } + } else if (format->lf_timestamp_flags & ETF_MILLIS_SET) { + fmt = "%Y-%m-%d %H:%M:%S.%L"; + } else { + fmt = "%Y-%m-%d %H:%M:%S"; } gmtime_r(&adjusted_time.tv_sec, &adjusted_tm.et_tm); adjusted_tm.et_nsec @@ -284,6 +380,14 @@ logfile_sub_source::text_value_for_line(textview_curses& tc, = std::chrono::duration_cast<std::chrono::nanoseconds>( std::chrono::microseconds{adjusted_time.tv_usec}) .count(); + adjusted_tm.et_flags = format->lf_timestamp_flags; + if (format->lf_timestamp_flags & ETF_ZONE_SET + && format->lf_date_time.dts_zoned_to_local) + { + adjusted_tm.et_flags &= ~ETF_Z_IS_UTC; + } + adjusted_tm.et_gmtoff + = format->lf_date_time.dts_local_offset_cache; len = format->lf_date_time.ftime( buffer, sizeof(buffer), @@ -303,14 +407,14 @@ logfile_sub_source::text_value_for_line(textview_curses& tc, std::string name; if (this->lss_flags & F_FILENAME) { file_offset_end = this->lss_filename_width; - name = this->lss_token_file->get_filename(); + name = fmt::to_string(this->lss_token_file->get_filename()); if (file_offset_end < name.size()) { file_offset_end = name.size(); this->lss_filename_width = name.size(); } } else { file_offset_end = this->lss_basename_width; - name = this->lss_token_file->get_unique_path(); + name = fmt::to_string(this->lss_token_file->get_unique_path()); if (file_offset_end < name.size()) { file_offset_end = name.size(); this->lss_basename_width = name.size(); @@ -324,35 +428,12 @@ logfile_sub_source::text_value_for_line(textview_curses& tc, value_out.insert(0, 1, ' '); } - if (this->lss_flags & F_TIME_OFFSET) { - auto curr_tv = this->lss_token_line->get_timeval(); - struct timeval diff_tv; + if (this->tas_display_time_offset) { auto row_vl = vis_line_t(row); - - auto prev_umark - = tc.get_bookmarks()[&textview_curses::BM_USER].prev(row_vl); - auto next_umark - = tc.get_bookmarks()[&textview_curses::BM_USER].next(row_vl); - auto prev_emark - = tc.get_bookmarks()[&textview_curses::BM_USER_EXPR].prev(row_vl); - auto next_emark - = tc.get_bookmarks()[&textview_curses::BM_USER_EXPR].next(row_vl); - if (!prev_umark && !prev_emark && (next_umark || next_emark)) { - auto next_line = this->find_line(this->at( - std::max(next_umark.value_or(0), next_emark.value_or(0)))); - - diff_tv = curr_tv - next_line->get_timeval(); - } else { - auto prev_row - = std::max(prev_umark.value_or(0), prev_emark.value_or(0)); - auto first_line = this->find_line(this->at(prev_row)); - auto start_tv = first_line->get_timeval(); - diff_tv = curr_tv - start_tv; - } - - auto relstr = humanize::time::duration::from_tv(diff_tv).to_string(); + auto relstr = this->get_time_offset_for_line(tc, row_vl); value_out = fmt::format(FMT_STRING("{: >12}|{}"), relstr, value_out); } + this->lss_in_value_for_line = false; } @@ -361,6 +442,10 @@ logfile_sub_source::text_attrs_for_line(textview_curses& lv, int row, string_attrs_t& value_out) { + if (this->lss_indexing_in_progress) { + return; + } + view_colors& vc = view_colors::singleton(); logline* next_line = nullptr; struct line_range lr; @@ -383,8 +468,7 @@ logfile_sub_source::text_attrs_for_line(textview_curses& lv, const auto& line_values = this->lss_token_values; lr.lr_start = 0; - lr.lr_end = this->lss_token_value.length(); - value_out.emplace_back(lr, SA_ORIGINAL_LINE.value()); + lr.lr_end = -1; value_out.emplace_back( lr, SA_LEVEL.value(this->lss_token_line->get_msg_level())); @@ -459,8 +543,7 @@ logfile_sub_source::text_attrs_for_line(textview_curses& lv, value_out.emplace_back(lr, VC_GRAPHIC.value(graph)); if (!(this->lss_token_flags & RF_FULL)) { - bookmark_vector<vis_line_t>& bv_search - = bm[&textview_curses::BM_SEARCH]; + auto& bv_search = bm[&textview_curses::BM_SEARCH]; if (binary_search(std::begin(bv_search), std::end(bv_search), @@ -492,7 +575,7 @@ logfile_sub_source::text_attrs_for_line(textview_curses& lv, this->lss_token_file->get_filename()))); } - if (this->lss_flags & F_TIME_OFFSET) { + if (this->tas_display_time_offset) { time_offset_end = 13; lr.lr_start = 0; lr.lr_end = time_offset_end; @@ -526,20 +609,15 @@ logfile_sub_source::text_attrs_for_line(textview_curses& lv, lr, SA_FORMAT.value(this->lss_token_file->get_format()->get_name())); { - const auto& bv = lv.get_bookmarks()[&textview_curses::BM_META]; - bookmark_vector<vis_line_t>::const_iterator bv_iter; - - bv_iter = lower_bound(bv.begin(), bv.end(), vis_line_t(row + 1)); - if (bv_iter != bv.begin()) { - --bv_iter; - auto line_meta_opt = this->find_bookmark_metadata(*bv_iter); - - if (line_meta_opt && !line_meta_opt.value()->bm_name.empty()) { - lr.lr_start = 0; - lr.lr_end = -1; - value_out.emplace_back( - lr, logline::L_PARTITION.value(line_meta_opt.value())); - } + auto line_meta_context = this->get_bookmark_metadata_context( + vis_line_t(row + 1), bookmark_metadata::categories::partition); + if (line_meta_context.bmc_current_metadata) { + lr.lr_start = 0; + lr.lr_end = -1; + value_out.emplace_back( + lr, + logline::L_PARTITION.value( + line_meta_context.bmc_current_metadata.value())); } auto line_meta_opt = this->find_bookmark_metadata(vis_line_t(row)); @@ -620,6 +698,53 @@ logfile_sub_source::text_attrs_for_line(textview_curses& lv, } } +struct logline_cmp { + logline_cmp(logfile_sub_source& lc) : llss_controller(lc) {} + + bool operator()(const content_line_t& lhs, const content_line_t& rhs) const + { + logline* ll_lhs = this->llss_controller.find_line(lhs); + logline* ll_rhs = this->llss_controller.find_line(rhs); + + return (*ll_lhs) < (*ll_rhs); + } + + bool operator()(const uint32_t& lhs, const uint32_t& rhs) const + { + content_line_t cl_lhs = (content_line_t) llss_controller.lss_index[lhs]; + content_line_t cl_rhs = (content_line_t) llss_controller.lss_index[rhs]; + logline* ll_lhs = this->llss_controller.find_line(cl_lhs); + logline* ll_rhs = this->llss_controller.find_line(cl_rhs); + + return (*ll_lhs) < (*ll_rhs); + } +#if 0 + bool operator()(const indexed_content &lhs, const indexed_content &rhs) + { + logline *ll_lhs = this->llss_controller.find_line(lhs.ic_value); + logline *ll_rhs = this->llss_controller.find_line(rhs.ic_value); + + return (*ll_lhs) < (*ll_rhs); + } +#endif + + bool operator()(const content_line_t& lhs, const time_t& rhs) const + { + logline* ll_lhs = this->llss_controller.find_line(lhs); + + return *ll_lhs < rhs; + } + + bool operator()(const content_line_t& lhs, const struct timeval& rhs) const + { + logline* ll_lhs = this->llss_controller.find_line(lhs); + + return *ll_lhs < rhs; + } + + logfile_sub_source& llss_controller; +}; + logfile_sub_source::rebuild_result logfile_sub_source::rebuild_index( nonstd::optional<ui_clock::time_point> deadline) @@ -628,6 +753,9 @@ logfile_sub_source::rebuild_index( return rebuild_result::rr_no_change; } + this->lss_indexing_in_progress = true; + auto fin = finally([this]() { this->lss_indexing_in_progress = false; }); + iterator iter; size_t total_lines = 0; bool full_sort = false; @@ -641,6 +769,7 @@ logfile_sub_source::rebuild_index( if (force) { log_debug("forced to full rebuild"); retval = rebuild_result::rr_full_rebuild; + full_sort = true; } std::vector<size_t> file_order(this->lss_files.size()); @@ -678,6 +807,7 @@ logfile_sub_source::rebuild_index( ld.ld_file_index); force = true; retval = rebuild_result::rr_full_rebuild; + full_sort = true; } } else { if (time_left && deadline && ui_clock::now() > deadline.value()) { @@ -733,6 +863,11 @@ logfile_sub_source::rebuild_index( { lowest_tv = new_file_line.get_timeval(); } + } else { + log_debug( + "already doing full rebuild, doing " + "full_sort as well"); + full_sort = true; } } } @@ -757,8 +892,12 @@ logfile_sub_source::rebuild_index( } if (this->lss_index.reserve(total_lines)) { + // The index array was reallocated, just do a full sort/rebuild since + // it's been cleared out. + log_debug("expanding index capacity %zu", this->lss_index.ba_capacity); force = true; retval = rebuild_result::rr_full_rebuild; + full_sort = true; } auto& vis_bm = this->tss_view->get_bookmarks(); @@ -854,12 +993,13 @@ logfile_sub_source::rebuild_index( if (lf == nullptr) { continue; } - this->lss_longest_line = std::max(this->lss_longest_line, - lf->get_longest_line_length()); - this->lss_basename_width = std::max(this->lss_basename_width, - lf->get_unique_path().size()); - this->lss_filename_width - = std::max(this->lss_filename_width, lf->get_filename().size()); + this->lss_longest_line = std::max( + this->lss_longest_line, lf->get_longest_line_length() + 1); + this->lss_basename_width + = std::max(this->lss_basename_width, + lf->get_unique_path().native().size()); + this->lss_filename_width = std::max( + this->lss_filename_width, lf->get_filename().native().size()); } if (full_sort) { @@ -873,13 +1013,41 @@ logfile_sub_source::rebuild_index( for (size_t line_index = 0; line_index < lf->size(); line_index++) { - if ((*lf)[line_index].is_ignored()) { + const auto lf_iter + = ld->get_file_ptr()->begin() + line_index; + if (lf_iter->is_ignored()) { continue; } content_line_t con_line( ld->ld_file_index * MAX_LINES_PER_FILE + line_index); + if (lf_iter->is_meta_marked()) { + auto start_iter = lf_iter; + while (start_iter->is_continued()) { + --start_iter; + } + int start_index + = start_iter - ld->get_file_ptr()->begin(); + content_line_t start_con_line(ld->ld_file_index + * MAX_LINES_PER_FILE + + start_index); + + auto& line_meta + = ld->get_file_ptr() + ->get_bookmark_metadata()[start_index]; + if (line_meta.has(bookmark_metadata::categories::notes)) + { + this->lss_user_marks[&textview_curses::BM_META] + .insert_once(start_con_line); + } + if (line_meta.has( + bookmark_metadata::categories::partition)) + { + this->lss_user_marks[&textview_curses::BM_PARTITION] + .insert_once(start_con_line); + } + } this->lss_index.push_back(con_line); } } @@ -932,7 +1100,7 @@ logfile_sub_source::rebuild_index( content_line_t con_line(file_index * MAX_LINES_PER_FILE + line_index); - if (lf_iter->is_marked()) { + if (lf_iter->is_meta_marked()) { auto start_iter = lf_iter; while (start_iter->is_continued()) { --start_iter; @@ -942,9 +1110,20 @@ logfile_sub_source::rebuild_index( content_line_t start_con_line( file_index * MAX_LINES_PER_FILE + start_index); - this->lss_user_marks[&textview_curses::BM_META] - .insert_once(start_con_line); - lf_iter->set_mark(false); + auto& line_meta + = ld->get_file_ptr() + ->get_bookmark_metadata()[start_index]; + if (line_meta.has(bookmark_metadata::categories::notes)) + { + this->lss_user_marks[&textview_curses::BM_META] + .insert_once(start_con_line); + } + if (line_meta.has( + bookmark_metadata::categories::partition)) + { + this->lss_user_marks[&textview_curses::BM_PARTITION] + .insert_once(start_con_line); + } } this->lss_index.push_back(con_line); } @@ -985,7 +1164,7 @@ logfile_sub_source::rebuild_index( index_index < this->lss_index.size(); index_index++) { - content_line_t cl = (content_line_t) this->lss_index[index_index]; + const auto cl = (content_line_t) this->lss_index[index_index]; uint64_t line_number; auto ld = this->find_data(cl, line_number); @@ -1028,6 +1207,8 @@ logfile_sub_source::rebuild_index( } } + this->lss_indexing_in_progress = false; + if (this->lss_index_delegate != nullptr) { this->lss_index_delegate->index_complete(*this); } @@ -1060,7 +1241,7 @@ logfile_sub_source::rebuild_index( void logfile_sub_source::text_update_marks(vis_bookmarks& bm) { - std::shared_ptr<logfile> last_file; + logfile* last_file = nullptr; vis_line_t vl; bm[&BM_WARNINGS].clear(); @@ -1074,7 +1255,7 @@ logfile_sub_source::text_update_marks(vis_bookmarks& bm) for (; vl < (int) this->lss_filtered_index.size(); ++vl) { const content_line_t orig_cl = this->at(vl); content_line_t cl = orig_cl; - auto lf = this->find(cl); + auto lf = this->find_file_ptr(cl); for (auto& lss_user_mark : this->lss_user_marks) { if (binary_search(lss_user_mark.second.begin(), @@ -1117,29 +1298,6 @@ logfile_sub_source::text_update_marks(vis_bookmarks& bm) } } -log_accel::direction_t -logfile_sub_source::get_line_accel_direction(vis_line_t vl) -{ - log_accel la; - - while (vl >= 0) { - logline* curr_line = this->find_line(this->at(vl)); - - if (!curr_line->is_message()) { - --vl; - continue; - } - - if (!la.add_point(curr_line->get_time_in_millis())) { - break; - } - - --vl; - } - - return la.get_direction(); -} - void logfile_sub_source::text_filters_changed() { @@ -1226,6 +1384,68 @@ bool logfile_sub_source::list_input_handle_key(listview_curses& lv, int ch) { switch (ch) { + case ' ': { + auto ov_vl = lv.get_overlay_selection(); + if (ov_vl) { + auto* fos = dynamic_cast<field_overlay_source*>( + lv.get_overlay_source()); + auto iter = fos->fos_row_to_field_meta.find(ov_vl.value()); + if (iter != fos->fos_row_to_field_meta.end()) { + auto find_res = this->find_line_with_file(lv.get_top()); + if (find_res) { + auto file_and_line = find_res.value(); + auto* format = file_and_line.first->get_format_ptr(); + auto fstates = format->get_field_states(); + auto state_iter = fstates.find(iter->second.lvm_name); + if (state_iter != fstates.end()) { + format->hide_field(iter->second.lvm_name, + !state_iter->second.is_hidden()); + lv.set_needs_update(); + } + } + } + return true; + } + return false; + } + case '#': { + auto ov_vl = lv.get_overlay_selection(); + if (ov_vl) { + auto* fos = dynamic_cast<field_overlay_source*>( + lv.get_overlay_source()); + auto iter = fos->fos_row_to_field_meta.find(ov_vl.value()); + if (iter != fos->fos_row_to_field_meta.end()) { + const auto& meta = iter->second; + std::string cmd; + + switch (meta.to_chart_type()) { + case chart_type_t::none: + break; + case chart_type_t::hist: { + auto prql = fmt::format( + FMT_STRING( + "from {} | stats.hist {} slice:'1h'"), + meta.lvm_format.value()->get_name(), + meta.lvm_name); + cmd = fmt::format(FMT_STRING(":prompt sql ; '{}'"), + shlex::escape(prql)); + break; + } + case chart_type_t::spectro: + cmd = fmt::format(FMT_STRING(":spectrogram {}"), + meta.lvm_name); + break; + } + if (!cmd.empty()) { + this->lss_exec_context + ->with_provenance(exec_context::mouse_input{}) + ->execute(cmd); + } + } + return true; + } + return false; + } case 'h': case 'H': case KEY_SLEFT: @@ -1258,6 +1478,24 @@ logfile_sub_source::get_grepper() (grep_proc_sink<vis_line_t>*) &this->lss_meta_grepper); } +/** + * Functor for comparing the ld_file field of the logfile_data struct. + */ +struct logfile_data_eq { + explicit logfile_data_eq(std::shared_ptr<logfile> lf) + : lde_file(std::move(lf)) + { + } + + bool operator()( + const std::unique_ptr<logfile_sub_source::logfile_data>& ld) const + { + return this->lde_file == ld->get_file(); + } + + std::shared_ptr<logfile> lde_file; +}; + bool logfile_sub_source::insert_file(const std::shared_ptr<logfile>& lf) { @@ -1483,6 +1721,26 @@ logfile_sub_source::eval_sql_filter(sqlite3_stmt* stmt, } continue; } + if (strcmp(name, ":log_annotations") == 0) { + const auto& bm = lf->get_bookmark_metadata(); + auto line_number + = static_cast<uint32_t>(std::distance(lf->cbegin(), ll)); + auto bm_iter = bm.find(line_number); + if (bm_iter != bm.end() + && !bm_iter->second.bm_annotations.la_pairs.empty()) + { + const auto& meta = bm_iter->second; + auto anno_str = logmsg_annotations_handlers.to_string( + meta.bm_annotations); + + sqlite3_bind_text(stmt, + lpc + 1, + anno_str.c_str(), + anno_str.length(), + SQLITE_TRANSIENT); + } + continue; + } if (strcmp(name, ":log_tags") == 0) { const auto& bm = lf->get_bookmark_metadata(); auto line_number @@ -1529,7 +1787,7 @@ logfile_sub_source::eval_sql_filter(sqlite3_stmt* stmt, sqlite3_bind_text(stmt, lpc + 1, filename.c_str(), - filename.length(), + filename.native().length(), SQLITE_STATIC); continue; } @@ -1538,7 +1796,7 @@ logfile_sub_source::eval_sql_filter(sqlite3_stmt* stmt, sqlite3_bind_text(stmt, lpc + 1, filename.c_str(), - filename.length(), + filename.native().length(), SQLITE_STATIC); continue; } @@ -1565,14 +1823,11 @@ logfile_sub_source::eval_sql_filter(sqlite3_stmt* stmt, } if (strcmp(name, ":log_opid") == 0) { auto opid_attr_opt = get_string_attr(sa, logline::L_OPID); - if (opid_attr_opt) { - const auto& sar - = opid_attr_opt.value().saw_string_attr->sa_range; - + if (values.lvv_opid_value) { sqlite3_bind_text(stmt, lpc + 1, - sbr.get_data_at(sar.lr_start), - sar.length(), + values.lvv_opid_value->c_str(), + values.lvv_opid_value->length(), SQLITE_STATIC); } else { sqlite3_bind_null(stmt, lpc + 1); @@ -1714,11 +1969,6 @@ logfile_sub_source::text_clear_marks(const bookmark_type_t* bm) for (iter = this->lss_user_marks[bm].begin(); iter != this->lss_user_marks[bm].end();) { - auto line_meta_opt = this->find_bookmark_metadata(*iter); - if (line_meta_opt) { - ++iter; - continue; - } this->find_line(*iter)->set_mark(false); iter = this->lss_user_marks[bm].erase(iter); } @@ -1891,25 +2141,30 @@ log_location_history::loc_history_forward(vis_line_t current_top) } bool -sql_filter::matches(const logfile& lf, - logfile::const_iterator ll, +sql_filter::matches(nonstd::optional<line_source> ls_opt, const shared_buffer_ref& line) { - if (!ll->is_message()) { + if (!ls_opt) { + return false; + } + + auto ls = ls_opt; + + if (!ls->ls_line->is_message()) { return false; } if (this->sf_filter_stmt == nullptr) { return false; } - auto lfp = lf.shared_from_this(); + auto lfp = ls->ls_file.shared_from_this(); auto ld = this->sf_log_source.find_data_i(lfp); if (ld == this->sf_log_source.end()) { return false; } - auto eval_res - = this->sf_log_source.eval_sql_filter(this->sf_filter_stmt, ld, ll); + auto eval_res = this->sf_log_source.eval_sql_filter( + this->sf_filter_stmt, ld, ls->ls_line); if (eval_res.unwrapOr(true)) { return false; } @@ -1949,6 +2204,21 @@ logfile_sub_source::meta_grepper::grep_value_for_line(vis_line_t line, value_out.append(tag); value_out.append("\x1c"); } + value_out.append("\x1c"); + for (const auto& pair : bm.bm_annotations.la_pairs) { + value_out.append(pair.first); + value_out.append("\x1c"); + + md2attr_line mdal; + + auto parse_res = md4cpp::parse(pair.second, mdal); + if (parse_res.isOk()) { + value_out.append(parse_res.unwrap().get_string()); + } else { + value_out.append(pair.second); + } + value_out.append("\x1c"); + } } return !this->lmg_done; @@ -1958,8 +2228,8 @@ vis_line_t logfile_sub_source::meta_grepper::grep_initial_line(vis_line_t start, vis_line_t highest) { - vis_bookmarks& bm = this->lmg_source.tss_view->get_bookmarks(); - bookmark_vector<vis_line_t>& bv = bm[&textview_curses::BM_META]; + auto& bm = this->lmg_source.tss_view->get_bookmarks(); + auto& bv = bm[&textview_curses::BM_META]; if (bv.empty()) { return -1_vl; @@ -1970,8 +2240,8 @@ logfile_sub_source::meta_grepper::grep_initial_line(vis_line_t start, void logfile_sub_source::meta_grepper::grep_next_line(vis_line_t& line) { - vis_bookmarks& bm = this->lmg_source.tss_view->get_bookmarks(); - bookmark_vector<vis_line_t>& bv = bm[&textview_curses::BM_META]; + auto& bm = this->lmg_source.tss_view->get_bookmarks(); + auto& bv = bm[&textview_curses::BM_META]; auto line_opt = bv.next(vis_line_t(line)); if (!line_opt) { @@ -2132,6 +2402,21 @@ timestamp_poss() return retval; } +static attr_line_t +to_display(const std::shared_ptr<logfile>& lf) +{ + attr_line_t retval; + + if (lf->get_open_options().loo_piper) { + if (!lf->get_open_options().loo_piper->is_finished()) { + retval.append("\u21bb "_list_glyph); + } + } + retval.append(lf->get_unique_path()); + + return retval; +} + void logfile_sub_source::text_crumbs_for_line(int line, std::vector<breadcrumb::crumb>& crumbs) @@ -2142,7 +2427,43 @@ logfile_sub_source::text_crumbs_for_line(int line, return; } - auto line_pair_opt = this->find_line_with_file(vis_line_t(line)); + auto vl = vis_line_t(line); + auto bmc = this->get_bookmark_metadata_context( + vl, bookmark_metadata::categories::partition); + if (bmc.bmc_current_metadata) { + const auto& name = bmc.bmc_current_metadata.value()->bm_name; + auto key = text_anchors::to_anchor_string(name); + auto display = attr_line_t() + .append("\u2291 "_symbol) + .append(lnav::roles::variable(name)); + crumbs.emplace_back( + key, + display, + [this]() -> std::vector<breadcrumb::possibility> { + auto& vb = this->tss_view->get_bookmarks(); + const auto& bv = vb[&textview_curses::BM_PARTITION]; + std::vector<breadcrumb::possibility> retval; + + for (const auto& vl : bv) { + auto meta_opt = this->find_bookmark_metadata(vl); + if (!meta_opt || meta_opt.value()->bm_name.empty()) { + continue; + } + + const auto& name = meta_opt.value()->bm_name; + retval.emplace_back(text_anchors::to_anchor_string(name), + name); + } + + return retval; + }, + [ec = this->lss_exec_context](const auto& part) { + ec->execute(fmt::format(FMT_STRING(":goto {}"), + part.template get<std::string>())); + }); + } + + auto line_pair_opt = this->find_line_with_file(vl); if (!line_pair_opt) { return; } @@ -2180,7 +2501,8 @@ logfile_sub_source::text_crumbs_for_line(int line, return breadcrumb::possibility{ elem.to_string(), }; - }); + }) + | lnav::itertools::to_vector(); }, [ec = this->lss_exec_context](const auto& format_name) { static const std::string MOVE_STMT = R"(;UPDATE lnav_views @@ -2198,9 +2520,7 @@ logfile_sub_source::text_crumbs_for_line(int line, auto file_line_number = std::distance(lf->begin(), msg_start_iter); crumbs.emplace_back( lf->get_unique_path(), - attr_line_t() - .append(lf->get_unique_path()) - .appendf(FMT_STRING("[{:L}]"), file_line_number), + to_display(lf).appendf(FMT_STRING("[{:L}]"), file_line_number), [this]() -> std::vector<breadcrumb::possibility> { return this->lss_files | lnav::itertools::filter_in([](const auto& file_data) { @@ -2209,8 +2529,7 @@ logfile_sub_source::text_crumbs_for_line(int line, | lnav::itertools::map([](const auto& file_data) { return breadcrumb::possibility{ file_data->get_file_ptr()->get_unique_path(), - attr_line_t( - file_data->get_file_ptr()->get_unique_path()), + to_display(file_data->get_file()), }; }); }, @@ -2254,10 +2573,10 @@ logfile_sub_source::text_crumbs_for_line(int line, if (file_data->get_file_ptr() == nullptr) { continue; } - safe::ReadAccess<logfile::safe_opid_map> r_opid_map( + safe::ReadAccess<logfile::safe_opid_state> r_opid_map( file_data->get_file_ptr()->get_opids()); - for (const auto& pair : *r_opid_map) { + for (const auto& pair : r_opid_map->los_opid_ranges) { retval.emplace_back(pair.first.to_string()); } } @@ -2424,8 +2743,62 @@ logfile_sub_source::get_bookmark_metadata(content_line_t cl) return line_pair.first->get_bookmark_metadata()[line_number]; } +logfile_sub_source::bookmark_metadata_context +logfile_sub_source::get_bookmark_metadata_context( + vis_line_t vl, bookmark_metadata::categories desired) const +{ + const auto& vb = this->tss_view->get_bookmarks(); + const auto bv_iter + = vb.find(desired == bookmark_metadata::categories::partition + ? &textview_curses::BM_PARTITION + : &textview_curses::BM_META); + if (bv_iter == vb.end()) { + return bookmark_metadata_context{}; + } + + const auto& bv = bv_iter->second; + auto vl_iter = std::lower_bound(bv.begin(), bv.end(), vl + 1_vl); + + nonstd::optional<vis_line_t> next_line; + for (auto next_vl_iter = vl_iter; next_vl_iter != bv.end(); ++next_vl_iter) + { + auto bm_opt = this->find_bookmark_metadata(*next_vl_iter); + if (!bm_opt) { + continue; + } + + if (bm_opt.value()->has(desired)) { + next_line = *next_vl_iter; + break; + } + } + if (vl_iter == bv.begin()) { + return bookmark_metadata_context{ + nonstd::nullopt, nonstd::nullopt, next_line}; + } + + --vl_iter; + while (true) { + auto bm_opt = this->find_bookmark_metadata(*vl_iter); + if (bm_opt) { + if (bm_opt.value()->has(desired)) { + return bookmark_metadata_context{ + *vl_iter, bm_opt.value(), next_line}; + } + } + + if (vl_iter == bv.begin()) { + return bookmark_metadata_context{ + nonstd::nullopt, nonstd::nullopt, next_line}; + } + --vl_iter; + } + return bookmark_metadata_context{ + nonstd::nullopt, nonstd::nullopt, next_line}; +} + nonstd::optional<bookmark_metadata*> -logfile_sub_source::find_bookmark_metadata(content_line_t cl) +logfile_sub_source::find_bookmark_metadata(content_line_t cl) const { auto line_pair = this->find_line_with_file(cl).value(); auto line_number = static_cast<uint32_t>( @@ -2465,3 +2838,240 @@ logfile_sub_source::clear_bookmark_metadata() ld->get_file_ptr()->get_bookmark_metadata().clear(); } } + +void +logfile_sub_source::increase_line_context() +{ + auto old_flags = this->lss_flags; + + if (this->lss_flags & F_FILENAME) { + // Nothing to do + } else if (this->lss_flags & F_BASENAME) { + this->lss_flags &= ~F_NAME_MASK; + this->lss_flags |= F_FILENAME; + } else { + this->lss_flags |= F_BASENAME; + } + if (old_flags != this->lss_flags) { + this->clear_line_size_cache(); + } +} + +bool +logfile_sub_source::decrease_line_context() +{ + auto old_flags = this->lss_flags; + + if (this->lss_flags & F_FILENAME) { + this->lss_flags &= ~F_NAME_MASK; + this->lss_flags |= F_BASENAME; + } else if (this->lss_flags & F_BASENAME) { + this->lss_flags &= ~F_NAME_MASK; + } + if (old_flags != this->lss_flags) { + this->clear_line_size_cache(); + + return true; + } + + return false; +} + +size_t +logfile_sub_source::get_filename_offset() const +{ + if (this->lss_flags & F_FILENAME) { + return this->lss_filename_width; + } else if (this->lss_flags & F_BASENAME) { + return this->lss_basename_width; + } + + return 0; +} + +void +logfile_sub_source::clear_min_max_log_times() +{ + if (this->lss_min_log_time.tv_sec != 0 + || this->lss_min_log_time.tv_usec != 0 + || this->lss_max_log_time.tv_sec != std::numeric_limits<time_t>::max() + || this->lss_max_log_time.tv_usec != 0) + { + memset(&this->lss_min_log_time, 0, sizeof(this->lss_min_log_time)); + this->lss_max_log_time.tv_sec = std::numeric_limits<time_t>::max(); + this->lss_max_log_time.tv_usec = 0; + this->text_filters_changed(); + } +} + +size_t +logfile_sub_source::file_count() const +{ + size_t retval = 0; + const_iterator iter; + + for (iter = this->cbegin(); iter != this->cend(); ++iter) { + if (*iter != nullptr && (*iter)->get_file() != nullptr) { + retval += 1; + } + } + + return retval; +} + +size_t +logfile_sub_source::text_size_for_line(textview_curses& tc, + int row, + text_sub_source::line_flags_t flags) +{ + size_t index = row % LINE_SIZE_CACHE_SIZE; + + if (this->lss_line_size_cache[index].first != row) { + std::string value; + + this->text_value_for_line(tc, row, value, flags); + scrub_ansi_string(value, nullptr); + this->lss_line_size_cache[index].second + = string_fragment::from_str(value).column_width(); + this->lss_line_size_cache[index].first = row; + } + return this->lss_line_size_cache[index].second; +} + +int +logfile_sub_source::get_filtered_count_for(size_t filter_index) const +{ + int retval = 0; + + for (const auto& ld : this->lss_files) { + retval += ld->ld_filter_state.lfo_filter_state + .tfs_filter_hits[filter_index]; + } + + return retval; +} + +nonstd::optional<vis_line_t> +logfile_sub_source::row_for(const row_info& ri) +{ + auto lb = std::lower_bound(this->lss_filtered_index.begin(), + this->lss_filtered_index.end(), + ri.ri_time, + filtered_logline_cmp(*this)); + if (lb != this->lss_filtered_index.end()) { + auto first_lb = lb; + while (true) { + auto cl = this->lss_index[*lb]; + if (content_line_t(ri.ri_id) == cl) { + first_lb = lb; + break; + } + auto ll = this->find_line(cl); + if (ll->get_timeval() != ri.ri_time) { + break; + } + ++lb; + } + + return vis_line_t(first_lb - this->lss_filtered_index.begin()); + } + + return nonstd::nullopt; +} + +nonstd::optional<vis_line_t> +logfile_sub_source::row_for_anchor(const std::string& id) +{ + auto& vb = this->tss_view->get_bookmarks(); + const auto& bv = vb[&textview_curses::BM_PARTITION]; + + for (const auto& vl : bv) { + auto meta_opt = this->find_bookmark_metadata(vl); + if (!meta_opt || meta_opt.value()->bm_name.empty()) { + continue; + } + + const auto& name = meta_opt.value()->bm_name; + if (id == text_anchors::to_anchor_string(name)) { + return vl; + } + } + + return nonstd::nullopt; +} + +nonstd::optional<vis_line_t> +logfile_sub_source::adjacent_anchor(vis_line_t vl, text_anchors::direction dir) +{ + auto bmc = this->get_bookmark_metadata_context( + vl, bookmark_metadata::categories::partition); + switch (dir) { + case text_anchors::direction::prev: { + if (bmc.bmc_current && bmc.bmc_current.value() != vl) { + return bmc.bmc_current; + } + if (!bmc.bmc_current || bmc.bmc_current.value() == 0_vl) { + return 0_vl; + } + auto prev_bmc = this->get_bookmark_metadata_context( + bmc.bmc_current.value() - 1_vl, + bookmark_metadata::categories::partition); + if (!prev_bmc.bmc_current) { + return 0_vl; + } + return prev_bmc.bmc_current; + } + case text_anchors::direction::next: + return bmc.bmc_next_line; + } + return nonstd::nullopt; +} + +nonstd::optional<std::string> +logfile_sub_source::anchor_for_row(vis_line_t vl) +{ + auto line_meta = this->get_bookmark_metadata_context( + vl, bookmark_metadata::categories::partition); + if (!line_meta.bmc_current_metadata) { + return nonstd::nullopt; + } + + return text_anchors::to_anchor_string( + line_meta.bmc_current_metadata.value()->bm_name); +} + +std::unordered_set<std::string> +logfile_sub_source::get_anchors() +{ + auto& vb = this->tss_view->get_bookmarks(); + const auto& bv = vb[&textview_curses::BM_PARTITION]; + std::unordered_set<std::string> retval; + + for (const auto& vl : bv) { + auto meta_opt = this->find_bookmark_metadata(vl); + if (!meta_opt || meta_opt.value()->bm_name.empty()) { + continue; + } + + const auto& name = meta_opt.value()->bm_name; + retval.emplace(text_anchors::to_anchor_string(name)); + } + + return retval; +} + +bool +logfile_sub_source::text_handle_mouse( + textview_curses& tc, + const listview_curses::display_line_content_t& mouse_line, + mouse_event& me) +{ + if (tc.get_overlay_selection()) { + if (me.is_click_in(mouse_button_t::BUTTON_LEFT, 2, 4)) { + this->list_input_handle_key(tc, ' '); + } else if (me.is_click_in(mouse_button_t::BUTTON_LEFT, 5, 6)) { + this->list_input_handle_key(tc, '#'); + } + } + return true; +} |