diff options
Diffstat (limited to '')
| -rw-r--r-- | src/line_buffer.cc | 113 |
1 files changed, 98 insertions, 15 deletions
diff --git a/src/line_buffer.cc b/src/line_buffer.cc index f370c02..a510169 100644 --- a/src/line_buffer.cc +++ b/src/line_buffer.cc @@ -57,8 +57,10 @@ #include "base/math_util.hh" #include "base/paths.hh" #include "fmtlib/fmt/format.h" +#include "hasher.hh" #include "line_buffer.hh" -#include "lnav_util.hh" +#include "piper.looper.hh" +#include "scn/scn.h" using namespace std::chrono_literals; @@ -135,7 +137,7 @@ private: #define SYNCPOINT_SIZE (1024 * 1024) line_buffer::gz_indexed::gz_indexed() { - if ((this->inbuf = (Bytef*) malloc(Z_BUFSIZE)) == NULL) { + if ((this->inbuf = auto_mem<Bytef>::malloc(Z_BUFSIZE)) == NULL) { throw std::bad_alloc(); } } @@ -191,7 +193,7 @@ line_buffer::gz_indexed::continue_stream() } void -line_buffer::gz_indexed::open(int fd, header_data& hd) +line_buffer::gz_indexed::open(int fd, lnav::gzip::header& hd) { this->close(); this->init_stream(); @@ -238,9 +240,9 @@ line_buffer::gz_indexed::open(int fd, header_data& hd) log_debug("%d: no gzip header data", fd); break; case 1: - hd.hd_mtime.tv_sec = gz_hd.time; - hd.hd_name = std::string((char*) name); - hd.hd_comment = std::string((char*) comment); + hd.h_mtime.tv_sec = gz_hd.time; + hd.h_name = std::string((char*) name); + hd.h_comment = std::string((char*) comment); break; default: log_error("%d: failed to read gzip header data", fd); @@ -408,7 +410,33 @@ line_buffer::set_fd(auto_fd& fd) char gz_id[2 + 1 + 1 + 4]; if (pread(fd, gz_id, sizeof(gz_id), 0) == sizeof(gz_id)) { - if (gz_id[0] == '\037' && gz_id[1] == '\213') { + auto piper_hdr_opt = lnav::piper::read_header(fd, gz_id); + + if (piper_hdr_opt) { + static intern_string_t SRC = intern_string::lookup("piper"); + + auto meta_buf = std::move(piper_hdr_opt.value()); + + auto meta_sf = string_fragment::from_bytes(meta_buf.in(), + meta_buf.size()); + auto meta_parse_res + = lnav::piper::header_handlers.parser_for(SRC).of( + meta_sf); + if (meta_parse_res.isErr()) { + log_error("failed to parse piper header: %s", + meta_parse_res.unwrapErr()[0] + .to_attr_line() + .get_string() + .c_str()); + throw error(EINVAL); + } + + this->lb_line_metadata = true; + this->lb_file_offset + = lnav::piper::HEADER_SIZE + meta_buf.size(); + this->lb_piper_header_size = this->lb_file_offset; + this->lb_header = meta_parse_res.unwrap(); + } else if (gz_id[0] == '\037' && gz_id[1] == '\213') { int gzfd = dup(fd); log_perror(fcntl(gzfd, F_SETFD, FD_CLOEXEC)); @@ -416,14 +444,19 @@ line_buffer::set_fd(auto_fd& fd) close(gzfd); throw error(errno); } - this->lb_gz_file.writeAccess()->open(gzfd, this->lb_header); + lnav::gzip::header hdr; + + this->lb_gz_file.writeAccess()->open(gzfd, hdr); this->lb_compressed = true; - this->lb_file_time = this->lb_header.hd_mtime.tv_sec; + this->lb_file_time = hdr.h_mtime.tv_sec; if (this->lb_file_time < 0) { this->lb_file_time = 0; } this->lb_compressed_offset = lseek(this->lb_fd, 0, SEEK_CUR); + if (!hdr.empty()) { + this->lb_header = std::move(hdr); + } this->resize_buffer(INITIAL_COMPRESSED_BUFFER_SIZE); } #ifdef HAVE_BZLIB_H @@ -763,7 +796,9 @@ line_buffer::fill_range(file_off_t start, ssize_t max_length) this->lb_alt_line_has_ansi.clear(); this->lb_stats.s_used_preloads += 1; } - if (this->in_range(start) && this->in_range(start + max_length - 1)) { + if (this->in_range(start) + && (max_length == 0 || this->in_range(start + max_length - 1))) + { /* Cache already has the data, nothing to do. */ retval = true; if (!lnav::pid::in_child && this->lb_seekable && this->lb_buffer.full() @@ -1022,11 +1057,16 @@ line_buffer::fill_range(file_off_t start, ssize_t max_length) Result<line_info, std::string> line_buffer::load_next_line(file_range prev_line) { + const char* line_start = nullptr; bool done = false; line_info retval; require(this->lb_fd != -1); + if (this->lb_line_metadata && prev_line.fr_offset == 0) { + prev_line.fr_offset = this->lb_piper_header_size; + } + auto offset = prev_line.next_offset(); ssize_t request_size = INITIAL_REQUEST_SIZE; retval.li_file_range.fr_offset = offset; @@ -1044,9 +1084,17 @@ line_buffer::load_next_line(file_range prev_line) return Ok(retval); } } + if (prev_line.next_offset() == 0) { + auto is_utf_res = is_utf8(string_fragment::from_bytes( + this->lb_buffer.begin(), this->lb_buffer.size())); + this->lb_is_utf8 = is_utf_res.is_valid(); + if (!this->lb_is_utf8) { + log_warning("input is not utf8 -- %s", is_utf_res.usr_message); + } + } while (!done) { auto old_retval_size = retval.li_file_range.fr_size; - const char *line_start, *lf = nullptr; + const char* lf = nullptr; /* Find the data in the cache and */ line_start = this->get_range(offset, retval.li_file_range.fr_size); @@ -1177,16 +1225,40 @@ line_buffer::load_next_line(file_range prev_line) = retval.li_utf8_scan_result.usr_has_ansi; retval.li_file_range.fr_metadata.m_valid_utf = retval.li_utf8_scan_result.is_valid(); + + if (this->lb_line_metadata) { + auto sv = scn::string_view{ + line_start, + (size_t) retval.li_file_range.fr_size, + }; + + char level; + auto scan_res = scn::scan(sv, + "{}.{}:{};", + retval.li_timestamp.tv_sec, + retval.li_timestamp.tv_usec, + level); + if (scan_res) { + retval.li_timestamp.tv_sec + = lnav::to_local_time(date::sys_seconds{std::chrono::seconds{ + retval.li_timestamp.tv_sec}}) + .time_since_epoch() + .count(); + retval.li_level = abbrev2level(&level, 1); + } + } + return Ok(retval); } Result<shared_buffer_ref, std::string> -line_buffer::read_range(const file_range fr) +line_buffer::read_range(file_range fr) { shared_buffer_ref retval; const char* line_start; file_ssize_t avail; +#if 0 if (this->lb_last_line_offset != -1 && fr.fr_offset > this->lb_last_line_offset) { @@ -1200,6 +1272,7 @@ line_buffer::read_range(const file_range fr) fr.fr_offset, this->lb_last_line_offset)); } +#endif if (!(this->in_range(fr.fr_offset) && this->in_range(fr.fr_offset + fr.fr_size - 1))) @@ -1214,6 +1287,15 @@ line_buffer::read_range(const file_range fr) return Err(fmt::format( FMT_STRING("short-read (need: {}; avail: {})"), fr.fr_size, avail)); } + if (this->lb_line_metadata) { + auto new_start + = static_cast<const char*>(memchr(line_start, ';', fr.fr_size)); + if (new_start) { + auto offset = new_start - line_start + 1; + line_start += offset; + fr.fr_size -= offset; + } + } retval.share(this->lb_share_manager, line_start, fr.fr_size); retval.get_metadata() = fr.fr_metadata; @@ -1388,12 +1470,13 @@ line_buffer::cleanup_cache() auto now = std::chrono::system_clock::now(); auto cache_path = line_buffer_cache_path(); std::vector<ghc::filesystem::path> to_remove; + std::error_code ec; for (const auto& cache_subdir : - ghc::filesystem::directory_iterator(cache_path)) + ghc::filesystem::directory_iterator(cache_path, ec)) { for (const auto& entry : - ghc::filesystem::directory_iterator(cache_subdir)) + ghc::filesystem::directory_iterator(cache_subdir, ec)) { auto mtime = ghc::filesystem::last_write_time(entry.path()); auto exp_time = mtime + 1h; @@ -1407,7 +1490,7 @@ line_buffer::cleanup_cache() for (auto& entry : to_remove) { log_debug("removing compressed file cache: %s", entry.c_str()); - ghc::filesystem::remove_all(entry); + ghc::filesystem::remove_all(entry, ec); } }); } |