Merging upstream version 0.12.2.

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

1 files changed, 98 insertions, 15 deletions

diff --git a/src/line_buffer.cc b/src/line_buffer.cc
index f370c02..a510169 100644
--- a/src/line_buffer.cc
+++ b/src/line_buffer.cc
@@ -57,8 +57,10 @@
 #include "base/math_util.hh"
 #include "base/paths.hh"
 #include "fmtlib/fmt/format.h"
+#include "hasher.hh"
 #include "line_buffer.hh"
-#include "lnav_util.hh"
+#include "piper.looper.hh"
+#include "scn/scn.h"
 
 using namespace std::chrono_literals;
 
@@ -135,7 +137,7 @@ private:
 #define SYNCPOINT_SIZE (1024 * 1024)
 line_buffer::gz_indexed::gz_indexed()
 {
-    if ((this->inbuf = (Bytef*) malloc(Z_BUFSIZE)) == NULL) {
+    if ((this->inbuf = auto_mem<Bytef>::malloc(Z_BUFSIZE)) == NULL) {
         throw std::bad_alloc();
     }
 }
@@ -191,7 +193,7 @@ line_buffer::gz_indexed::continue_stream()
 }
 
 void
-line_buffer::gz_indexed::open(int fd, header_data& hd)
+line_buffer::gz_indexed::open(int fd, lnav::gzip::header& hd)
 {
     this->close();
     this->init_stream();
@@ -238,9 +240,9 @@ line_buffer::gz_indexed::open(int fd, header_data& hd)
                     log_debug("%d: no gzip header data", fd);
                     break;
                 case 1:
-                    hd.hd_mtime.tv_sec = gz_hd.time;
-                    hd.hd_name = std::string((char*) name);
-                    hd.hd_comment = std::string((char*) comment);
+                    hd.h_mtime.tv_sec = gz_hd.time;
+                    hd.h_name = std::string((char*) name);
+                    hd.h_comment = std::string((char*) comment);
                     break;
                 default:
                     log_error("%d: failed to read gzip header data", fd);
@@ -408,7 +410,33 @@ line_buffer::set_fd(auto_fd& fd)
             char gz_id[2 + 1 + 1 + 4];
 
             if (pread(fd, gz_id, sizeof(gz_id), 0) == sizeof(gz_id)) {
-                if (gz_id[0] == '\037' && gz_id[1] == '\213') {
+                auto piper_hdr_opt = lnav::piper::read_header(fd, gz_id);
+
+                if (piper_hdr_opt) {
+                    static intern_string_t SRC = intern_string::lookup("piper");
+
+                    auto meta_buf = std::move(piper_hdr_opt.value());
+
+                    auto meta_sf = string_fragment::from_bytes(meta_buf.in(),
+                                                               meta_buf.size());
+                    auto meta_parse_res
+                        = lnav::piper::header_handlers.parser_for(SRC).of(
+                            meta_sf);
+                    if (meta_parse_res.isErr()) {
+                        log_error("failed to parse piper header: %s",
+                                  meta_parse_res.unwrapErr()[0]
+                                      .to_attr_line()
+                                      .get_string()
+                                      .c_str());
+                        throw error(EINVAL);
+                    }
+
+                    this->lb_line_metadata = true;
+                    this->lb_file_offset
+                        = lnav::piper::HEADER_SIZE + meta_buf.size();
+                    this->lb_piper_header_size = this->lb_file_offset;
+                    this->lb_header = meta_parse_res.unwrap();
+                } else if (gz_id[0] == '\037' && gz_id[1] == '\213') {
                     int gzfd = dup(fd);
 
                     log_perror(fcntl(gzfd, F_SETFD, FD_CLOEXEC));
@@ -416,14 +444,19 @@ line_buffer::set_fd(auto_fd& fd)
                         close(gzfd);
                         throw error(errno);
                     }
-                    this->lb_gz_file.writeAccess()->open(gzfd, this->lb_header);
+                    lnav::gzip::header hdr;
+
+                    this->lb_gz_file.writeAccess()->open(gzfd, hdr);
                     this->lb_compressed = true;
-                    this->lb_file_time = this->lb_header.hd_mtime.tv_sec;
+                    this->lb_file_time = hdr.h_mtime.tv_sec;
                     if (this->lb_file_time < 0) {
                         this->lb_file_time = 0;
                     }
                     this->lb_compressed_offset
                         = lseek(this->lb_fd, 0, SEEK_CUR);
+                    if (!hdr.empty()) {
+                        this->lb_header = std::move(hdr);
+                    }
                     this->resize_buffer(INITIAL_COMPRESSED_BUFFER_SIZE);
                 }
 #ifdef HAVE_BZLIB_H
@@ -763,7 +796,9 @@ line_buffer::fill_range(file_off_t start, ssize_t max_length)
         this->lb_alt_line_has_ansi.clear();
         this->lb_stats.s_used_preloads += 1;
     }
-    if (this->in_range(start) && this->in_range(start + max_length - 1)) {
+    if (this->in_range(start)
+        && (max_length == 0 || this->in_range(start + max_length - 1)))
+    {
         /* Cache already has the data, nothing to do. */
         retval = true;
         if (!lnav::pid::in_child && this->lb_seekable && this->lb_buffer.full()
@@ -1022,11 +1057,16 @@ line_buffer::fill_range(file_off_t start, ssize_t max_length)
 Result<line_info, std::string>
 line_buffer::load_next_line(file_range prev_line)
 {
+    const char* line_start = nullptr;
     bool done = false;
     line_info retval;
 
     require(this->lb_fd != -1);
 
+    if (this->lb_line_metadata && prev_line.fr_offset == 0) {
+        prev_line.fr_offset = this->lb_piper_header_size;
+    }
+
     auto offset = prev_line.next_offset();
     ssize_t request_size = INITIAL_REQUEST_SIZE;
     retval.li_file_range.fr_offset = offset;
@@ -1044,9 +1084,17 @@ line_buffer::load_next_line(file_range prev_line)
             return Ok(retval);
         }
     }
+    if (prev_line.next_offset() == 0) {
+        auto is_utf_res = is_utf8(string_fragment::from_bytes(
+            this->lb_buffer.begin(), this->lb_buffer.size()));
+        this->lb_is_utf8 = is_utf_res.is_valid();
+        if (!this->lb_is_utf8) {
+            log_warning("input is not utf8 -- %s", is_utf_res.usr_message);
+        }
+    }
     while (!done) {
         auto old_retval_size = retval.li_file_range.fr_size;
-        const char *line_start, *lf = nullptr;
+        const char* lf = nullptr;
 
         /* Find the data in the cache and */
         line_start = this->get_range(offset, retval.li_file_range.fr_size);
@@ -1177,16 +1225,40 @@ line_buffer::load_next_line(file_range prev_line)
         = retval.li_utf8_scan_result.usr_has_ansi;
     retval.li_file_range.fr_metadata.m_valid_utf
         = retval.li_utf8_scan_result.is_valid();
+
+    if (this->lb_line_metadata) {
+        auto sv = scn::string_view{
+            line_start,
+            (size_t) retval.li_file_range.fr_size,
+        };
+
+        char level;
+        auto scan_res = scn::scan(sv,
+                                  "{}.{}:{};",
+                                  retval.li_timestamp.tv_sec,
+                                  retval.li_timestamp.tv_usec,
+                                  level);
+        if (scan_res) {
+            retval.li_timestamp.tv_sec
+                = lnav::to_local_time(date::sys_seconds{std::chrono::seconds{
+                                          retval.li_timestamp.tv_sec}})
+                      .time_since_epoch()
+                      .count();
+            retval.li_level = abbrev2level(&level, 1);
+        }
+    }
+
     return Ok(retval);
 }
 
 Result<shared_buffer_ref, std::string>
-line_buffer::read_range(const file_range fr)
+line_buffer::read_range(file_range fr)
 {
     shared_buffer_ref retval;
     const char* line_start;
     file_ssize_t avail;
 
+#if 0
     if (this->lb_last_line_offset != -1
         && fr.fr_offset > this->lb_last_line_offset)
     {
@@ -1200,6 +1272,7 @@ line_buffer::read_range(const file_range fr)
                         fr.fr_offset,
                         this->lb_last_line_offset));
     }
+#endif
 
     if (!(this->in_range(fr.fr_offset)
           && this->in_range(fr.fr_offset + fr.fr_size - 1)))
@@ -1214,6 +1287,15 @@ line_buffer::read_range(const file_range fr)
         return Err(fmt::format(
             FMT_STRING("short-read (need: {}; avail: {})"), fr.fr_size, avail));
     }
+    if (this->lb_line_metadata) {
+        auto new_start
+            = static_cast<const char*>(memchr(line_start, ';', fr.fr_size));
+        if (new_start) {
+            auto offset = new_start - line_start + 1;
+            line_start += offset;
+            fr.fr_size -= offset;
+        }
+    }
     retval.share(this->lb_share_manager, line_start, fr.fr_size);
     retval.get_metadata() = fr.fr_metadata;
 
@@ -1388,12 +1470,13 @@ line_buffer::cleanup_cache()
         auto now = std::chrono::system_clock::now();
         auto cache_path = line_buffer_cache_path();
         std::vector<ghc::filesystem::path> to_remove;
+        std::error_code ec;
 
         for (const auto& cache_subdir :
-             ghc::filesystem::directory_iterator(cache_path))
+             ghc::filesystem::directory_iterator(cache_path, ec))
         {
             for (const auto& entry :
-                 ghc::filesystem::directory_iterator(cache_subdir))
+                 ghc::filesystem::directory_iterator(cache_subdir, ec))
             {
                 auto mtime = ghc::filesystem::last_write_time(entry.path());
                 auto exp_time = mtime + 1h;
@@ -1407,7 +1490,7 @@ line_buffer::cleanup_cache()
 
         for (auto& entry : to_remove) {
             log_debug("removing compressed file cache: %s", entry.c_str());
-            ghc::filesystem::remove_all(entry);
+            ghc::filesystem::remove_all(entry, ec);
         }
     });
 }