Merging upstream version 0.12.2.

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

1 files changed, 774 insertions, 592 deletions

diff --git a/src/lnav.cc b/src/lnav.cc
index 6d56b4d..136f61f 100644
--- a/src/lnav.cc
+++ b/src/lnav.cc
@@ -36,21 +36,14 @@
 #    include <alloca.h>
 #endif
 
-#include <errno.h>
-#include <fcntl.h>
-#include <glob.h>
 #include <locale.h>
 #include <signal.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
-#include <sys/ioctl.h>
 #include <sys/stat.h>
 #include <sys/time.h>
-#include <sys/types.h>
 #include <sys/wait.h>
-#include <termios.h>
-#include <time.h>
 #include <unistd.h>
 
 #include "config.h"
@@ -60,7 +53,6 @@
 #    define _WCHAR_H_CPLUSPLUS_98_CONFORMANCE_
 #endif
 #include <algorithm>
-#include <functional>
 #include <map>
 #include <memory>
 #include <set>
@@ -75,9 +67,9 @@
 
 #include "all_logs_vtab.hh"
 #include "base/ansi_scrubber.hh"
+#include "base/ansi_vars.hh"
 #include "base/fs_util.hh"
 #include "base/func_util.hh"
-#include "base/future_util.hh"
 #include "base/humanize.hh"
 #include "base/humanize.time.hh"
 #include "base/injector.bind.hh"
@@ -87,16 +79,18 @@
 #include "base/lnav_log.hh"
 #include "base/paths.hh"
 #include "base/string_util.hh"
-#include "bookmarks.hh"
 #include "bottom_status_source.hh"
 #include "bound_tags.hh"
 #include "breadcrumb_curses.hh"
 #include "CLI/CLI.hpp"
+#include "date/tz.h"
 #include "dump_internals.hh"
 #include "environ_vtab.hh"
+#include "file_converter_manager.hh"
+#include "file_options.hh"
 #include "filter_sub_source.hh"
 #include "fstat_vtab.hh"
-#include "grep_proc.hh"
+#include "gantt_source.hh"
 #include "hist_source.hh"
 #include "init-sql.h"
 #include "listview_curses.hh"
@@ -114,7 +108,8 @@
 #include "log_vtab_impl.hh"
 #include "logfile.hh"
 #include "logfile_sub_source.hh"
-#include "piper_proc.hh"
+#include "md4cpp.hh"
+#include "piper.looper.hh"
 #include "readline_curses.hh"
 #include "readline_highlighters.hh"
 #include "regexp_vtab.hh"
@@ -137,7 +132,6 @@
 #include "view_helpers.examples.hh"
 #include "view_helpers.hist.hh"
 #include "views_vtab.hh"
-#include "vt52_curses.hh"
 #include "xpath_vtab.hh"
 #include "xterm_mouse.hh"
 
@@ -155,12 +149,10 @@
 #include "command_executor.hh"
 #include "field_overlay_source.hh"
 #include "hotkeys.hh"
-#include "log_actions.hh"
 #include "readline_callbacks.hh"
 #include "readline_possibilities.hh"
-#include "shlex.hh"
 #include "url_loader.hh"
-#include "yajlpp/yajlpp.hh"
+#include "yajlpp/json_ptr.hh"
 
 #ifndef SYSCONFDIR
 #    define SYSCONFDIR "/usr/etc"
@@ -168,6 +160,7 @@
 
 using namespace std::literals::chrono_literals;
 using namespace lnav::roles::literals;
+using namespace md4cpp::literals;
 
 static std::vector<std::string> DEFAULT_FILES;
 static auto intern_lifetime = intern_string::get_table_lifetime();
@@ -201,25 +194,15 @@ const std::vector<std::string> lnav_zoom_strings = {
 };
 
 static const std::vector<std::string> DEFAULT_DB_KEY_NAMES = {
-    "match_index",
-    "capture_index",
-    "capture_count",
-    "range_start",
-    "range_stop",
-    "inode",
-    "device",
-    "inode",
-    "rowid",
-    "st_dev",
-    "st_ino",
-    "st_mode",
-    "st_rdev",
-    "st_uid",
-    "st_gid",
+    "$id",         "capture_count", "capture_index",
+    "device",      "enabled",       "filter_id",
+    "id",          "inode",         "key",
+    "match_index", "parent",        "range_start",
+    "range_stop",  "rowid",         "st_dev",
+    "st_gid",      "st_ino",        "st_mode",
+    "st_rdev",     "st_uid",
 };
 
-const static file_ssize_t MAX_STDIN_CAPTURE_SIZE = 10 * 1024 * 1024;
-
 static auto bound_pollable_supervisor
     = injector::bind<pollable_supervisor>::to_singleton();
 
@@ -236,6 +219,9 @@ static auto bound_lnav_flags
     = injector::bind<unsigned long, lnav_flags_tag>::to_instance(
         &lnav_data.ld_flags);
 
+static auto bound_lnav_exec_context
+    = injector::bind<exec_context>::to_instance(&lnav_data.ld_exec_context);
+
 static auto bound_last_rel_time
     = injector::bind<relative_time, last_relative_time_tag>::to_singleton();
 
@@ -245,6 +231,8 @@ static auto bound_xterm_mouse = injector::bind<xterm_mouse>::to_singleton();
 
 static auto bound_scripts = injector::bind<available_scripts>::to_singleton();
 
+static auto bound_crumbs = injector::bind<breadcrumb_curses>::to_singleton();
+
 static auto bound_curl
     = injector::bind_multiple<isc::service_base>()
           .add_singleton<curl_looper, services::curl_streamer_t>();
@@ -256,6 +244,9 @@ static auto bound_tailer
 static auto bound_main = injector::bind_multiple<static_service>()
                              .add_singleton<main_looper, services::main_t>();
 
+static auto bound_file_options_hier
+    = injector::bind<lnav::safe_file_options_hier>::to_singleton();
+
 namespace injector {
 template<>
 void
@@ -288,41 +279,18 @@ force_linking(services::main_t anno)
 }
 }  // namespace injector
 
-static breadcrumb_curses breadcrumb_view;
-
 struct lnav_data_t lnav_data;
 
 bool
 setup_logline_table(exec_context& ec)
 {
-    // Hidden columns don't show up in the table_info pragma.
-    static const char* hidden_table_columns[] = {
-        "log_time_msecs",
-        "log_path",
-        "log_text",
-        "log_body",
-
-        nullptr,
-    };
-
-    textview_curses& log_view = lnav_data.ld_views[LNV_LOG];
+    auto& log_view = lnav_data.ld_views[LNV_LOG];
     bool retval = false;
-    bool update_possibilities
-        = (lnav_data.ld_rl_view != nullptr && ec.ec_local_vars.size() == 1);
-
-    if (update_possibilities) {
-        lnav_data.ld_rl_view->clear_possibilities(ln_mode_t::SQL, "*");
-        add_view_text_possibilities(lnav_data.ld_rl_view,
-                                    ln_mode_t::SQL,
-                                    "*",
-                                    &log_view,
-                                    text_quoting::sql);
-    }
 
     if (log_view.get_inner_height()) {
         static intern_string_t logline = intern_string::lookup("logline");
-        vis_line_t vl = log_view.get_selection();
-        content_line_t cl = lnav_data.ld_log_source.at_base(vl);
+        auto vl = log_view.get_selection();
+        auto cl = lnav_data.ld_log_source.at_base(vl);
 
         lnav_data.ld_vtab_manager->unregister_vtab(logline);
         lnav_data.ld_vtab_manager->register_vtab(
@@ -331,26 +299,6 @@ setup_logline_table(exec_context& ec)
                                              cl,
                                              logline));
 
-        if (update_possibilities) {
-            log_data_helper ldh(lnav_data.ld_log_source);
-
-            ldh.parse_line(cl);
-
-            std::map<const intern_string_t,
-                     json_ptr_walk::walk_list_t>::const_iterator pair_iter;
-            for (pair_iter = ldh.ldh_json_pairs.begin();
-                 pair_iter != ldh.ldh_json_pairs.end();
-                 ++pair_iter)
-            {
-                for (size_t lpc = 0; lpc < pair_iter->second.size(); lpc++) {
-                    lnav_data.ld_rl_view->add_possibility(
-                        ln_mode_t::SQL,
-                        "*",
-                        ldh.format_json_getter(pair_iter->first, lpc));
-                }
-            }
-        }
-
         retval = true;
     }
 
@@ -358,60 +306,6 @@ setup_logline_table(exec_context& ec)
 
     db_key_names = DEFAULT_DB_KEY_NAMES;
 
-    if (update_possibilities) {
-        add_env_possibilities(ln_mode_t::SQL);
-
-        lnav_data.ld_rl_view->add_possibility(ln_mode_t::SQL,
-                                              "*",
-                                              std::begin(sql_keywords),
-                                              std::end(sql_keywords));
-        lnav_data.ld_rl_view->add_possibility(
-            ln_mode_t::SQL, "*", sql_function_names);
-        lnav_data.ld_rl_view->add_possibility(
-            ln_mode_t::SQL, "*", hidden_table_columns);
-
-        for (int lpc = 0; sqlite_registration_funcs[lpc]; lpc++) {
-            struct FuncDef* basic_funcs;
-            struct FuncDefAgg* agg_funcs;
-
-            sqlite_registration_funcs[lpc](&basic_funcs, &agg_funcs);
-            for (int lpc2 = 0; basic_funcs && basic_funcs[lpc2].zName; lpc2++) {
-                const FuncDef& func_def = basic_funcs[lpc2];
-
-                lnav_data.ld_rl_view->add_possibility(
-                    ln_mode_t::SQL,
-                    "*",
-                    std::string(func_def.zName) + (func_def.nArg ? "(" : "()"));
-            }
-            for (int lpc2 = 0; agg_funcs && agg_funcs[lpc2].zName; lpc2++) {
-                const FuncDefAgg& func_def = agg_funcs[lpc2];
-
-                lnav_data.ld_rl_view->add_possibility(
-                    ln_mode_t::SQL,
-                    "*",
-                    std::string(func_def.zName) + (func_def.nArg ? "(" : "()"));
-            }
-        }
-
-        for (const auto& pair : sqlite_function_help) {
-            switch (pair.second->ht_context) {
-                case help_context_t::HC_SQL_FUNCTION:
-                case help_context_t::HC_SQL_TABLE_VALUED_FUNCTION: {
-                    std::string poss = pair.first
-                        + (pair.second->ht_parameters.empty() ? "()" : ("("));
-
-                    lnav_data.ld_rl_view->add_possibility(
-                        ln_mode_t::SQL, "*", poss);
-                    break;
-                }
-                default:
-                    break;
-            }
-        }
-    }
-
-    walk_sqlite_metadata(lnav_data.ld_db.in(), lnav_sql_meta_callbacks);
-
     for (const auto& iter : *lnav_data.ld_vtab_manager) {
         iter.second->get_foreign_keys(db_key_names);
     }
@@ -454,10 +348,7 @@ append_default_files()
 static void
 sigint(int sig)
 {
-    static size_t counter = 0;
-
-    lnav_data.ld_looping = false;
-    counter += 1;
+    auto counter = lnav_data.ld_sigint_count.fetch_add(1);
     if (counter >= 3) {
         abort();
     }
@@ -476,16 +367,22 @@ sigchld(int sig)
 }
 
 static void
-handle_rl_key(int ch)
+handle_rl_key(int ch, const char* keyseq)
 {
     switch (ch) {
+        case KEY_F(2):
+            if (xterm_mouse::is_available()) {
+                auto& mouse_i = injector::get<xterm_mouse&>();
+                mouse_i.set_enabled(!mouse_i.is_enabled());
+            }
+            break;
         case KEY_PPAGE:
         case KEY_NPAGE:
-        case KEY_CTRL_P:
-            handle_paging_key(ch);
+        case KEY_CTRL('p'):
+            handle_paging_key(ch, keyseq);
             break;
 
-        case KEY_CTRL_RBRACKET:
+        case KEY_CTRL(']'):
             lnav_data.ld_rl_view->abort();
             break;
 
@@ -543,11 +440,14 @@ usage()
 
     ex3_term.append(lnav::roles::ok("$"))
         .append(" ")
-        .append(lnav::roles::file("make"))
-        .append(" 2>&1 | ")
         .append(lnav::roles::file("lnav"))
         .append(" ")
-        .append("-t"_symbol)
+        .append("-e"_symbol)
+        .append(" '")
+        .append(lnav::roles::file("make"))
+        .append(" ")
+        .append("-j4"_symbol)
+        .append("' ")
         .pad_to(40)
         .with_attr_for_all(VC_ROLE.value(role_t::VCR_QUOTED_CODE));
 
@@ -623,19 +523,6 @@ make it easier to navigate through files quickly.
         .append("         ")
         .append("Load older rotated log files as well.\n")
         .append("  ")
-        .append("-t"_symbol)
-        .append("         ")
-        .append(R"(Prepend timestamps to the lines of data being read in
-             from the standard input.
-)")
-        .append("  ")
-        .append("-w"_symbol)
-        .append(" ")
-        .append("file"_variable)
-        .append("    ")
-        .append("Write the contents of the standard input to this file.\n")
-        .append("\n")
-        .append("  ")
         .append("-c"_symbol)
         .append(" ")
         .append("cmd"_variable)
@@ -648,6 +535,16 @@ make it easier to navigate through files quickly.
         .append("    ")
         .append("Execute the commands in the given file.\n")
         .append("  ")
+        .append("-e"_symbol)
+        .append(" ")
+        .append("cmd"_variable)
+        .append("     ")
+        .append("Execute a shell command-line.\n")
+        .append("  ")
+        .append("-t"_symbol)
+        .append("         ")
+        .append("Treat data piped into standard in as a log file.\n")
+        .append("  ")
         .append("-n"_symbol)
         .append("         ")
         .append("Run without the curses UI. (headless mode)\n")
@@ -658,10 +555,7 @@ make it easier to navigate through files quickly.
         .append("  ")
         .append("-q"_symbol)
         .append("         ")
-        .append(
-            R"(Do not print the log messages after executing all
-             of the commands.
-)")
+        .append("Do not print informational messages.\n")
         .append("\n")
         .append("Optional arguments"_h2)
         .append("\n")
@@ -704,7 +598,7 @@ make it easier to navigate through files quickly.
         .append("\u2022"_list_glyph)
         .append(" To watch the output of ")
         .append(lnav::roles::file("make"))
-        .append(" with timestamps prepended:\n")
+        .append(":\n")
         .append("     ")
         .append(ex3_term)
         .append("\n\n")
@@ -712,28 +606,42 @@ make it easier to navigate through files quickly.
         .append("\n ")
         .append("\u2022"_list_glyph)
         .append(" Format files are read from:")
-        .append("\n    \U0001F4C2 ")
+        .append("\n    ")
+        .append(":open_file_folder:"_emoji)
+        .append(" ")
         .append(lnav::roles::file("/etc/lnav"))
-        .append("\n    \U0001F4C2 ")
+        .append("\n    ")
+        .append(":open_file_folder:"_emoji)
+        .append(" ")
         .append(lnav::roles::file(SYSCONFDIR "/lnav"))
         .append("\n ")
         .append("\u2022"_list_glyph)
         .append(" Configuration, session, and format files are stored in:\n")
-        .append("    \U0001F4C2 ")
+        .append("    ")
+        .append(":open_file_folder:"_emoji)
+        .append(" ")
         .append(lnav::roles::file(lnav::paths::dotlnav().string()))
         .append("\n\n ")
         .append("\u2022"_list_glyph)
-        .append(" Local copies of remote files and files extracted from\n")
-        .append("   archives are stored in:\n")
-        .append("    \U0001F4C2 ")
+        .append(" Local copies of remote files, files extracted from\n")
+        .append("   archives, execution output, and so on are stored in:\n")
+        .append("    ")
+        .append(":open_file_folder:"_emoji)
+        .append(" ")
         .append(lnav::roles::file(lnav::paths::workdir().string()))
         .append("\n\n")
         .append("Documentation"_h1)
-        .append(": https://docs.lnav.org\n")
+        .append(": ")
+        .append("https://docs.lnav.org"_hyperlink)
+        .append("\n")
         .append("Contact"_h1)
         .append("\n")
-        .append("  \U0001F4AC https://github.com/tstack/lnav/discussions\n")
-        .appendf(FMT_STRING("  \U0001F4EB {}\n"), PACKAGE_BUGREPORT)
+        .append("  ")
+        .append(":speech_balloon:"_emoji)
+        .append(" https://github.com/tstack/lnav/discussions\n")
+        .append("  ")
+        .append(":mailbox:"_emoji)
+        .appendf(FMT_STRING(" {}\n"), PACKAGE_BUGREPORT)
         .append("Version"_h1)
         .appendf(FMT_STRING(": {}"), VCS_PACKAGE_STRING);
 
@@ -743,7 +651,7 @@ make it easier to navigate through files quickly.
 static void
 clear_last_user_mark(listview_curses* lv)
 {
-    textview_curses* tc = (textview_curses*) lv;
+    auto* tc = (textview_curses*) lv;
     if (lnav_data.ld_select_start.find(tc) != lnav_data.ld_select_start.end()
         && !tc->is_line_visible(vis_line_t(lnav_data.ld_last_user_mark[tc])))
     {
@@ -766,52 +674,19 @@ update_view_position(listview_curses* lv)
     };
 }
 
-class lnav_behavior : public mouse_behavior {
-public:
-    void mouse_event(int button, bool release, int x, int y) override
-    {
-        textview_curses* tc = *(lnav_data.ld_view_stack.top());
-        struct mouse_event me;
-
-        switch (button & xterm_mouse::XT_BUTTON__MASK) {
-            case xterm_mouse::XT_BUTTON1:
-                me.me_button = mouse_button_t::BUTTON_LEFT;
-                break;
-            case xterm_mouse::XT_BUTTON2:
-                me.me_button = mouse_button_t::BUTTON_MIDDLE;
-                break;
-            case xterm_mouse::XT_BUTTON3:
-                me.me_button = mouse_button_t::BUTTON_RIGHT;
-                break;
-            case xterm_mouse::XT_SCROLL_UP:
-                me.me_button = mouse_button_t::BUTTON_SCROLL_UP;
-                break;
-            case xterm_mouse::XT_SCROLL_DOWN:
-                me.me_button = mouse_button_t::BUTTON_SCROLL_DOWN;
-                break;
-        }
-
-        if (button & xterm_mouse::XT_DRAG_FLAG) {
-            me.me_state = mouse_button_state_t::BUTTON_STATE_DRAGGED;
-        } else if (release) {
-            me.me_state = mouse_button_state_t::BUTTON_STATE_RELEASED;
-        } else {
-            me.me_state = mouse_button_state_t::BUTTON_STATE_PRESSED;
-        }
-
-        gettimeofday(&me.me_time, nullptr);
-        me.me_x = x - 1;
-        me.me_y = y - tc->get_y() - 1;
-
-        tc->handle_mouse(me);
-    }
-};
-
 static bool
-handle_config_ui_key(int ch)
+handle_config_ui_key(int ch, const char* keyseq)
 {
     bool retval = false;
 
+    if (ch == KEY_F(2)) {
+        if (xterm_mouse::is_available()) {
+            auto& mouse_i = injector::get<xterm_mouse&>();
+            mouse_i.set_enabled(!mouse_i.is_enabled());
+        }
+        return retval;
+    }
+
     switch (lnav_data.ld_mode) {
         case ln_mode_t::FILES:
             retval = lnav_data.ld_files_view.handle_key(ch);
@@ -840,7 +715,7 @@ handle_config_ui_key(int ch)
         } else {
             new_mode = ln_mode_t::FILES;
         }
-    } else if (ch == 'q') {
+    } else if (ch == 'q' || ch == KEY_ESCAPE) {
         new_mode = ln_mode_t::PAGING;
     }
 
@@ -855,15 +730,17 @@ handle_config_ui_key(int ch)
         lnav_data.ld_filter_view.reload_data();
         lnav_data.ld_status[LNS_FILTER].set_needs_update();
     } else {
-        return handle_paging_key(ch);
+        return handle_paging_key(ch, keyseq);
     }
 
     return true;
 }
 
 static bool
-handle_key(int ch)
+handle_key(int ch, const char* keyseq)
 {
+    static auto* breadcrumb_view = injector::get<breadcrumb_curses*>();
+
     lnav_data.ld_input_state.push_back(ch);
 
     switch (ch) {
@@ -872,16 +749,10 @@ handle_key(int ch)
         default: {
             switch (lnav_data.ld_mode) {
                 case ln_mode_t::PAGING:
-                    if (ch == KEY_ENTER || ch == '\n' || ch == '\r') {
-                        breadcrumb_view.focus();
-                        lnav_data.ld_mode = ln_mode_t::BREADCRUMBS;
-                        return true;
-                    }
-
-                    return handle_paging_key(ch);
+                    return handle_paging_key(ch, keyseq);
 
                 case ln_mode_t::BREADCRUMBS:
-                    if (!breadcrumb_view.handle_key(ch)) {
+                    if (ch == '`' || !breadcrumb_view->handle_key(ch)) {
                         lnav_data.ld_mode = ln_mode_t::PAGING;
                         lnav_data.ld_view_stack.set_needs_update();
                         return true;
@@ -890,7 +761,7 @@ handle_key(int ch)
 
                 case ln_mode_t::FILTER:
                 case ln_mode_t::FILES:
-                    return handle_config_ui_key(ch);
+                    return handle_config_ui_key(ch, keyseq);
 
                 case ln_mode_t::SPECTRO_DETAILS: {
                     if (ch == '\t' || ch == 'q') {
@@ -929,12 +800,13 @@ handle_key(int ch)
                 case ln_mode_t::SQL:
                 case ln_mode_t::EXEC:
                 case ln_mode_t::USER:
-                    handle_rl_key(ch);
+                    handle_rl_key(ch, keyseq);
                     break;
 
                 case ln_mode_t::BUSY:
                     switch (ch) {
-                        case KEY_CTRL_RBRACKET:
+                        case KEY_ESCAPE:
+                        case KEY_CTRL(']'):
                             log_vtab_data.lvd_looping = false;
                             break;
                     }
@@ -982,18 +854,6 @@ match_escape_seq(const char* keyseq)
 static void
 gather_pipers()
 {
-    for (auto iter = lnav_data.ld_pipers.begin();
-         iter != lnav_data.ld_pipers.end();)
-    {
-        pid_t child_pid = (*iter)->get_child_pid();
-        if ((*iter)->has_exited()) {
-            log_info("child piper has exited -- %d", child_pid);
-            iter = lnav_data.ld_pipers.erase(iter);
-        } else {
-            ++iter;
-        }
-    }
-
     for (auto iter = lnav_data.ld_child_pollers.begin();
          iter != lnav_data.ld_child_pollers.end();)
     {
@@ -1007,21 +867,37 @@ gather_pipers()
     }
 }
 
-static void
-wait_for_pipers()
+void
+wait_for_pipers(nonstd::optional<timeval> deadline)
 {
+    static const auto MAX_SLEEP_TIME = std::chrono::milliseconds(300);
+    auto sleep_time = std::chrono::milliseconds(10);
+
     for (;;) {
         gather_pipers();
-        if (lnav_data.ld_pipers.empty() && lnav_data.ld_child_pollers.empty()) {
+        auto piper_count = lnav_data.ld_active_files.active_pipers();
+        if (piper_count == 0 && lnav_data.ld_child_pollers.empty()) {
             log_debug("all pipers finished");
             break;
         }
-        usleep(10000);
+        if (deadline && (deadline.value() < current_timeval())) {
+            break;
+        }
+        // Use usleep() since it is defined to be interruptable by a signal.
+        auto urc = usleep(
+            std::chrono::duration_cast<std::chrono::microseconds>(sleep_time)
+                .count());
+        if (urc == -1 && errno == EINTR) {
+            log_trace("wait_for_pipers(): sleep interrupted");
+        }
         rebuild_indexes();
 
-        log_debug("%d pipers and %d children still active",
-                  lnav_data.ld_pipers.size(),
+        log_debug("%d pipers and %d children are still active",
+                  piper_count,
                   lnav_data.ld_child_pollers.size());
+        if (sleep_time < MAX_SLEEP_TIME) {
+            sleep_time = sleep_time * 2;
+        }
     }
 }
 
@@ -1075,10 +951,55 @@ struct refresh_status_bars {
 };
 
 static void
+check_for_file_zones()
+{
+    auto with_tz_count = 0;
+    std::vector<std::string> without_tz_files;
+
+    for (const auto& lf : lnav_data.ld_active_files.fc_files) {
+        auto format = lf->get_format_ptr();
+        if (format == nullptr) {
+            continue;
+        }
+
+        if (format->lf_timestamp_flags & ETF_ZONE_SET
+            || format->lf_date_time.dts_default_zone != nullptr)
+        {
+            with_tz_count += 1;
+        } else {
+            without_tz_files.emplace_back(lf->get_unique_path());
+        }
+    }
+    if (with_tz_count > 0 && !without_tz_files.empty()) {
+        auto note
+            = attr_line_t("The file(s) without a zone: ")
+                  .join(
+                      without_tz_files, VC_ROLE.value(role_t::VCR_FILE), ", ");
+        auto um
+            = lnav::console::user_message::warning(
+                  "Some messages may not be sorted by time correctly")
+                  .with_reason(
+                      "There are one or more files whose messages do not have "
+                      "a timezone in their timestamps mixed in with files that "
+                      "do have timezones")
+                  .with_note(note)
+                  .with_help(
+                      attr_line_t("Use the ")
+                          .append(":set-file-timezone"_symbol)
+                          .append(
+                              " command to set the zone for messages in files "
+                              "that do not include a zone in the timestamp"));
+
+        lnav_data.ld_exec_context.ec_error_callback_stack.back()(um);
+    }
+}
+
+static void
 looper()
 {
     static auto* ps = injector::get<pollable_supervisor*>();
     static auto* filter_source = injector::get<filter_sub_source*>();
+    static auto* breadcrumb_view = injector::get<breadcrumb_curses*>();
 
     try {
         auto* sql_cmd_map = injector::get<readline_context::command_map_t*,
@@ -1113,7 +1034,8 @@ looper()
         sql_context.set_highlighter(readline_sqlite_highlighter)
             .set_quote_chars("\"")
             .with_readline_var((char**) &rl_completer_word_break_characters,
-                               " \t\n(),");
+                               " \t\n(),")
+            .with_splitter(prql_splitter);
         exec_context.set_highlighter(readline_shlex_highlighter);
 
         lnav_data.ld_log_source.lss_sorting_observer
@@ -1209,10 +1131,30 @@ looper()
         }
         auto echo_views_stmt = echo_views_stmt_res.unwrap();
 
+        if (xterm_mouse::is_available()
+            && lnav_config.lc_mouse_mode == lnav_mouse_mode::disabled)
+        {
+            auto mouse_note = prepare_stmt(lnav_data.ld_db, R"(
+INSERT INTO lnav_user_notifications (id, priority, expiration, message)
+VALUES ('org.lnav.mouse-support', -1, DATETIME('now', '+1 minute'),
+        'Press <span class="-lnav_status-styles_hotkey">F2</span> to enable mouse support');
+)");
+            if (mouse_note.isErr()) {
+                lnav::console::print(
+                    stderr,
+                    lnav::console::user_message::error(
+                        "unable to prepare INSERT statement for "
+                        "lnav_user_notifications table")
+                        .with_reason(mouse_note.unwrapErr()));
+                return;
+            }
+
+            mouse_note.unwrap().execute();
+        }
+
         (void) signal(SIGINT, sigint);
         (void) signal(SIGTERM, sigint);
         (void) signal(SIGWINCH, sigwinch);
-        (void) signal(SIGCHLD, sigchld);
 
         auto create_screen_res = screen_curses::create();
 
@@ -1231,6 +1173,8 @@ looper()
         auto_fd errpipe[2];
         auto_fd::pipe(errpipe);
 
+        errpipe[0].close_on_exec();
+        errpipe[1].close_on_exec();
         dup2(errpipe[1], STDERR_FILENO);
         errpipe[1].reset();
         log_pipe_err(errpipe[0]);
@@ -1238,10 +1182,12 @@ looper()
 
         ui_periodic_timer::singleton();
 
-        auto mouse_i = injector::get<xterm_mouse&>();
+        auto& mouse_i = injector::get<xterm_mouse&>();
 
         mouse_i.set_behavior(&lb);
-        mouse_i.set_enabled(check_experimental("mouse"));
+        mouse_i.set_enabled(check_experimental("mouse")
+                            || lnav_config.lc_mouse_mode
+                                == lnav_mouse_mode::enabled);
 
         lnav_data.ld_window = sc.get_window();
         keypad(stdscr, TRUE);
@@ -1287,7 +1233,8 @@ looper()
             setup_highlights(lnav_data.ld_views[LNV_TEXT].get_highlights());
             setup_highlights(lnav_data.ld_views[LNV_SCHEMA].get_highlights());
             setup_highlights(lnav_data.ld_views[LNV_PRETTY].get_highlights());
-            setup_highlights(lnav_data.ld_preview_view.get_highlights());
+            setup_highlights(lnav_data.ld_preview_view[0].get_highlights());
+            setup_highlights(lnav_data.ld_preview_view[1].get_highlights());
 
             for (const auto& format : log_format::get_root_formats()) {
                 for (auto& hl : format->lf_highlighters) {
@@ -1309,7 +1256,6 @@ looper()
         execute_examples();
 
         rlc->set_window(lnav_data.ld_window);
-        rlc->set_y(-1);
         rlc->set_focus_action(rl_focus);
         rlc->set_change_action(rl_change);
         rlc->set_perform_action(rl_callback);
@@ -1342,9 +1288,15 @@ looper()
 
         vsb.push_back(sb);
 
-        breadcrumb_view.set_y(1);
-        breadcrumb_view.set_window(lnav_data.ld_window);
-        breadcrumb_view.set_line_source(lnav_crumb_source);
+        breadcrumb_view->on_focus
+            = [](breadcrumb_curses&) { set_view_mode(ln_mode_t::BREADCRUMBS); };
+        breadcrumb_view->on_blur = [](breadcrumb_curses&) {
+            set_view_mode(ln_mode_t::PAGING);
+            lnav_data.ld_view_stack.set_needs_update();
+        };
+        breadcrumb_view->set_y(1);
+        breadcrumb_view->set_window(lnav_data.ld_window);
+        breadcrumb_view->set_line_source(lnav_crumb_source);
         auto event_handler = [](auto&& tc) {
             auto top_view = lnav_data.ld_view_stack.top();
 
@@ -1360,8 +1312,17 @@ looper()
             lnav_data.ld_views[lpc].set_scroll_action(sb);
             lnav_data.ld_views[lpc].set_search_action(update_hits);
             lnav_data.ld_views[lpc].tc_cursor_role = role_t::VCR_CURSOR_LINE;
+            lnav_data.ld_views[lpc].tc_disabled_cursor_role
+                = role_t::VCR_DISABLED_CURSOR_LINE;
             lnav_data.ld_views[lpc].tc_state_event_handler = event_handler;
         }
+        lnav_data.ld_views[LNV_DB].set_supports_marks(true);
+        lnav_data.ld_views[LNV_HELP].set_supports_marks(true);
+        lnav_data.ld_views[LNV_HISTOGRAM].set_supports_marks(true);
+        lnav_data.ld_views[LNV_LOG].set_supports_marks(true);
+        lnav_data.ld_views[LNV_TEXT].set_supports_marks(true);
+        lnav_data.ld_views[LNV_SCHEMA].set_supports_marks(true);
+        lnav_data.ld_views[LNV_PRETTY].set_supports_marks(true);
 
         lnav_data.ld_doc_view.set_window(lnav_data.ld_window);
         lnav_data.ld_doc_view.set_show_scrollbar(false);
@@ -1371,13 +1332,17 @@ looper()
 
         lnav_data.ld_match_view.set_window(lnav_data.ld_window);
 
-        lnav_data.ld_preview_view.set_window(lnav_data.ld_window);
-        lnav_data.ld_preview_view.set_show_scrollbar(false);
+        lnav_data.ld_preview_view[0].set_window(lnav_data.ld_window);
+        lnav_data.ld_preview_view[0].set_show_scrollbar(false);
+        lnav_data.ld_preview_view[1].set_window(lnav_data.ld_window);
+        lnav_data.ld_preview_view[1].set_show_scrollbar(false);
 
+        lnav_data.ld_filter_view.set_title("Text Filters");
         lnav_data.ld_filter_view.set_selectable(true);
         lnav_data.ld_filter_view.set_window(lnav_data.ld_window);
         lnav_data.ld_filter_view.set_show_scrollbar(true);
 
+        lnav_data.ld_files_view.set_title("Files");
         lnav_data.ld_files_view.set_selectable(true);
         lnav_data.ld_files_view.set_window(lnav_data.ld_window);
         lnav_data.ld_files_view.set_show_scrollbar(true);
@@ -1387,6 +1352,7 @@ looper()
 
         lnav_data.ld_user_message_view.set_window(lnav_data.ld_window);
 
+        lnav_data.ld_spectro_details_view.set_title("spectro-details");
         lnav_data.ld_spectro_details_view.set_window(lnav_data.ld_window);
         lnav_data.ld_spectro_details_view.set_show_scrollbar(true);
         lnav_data.ld_spectro_details_view.set_height(5_vl);
@@ -1405,16 +1371,49 @@ looper()
         lnav_data.ld_spectro_source->ss_exec_context
             = &lnav_data.ld_exec_context;
 
+        lnav_data.ld_gantt_details_view.set_title("gantt-details");
+        lnav_data.ld_gantt_details_view.set_window(lnav_data.ld_window);
+        lnav_data.ld_gantt_details_view.set_show_scrollbar(false);
+        lnav_data.ld_gantt_details_view.set_height(5_vl);
+        lnav_data.ld_gantt_details_view.set_sub_source(
+            &lnav_data.ld_gantt_details_source);
+
         auto top_status_lifetime
             = injector::bind<top_status_source>::to_scoped_singleton();
 
         auto top_source = injector::get<std::shared_ptr<top_status_source>>();
 
-        lnav_data.ld_status[LNS_TOP].set_top(0);
+        lnav_data.ld_bottom_source.get_field(bottom_status_source::BSF_HELP)
+            .on_click
+            = [](status_field&) { ensure_view(&lnav_data.ld_views[LNV_HELP]); };
+        lnav_data.ld_bottom_source
+            .get_field(bottom_status_source::BSF_LINE_NUMBER)
+            .on_click
+            = [](status_field&) {
+                  auto cmd = fmt::format(
+                      FMT_STRING("prompt command : 'goto {}'"),
+                      (int) lnav_data.ld_view_stack.top().value()->get_top());
+
+                  execute_command(lnav_data.ld_exec_context, cmd);
+              };
+        lnav_data.ld_bottom_source
+            .get_field(bottom_status_source::BSF_SEARCH_TERM)
+            .on_click
+            = [](status_field&) {
+                  auto term = lnav_data.ld_view_stack.top()
+                                  .value()
+                                  ->get_current_search();
+                  auto cmd
+                      = fmt::format(FMT_STRING("prompt search / '{}'"), term);
+
+                  execute_command(lnav_data.ld_exec_context, cmd);
+              };
+
+        lnav_data.ld_status[LNS_TOP].set_y(0);
         lnav_data.ld_status[LNS_TOP].set_default_role(
             role_t::VCR_INACTIVE_STATUS);
         lnav_data.ld_status[LNS_TOP].set_data_source(top_source.get());
-        lnav_data.ld_status[LNS_BOTTOM].set_top(-(rlc->get_height() + 1));
+        lnav_data.ld_status[LNS_BOTTOM].set_y(-(rlc->get_height() + 1));
         for (auto& stat_bar : lnav_data.ld_status) {
             stat_bar.set_window(lnav_data.ld_window);
         }
@@ -1426,12 +1425,17 @@ looper()
             &lnav_data.ld_filter_help_status_source);
         lnav_data.ld_status[LNS_DOC].set_data_source(
             &lnav_data.ld_doc_status_source);
-        lnav_data.ld_status[LNS_PREVIEW].set_data_source(
-            &lnav_data.ld_preview_status_source);
+        lnav_data.ld_status[LNS_PREVIEW0].set_data_source(
+            &lnav_data.ld_preview_status_source[0]);
+        lnav_data.ld_status[LNS_PREVIEW1].set_data_source(
+            &lnav_data.ld_preview_status_source[1]);
         lnav_data.ld_spectro_status_source
             = std::make_unique<spectro_status_source>();
         lnav_data.ld_status[LNS_SPECTRO].set_data_source(
             lnav_data.ld_spectro_status_source.get());
+        lnav_data.ld_status[LNS_GANTT].set_enabled(false);
+        lnav_data.ld_status[LNS_GANTT].set_data_source(
+            &lnav_data.ld_gantt_status_source);
 
         lnav_data.ld_match_view.set_show_bottom_border(true);
         lnav_data.ld_user_message_view.set_show_bottom_border(true);
@@ -1469,13 +1473,13 @@ looper()
                 // pre-filled with a suggestion that the user can complete.
                 // This quick-fix key could be used for other stuff as well
                 lnav_data.ld_rl_view->set_value(fmt::format(
-                    ANSI_CSI ANSI_COLOR_PARAM(
+                    FMT_STRING(ANSI_CSI ANSI_COLOR_PARAM(
                         COLOR_YELLOW) ";" ANSI_BOLD_PARAM ANSI_CHAR_ATTR
                                       "Unrecognized key" ANSI_NORM
                                       ", bind to a command using "
                                       "\u2014 " ANSI_BOLD(
                                           ":config") " /ui/keymap-defs/{}/{}/"
-                                                     "command <cmd>",
+                                                     "command <cmd>"),
                     encoded_name,
                     keyseq));
                 alerter::singleton().chime("unrecognized key");
@@ -1499,15 +1503,13 @@ looper()
 
         timer.start_fade(index_counter, 1);
 
-        file_collection active_copy;
-        log_debug("rescan started %p", &active_copy);
-        active_copy.merge(lnav_data.ld_active_files);
-        active_copy.fc_progress = lnav_data.ld_active_files.fc_progress;
-        std::future<file_collection> rescan_future
-            = std::async(std::launch::async,
-                         &file_collection::rescan_files,
-                         std::move(active_copy),
-                         false);
+        std::future<file_collection> rescan_future;
+
+        log_debug("rescan started");
+        rescan_future = std::async(std::launch::async,
+                                   &file_collection::rescan_files,
+                                   lnav_data.ld_active_files.copy(),
+                                   false);
         bool initial_rescan_completed = false;
         int session_stage = 0;
 
@@ -1529,7 +1531,8 @@ looper()
             gettimeofday(&current_time, nullptr);
 
             top_source->update_time(current_time);
-            lnav_data.ld_preview_view.set_needs_update();
+            lnav_data.ld_preview_view[0].set_needs_update();
+            lnav_data.ld_preview_view[1].set_needs_update();
 
             layout_views();
 
@@ -1538,16 +1541,14 @@ looper()
                 && rescan_future.wait_for(scan_timeout)
                     == std::future_status::ready)
             {
+                auto ui_now = ui_clock::now();
                 auto new_files = rescan_future.get();
-                if (!initial_rescan_completed && new_files.fc_file_names.empty()
-                    && new_files.fc_files.empty()
-                    && lnav_data.ld_active_files.fc_progress->readAccess()
-                           ->sp_tailers.empty())
-                {
+                if (!initial_rescan_completed && new_files.empty()) {
                     initial_rescan_completed = true;
 
                     log_debug("initial rescan rebuild");
-                    changes += rebuild_indexes(loop_deadline);
+                    auto rebuild_res = rebuild_indexes(loop_deadline);
+                    changes += rebuild_res.rir_changes;
                     load_session();
                     if (session_data.sd_save_time) {
                         std::string ago;
@@ -1566,9 +1567,9 @@ looper()
 
                     lnav_data.ld_session_loaded = true;
                     session_stage += 1;
-                    loop_deadline = ui_clock::now();
+                    loop_deadline = ui_now;
                     log_debug("file count %d",
-                              lnav_data.ld_active_files.fc_files.size())
+                              lnav_data.ld_active_files.fc_files.size());
                 }
                 update_active_files(new_files);
                 if (!initial_rescan_completed) {
@@ -1580,21 +1581,20 @@ looper()
                     }
                 }
 
-                active_copy.clear();
                 rescan_future = std::future<file_collection>{};
-                next_rescan_time = ui_clock::now() + 333ms;
+                next_rescan_time = ui_now + 333ms;
             }
 
             if (!rescan_future.valid()
-                && (session_stage < 2 || ui_clock::now() >= next_rescan_time))
+                && (session_stage < 2
+                    || (lnav_data.ld_active_files.is_below_open_file_limit()
+                        && ui_clock::now() >= next_rescan_time)))
             {
-                active_copy.clear();
-                active_copy.merge(lnav_data.ld_active_files);
-                active_copy.fc_progress = lnav_data.ld_active_files.fc_progress;
                 rescan_future = std::async(std::launch::async,
                                            &file_collection::rescan_files,
-                                           std::move(active_copy),
+                                           lnav_data.ld_active_files.copy(),
                                            false);
+                loop_deadline = ui_clock::now() + 10ms;
             }
 
             {
@@ -1607,7 +1607,8 @@ looper()
             if (initial_rescan_completed) {
                 if (ui_now >= next_rebuild_time) {
                     auto text_file_count = lnav_data.ld_text_source.size();
-                    changes += rebuild_indexes(loop_deadline);
+                    auto rebuild_res = rebuild_indexes(loop_deadline);
+                    changes += rebuild_res.rir_changes;
                     if (!changes && ui_clock::now() < loop_deadline) {
                         next_rebuild_time = ui_clock::now() + 333ms;
                     }
@@ -1627,16 +1628,20 @@ looper()
             }
 
             if (lnav_data.ld_mode == ln_mode_t::BREADCRUMBS
-                && breadcrumb_view.get_needs_update())
+                && breadcrumb_view->get_needs_update())
             {
                 lnav_data.ld_view_stack.set_needs_update();
             }
-            lnav_data.ld_view_stack.do_update();
+            if (lnav_data.ld_view_stack.do_update()) {
+                breadcrumb_view->set_needs_update();
+            }
             lnav_data.ld_doc_view.do_update();
             lnav_data.ld_example_view.do_update();
             lnav_data.ld_match_view.do_update();
-            lnav_data.ld_preview_view.do_update();
+            lnav_data.ld_preview_view[0].do_update();
+            lnav_data.ld_preview_view[1].do_update();
             lnav_data.ld_spectro_details_view.do_update();
+            lnav_data.ld_gantt_details_view.do_update();
             lnav_data.ld_user_message_view.do_update();
             if (ui_clock::now() >= next_status_update_time) {
                 echo_views_stmt.execute();
@@ -1649,7 +1654,7 @@ looper()
             if (filter_source->fss_editing) {
                 filter_source->fss_match_view.set_needs_update();
             }
-            breadcrumb_view.do_update();
+            breadcrumb_view->do_update();
             // These updates need to be done last so their readline views can
             // put the cursor in the right place.
             switch (lnav_data.ld_mode) {
@@ -1704,7 +1709,7 @@ looper()
             auto poll_to
                 = (!changes && ui_now < loop_deadline && session_stage >= 1)
                 ? std::chrono::duration_cast<std::chrono::milliseconds>(
-                    loop_deadline - ui_now)
+                      loop_deadline - ui_now)
                 : 0ms;
 
             if (initial_rescan_completed
@@ -1718,6 +1723,11 @@ looper()
             gettimeofday(&current_time, nullptr);
             lnav_data.ld_input_dispatcher.poll(current_time);
 
+            if (lb.lb_last_view != nullptr) {
+                lb.lb_last_event.me_time = current_time;
+                lb.lb_last_view->handle_mouse(lb.lb_last_event);
+            }
+
             if (rc < 0) {
                 switch (errno) {
                     case 0:
@@ -1806,7 +1816,7 @@ looper()
                         case ln_mode_t::PAGING:
                         case ln_mode_t::FILTER:
                         case ln_mode_t::FILES:
-                            next_rescan_time = next_status_update_time + 1s;
+                            next_rescan_time = next_status_update_time;
                             next_rebuild_time = next_rescan_time;
                             break;
                         default:
@@ -1814,7 +1824,8 @@ looper()
                     }
                 }
                 if (old_file_names_size
-                    != lnav_data.ld_active_files.fc_file_names.size())
+                        != lnav_data.ld_active_files.fc_file_names.size()
+                    || lnav_data.ld_active_files.finished_pipers() > 0)
                 {
                     next_rescan_time = ui_clock::now();
                     next_rebuild_time = next_rescan_time;
@@ -1837,21 +1848,14 @@ looper()
                     timer.start_fade(index_counter, 3);
                 }
                 // log_debug("initial build rebuild");
-                changes += rebuild_indexes(loop_deadline);
-                if (!lnav_data.ld_initial_build
-                    && lnav_data.ld_log_source.text_line_count() == 0
-                    && lnav_data.ld_text_source.text_line_count() > 0)
-                {
-                    ensure_view(&lnav_data.ld_views[LNV_TEXT]);
-                    lnav_data.ld_rl_view->set_alt_value(HELP_MSG_2(
-                        f, F, "to switch to the next/previous file"));
-                }
+                auto rebuild_res = rebuild_indexes(loop_deadline);
+                changes += rebuild_res.rir_changes;
                 if (lnav_data.ld_view_stack.top().value_or(nullptr)
                         == &lnav_data.ld_views[LNV_TEXT]
                     && lnav_data.ld_text_source.empty()
                     && lnav_data.ld_log_source.text_line_count() > 0)
                 {
-                    textview_curses* tc_log = &lnav_data.ld_views[LNV_LOG];
+                    auto* tc_log = &lnav_data.ld_views[LNV_LOG];
                     lnav_data.ld_view_stack.pop_back();
 
                     lnav_data.ld_views[LNV_LOG].set_top(
@@ -1881,10 +1885,14 @@ looper()
                 {
                     lnav_data.ld_initial_build = true;
                 }
-                if (lnav_data.ld_log_source.text_line_count() > 0
-                    || lnav_data.ld_text_source.text_line_count() > 0
-                    || !lnav_data.ld_active_files.fc_other_files.empty())
+                if (rebuild_res.rir_completed
+                    && (lnav_data.ld_log_source.text_line_count() > 0
+                        || lnav_data.ld_text_source.text_line_count() > 0
+                        || lnav_data.ld_active_files.other_file_format_count(
+                               file_format_t::SQLITE_DB)
+                            > 0))
                 {
+                    log_debug("initial build completed");
                     lnav_data.ld_initial_build = true;
                 }
 
@@ -1915,52 +1923,46 @@ looper()
                         line_buffer::cleanup_cache();
                         archive_manager::cleanup_cache();
                         tailer::cleanup_cache();
+                        lnav::piper::cleanup();
+                        file_converter_manager::cleanup();
                         ran_cleanup = true;
                     }
                 }
 
-                if (session_stage == 1
+                if (session_stage == 1 && lnav_data.ld_initial_build
                     && (lnav_data.ld_active_files.fc_file_names.empty()
                         || lnav_data.ld_log_source.text_line_count() > 0
                         || lnav_data.ld_text_source.text_line_count() > 0
                         || !lnav_data.ld_active_files.fc_other_files.empty()))
                 {
-                    log_debug("restoring view states");
-                    for (size_t view_index = 0; view_index < LNV__MAX;
-                         view_index++)
-                    {
-                        const auto& vs
-                            = session_data.sd_view_states[view_index];
-                        auto& tview = lnav_data.ld_views[view_index];
-
-                        if (vs.vs_top >= 0
-                            && (view_index == LNV_LOG
-                                || tview.get_top() == 0_vl))
+                    lnav::session::restore_view_states();
+                    if (lnav_data.ld_mode == ln_mode_t::FILES) {
+                        if (lnav_data.ld_log_source.text_line_count() == 0
+                            && lnav_data.ld_text_source.text_line_count() > 0
+                            && lnav_data.ld_view_stack.size() == 1)
                         {
-                            log_info("restoring %s view top: %d",
-                                     lnav_view_strings[view_index],
-                                     vs.vs_top);
-                            lnav_data.ld_views[view_index].set_top(
-                                vis_line_t(vs.vs_top));
-                            if (vs.vs_selection) {
-                                lnav_data.ld_views[view_index].set_selection(
-                                    vis_line_t(vs.vs_selection.value()));
-                            }
+                            log_debug("no logs, just text...");
+                            ensure_view(&lnav_data.ld_views[LNV_TEXT]);
+                            lnav_data.ld_rl_view->set_alt_value(HELP_MSG_2(
+                                f, F, "to switch to the next/previous file"));
                         }
-                    }
-                    if (lnav_data.ld_mode == ln_mode_t::FILES) {
-                        if (lnav_data.ld_active_files.fc_name_to_errors.empty())
+                        if (lnav_data.ld_active_files.fc_name_to_errors
+                                ->readAccess()
+                                ->empty())
                         {
                             log_info("switching to paging!");
                             lnav_data.ld_mode = ln_mode_t::PAGING;
                             lnav_data.ld_active_files.fc_files
                                 | lnav::itertools::for_each(
                                     &logfile::dump_stats);
+
+                            check_for_file_zones();
                         } else {
                             lnav_data.ld_files_view.set_selection(0_vl);
                         }
                     }
                     session_stage += 1;
+                    lnav_data.ld_exec_phase = lnav_exec_phase::INTERACTIVE;
                     load_time_bookmarks();
                 }
             }
@@ -1986,6 +1988,10 @@ looper()
                 }
 
                 gather_pipers();
+
+                next_rescan_time = ui_clock::now();
+                next_rebuild_time = next_rescan_time;
+                next_status_update_time = next_rescan_time;
             }
 
             if (lnav_data.ld_view_stack.empty()
@@ -1996,6 +2002,48 @@ looper()
             {
                 lnav_data.ld_looping = false;
             }
+
+            if (lnav_data.ld_sigint_count > 0) {
+                bool found_piper = false;
+
+                lnav_data.ld_sigint_count = 0;
+                if (!lnav_data.ld_view_stack.empty()) {
+                    auto* tc = *lnav_data.ld_view_stack.top();
+
+                    if (tc->get_inner_height() > 0_vl) {
+                        std::vector<attr_line_t> rows(1);
+
+                        tc->get_data_source()->listview_value_for_rows(
+                            *tc, tc->get_selection(), rows);
+                        auto& sa = rows[0].get_attrs();
+                        auto line_attr_opt
+                            = get_string_attr(sa, logline::L_FILE);
+                        if (line_attr_opt) {
+                            auto lf = line_attr_opt.value().get();
+
+                            log_debug("file name when SIGINT: %s",
+                                      lf->get_filename().c_str());
+                            for (auto& cp : lnav_data.ld_child_pollers) {
+                                auto cp_name = cp.get_filename();
+
+                                if (!cp_name) {
+                                    log_debug("no child_poller");
+                                    continue;
+                                }
+
+                                if (lf->get_filename() == cp_name.value()) {
+                                    log_debug("found it, sending signal!");
+                                    cp.send_sigint();
+                                    found_piper = true;
+                                }
+                            }
+                        }
+                    }
+                }
+                if (!found_piper) {
+                    lnav_data.ld_looping = false;
+                }
+            }
         }
     } catch (readline_curses::error& e) {
         log_error("error: %s", strerror(e.e_err));
@@ -2100,39 +2148,28 @@ print_user_msgs(std::vector<lnav::console::user_message> error_list,
     return retval;
 }
 
-enum class verbosity_t : int {
-    quiet,
-    standard,
-    verbose,
-};
-
-struct stdin_options_t {
-    ghc::filesystem::path so_out;
-    bool so_timestamp{false};
-    auto_fd so_out_fd;
-};
+verbosity_t verbosity = verbosity_t::standard;
 
 int
 main(int argc, char* argv[])
 {
     std::vector<lnav::console::user_message> config_errors;
     std::vector<lnav::console::user_message> loader_errors;
-    exec_context& ec = lnav_data.ld_exec_context;
+    auto& ec = lnav_data.ld_exec_context;
     int retval = EXIT_SUCCESS;
 
-    std::shared_ptr<piper_proc> stdin_reader;
-    stdin_options_t stdin_opts;
-    bool exec_stdin = false, load_stdin = false, stdin_captured = false;
+    bool exec_stdin = false, load_stdin = false;
     mode_flags_t mode_flags;
     const char* LANG = getenv("LANG");
-    ghc::filesystem::path stdin_tmp_path;
-    verbosity_t verbosity = verbosity_t::standard;
 
     if (LANG == nullptr || strcmp(LANG, "C") == 0) {
         setenv("LANG", "en_US.UTF-8", 1);
     }
 
+    ec.ec_label_source_stack.push_back(&lnav_data.ld_db_row_source);
+
     (void) signal(SIGPIPE, SIG_IGN);
+    (void) signal(SIGCHLD, sigchld);
     setlocale(LC_ALL, "");
     try {
         std::locale::global(std::locale(""));
@@ -2148,6 +2185,58 @@ main(int argc, char* argv[])
         lnav_data.ld_flags |= LNF_SECURE_MODE;
     }
 
+    // Set PAGER so that stuff run from `:sh` will just dump their
+    // output for lnav to display.  One example would be `man`, as
+    // in `:sh man ls`.
+    setenv("PAGER", "cat", 1);
+    setenv("LNAV_HOME_DIR", lnav::paths::dotlnav().c_str(), 1);
+    setenv("LNAV_WORK_DIR", lnav::paths::workdir().c_str(), 1);
+
+    try {
+        auto& safe_options_hier
+            = injector::get<lnav::safe_file_options_hier&>();
+
+        auto opt_path = lnav::paths::dotlnav() / "file-options.json";
+        auto read_res = lnav::filesystem::read_file(opt_path);
+        auto curr_tz = date::get_tzdb().current_zone();
+        auto options_coll = lnav::file_options_collection{};
+
+        if (read_res.isOk()) {
+            intern_string_t opt_path_src = intern_string::lookup(opt_path);
+            auto parse_res = lnav::file_options_collection::from_json(
+                opt_path_src, read_res.unwrap());
+            if (parse_res.isErr()) {
+                for (const auto& um : parse_res.unwrapErr()) {
+                    lnav::console::print(stderr, um);
+                }
+                return EXIT_FAILURE;
+            }
+
+            options_coll = parse_res.unwrap();
+        }
+
+        safe::WriteAccess<lnav::safe_file_options_hier> options_hier(
+            safe_options_hier);
+
+        options_hier->foh_generation += 1;
+        auto_mem<char> var_path;
+
+        var_path = realpath("/var/log", nullptr);
+        options_coll.foc_pattern_to_options[fmt::format(FMT_STRING("{}/*"),
+                                                        var_path.in())]
+            = lnav::file_options{
+                {
+                    intern_string_t{},
+                    source_location{},
+                    curr_tz,
+                },
+            };
+        options_hier->foh_path_to_collection.emplace(ghc::filesystem::path("/"),
+                                                     options_coll);
+    } catch (const std::runtime_error& e) {
+        log_error("failed to setup tz: %s", e.what());
+    }
+
     lnav_data.ld_exec_context.ec_sql_callback = sql_callback;
     lnav_data.ld_exec_context.ec_pipe_callback = pipe_callback;
 
@@ -2204,6 +2293,8 @@ main(int argc, char* argv[])
 SELECT tbl_name FROM sqlite_master WHERE sql LIKE 'CREATE VIRTUAL TABLE%'
 )";
 
+        lnav_data.ld_child_pollers.clear();
+
         for (auto& lf : lnav_data.ld_active_files.fc_files) {
             lf->close();
         }
@@ -2287,6 +2378,21 @@ SELECT tbl_name FROM sqlite_master WHERE sql LIKE 'CREATE VIRTUAL TABLE%'
                    lnav_data.ld_debug_log_name,
                    "Write debug messages to the given file.")
         ->type_name("FILE");
+    app.add_option("-I", lnav_data.ld_config_paths, "include paths")
+        ->check(CLI::ExistingDirectory)
+        ->check([&arg_errors](std::string inc_path) -> std::string {
+            if (access(inc_path.c_str(), X_OK) != 0) {
+                arg_errors.emplace_back(
+                    lnav::console::user_message::error(
+                        attr_line_t("invalid configuration directory: ")
+                            .append(lnav::roles::file(inc_path)))
+                        .with_errno_reason());
+                return "unreadable";
+            }
+
+            return std::string();
+        })
+        ->allow_extra_args(false);
     app.add_flag("-q{0},-v{2}", verbosity, "Control the verbosity");
     app.set_version_flag("-V,--version");
     app.footer(fmt::format(FMT_STRING("Version: {}"), VCS_PACKAGE_STRING));
@@ -2295,34 +2401,18 @@ SELECT tbl_name FROM sqlite_master WHERE sql LIKE 'CREATE VIRTUAL TABLE%'
 
     if (argc < 2 || strcmp(argv[1], "-m") != 0) {
         app.add_flag("-H", lnav_data.ld_show_help_view, "show help");
-        app.add_option("-I", lnav_data.ld_config_paths, "include paths")
-            ->check(CLI::ExistingDirectory)
-            ->check([&arg_errors](std::string inc_path) -> std::string {
-                if (access(inc_path.c_str(), X_OK) != 0) {
-                    arg_errors.emplace_back(
-                        lnav::console::user_message::error(
-                            attr_line_t("invalid configuration directory: ")
-                                .append(lnav::roles::file(inc_path)))
-                            .with_errno_reason());
-                    return "unreadable";
-                }
-
-                return std::string();
-            })
-            ->allow_extra_args(false);
         app.add_flag("-C", mode_flags.mf_check_configs, "check");
         auto* install_flag
             = app.add_flag("-i", mode_flags.mf_install, "install");
         app.add_flag("-u", mode_flags.mf_update_formats, "update");
-        auto* write_flag = app.add_option("-w", stdin_opts.so_out, "write");
-        auto* ts_flag
-            = app.add_flag("-t", stdin_opts.so_timestamp, "timestamp");
         auto* no_default_flag
             = app.add_flag("-N", mode_flags.mf_no_default, "no def");
         auto* rotated_flag = app.add_flag(
             "-R", lnav_data.ld_active_files.fc_rotated, "rotated");
         auto* recurse_flag = app.add_flag(
             "-r", lnav_data.ld_active_files.fc_recursive, "recurse");
+        auto* as_log_flag
+            = app.add_flag("-t", lnav_data.ld_treat_stdin_as_log, "as-log");
         app.add_flag("-W", mode_flags.mf_print_warnings);
         auto* headless_flag = app.add_flag(
             "-n",
@@ -2391,15 +2481,25 @@ SELECT tbl_name FROM sqlite_master WHERE sql LIKE 'CREATE VIRTUAL TABLE%'
                                   ->allow_extra_args(false)
                                   ->each(file_appender);
 
+        auto shexec_appender = [&mode_flags](std::string cmd) {
+            mode_flags.mf_no_default = true;
+            lnav_data.ld_commands.emplace_back(
+                fmt::format(FMT_STRING(":sh {}"), cmd));
+        };
+        auto* cmdline_opt = app.add_option("-e")
+                                ->each(shexec_appender)
+                                ->allow_extra_args(false)
+                                ->trigger_on_parse(true);
+
         install_flag->needs(file_opt);
-        install_flag->excludes(write_flag,
-                               ts_flag,
-                               no_default_flag,
+        install_flag->excludes(no_default_flag,
+                               as_log_flag,
                                rotated_flag,
                                recurse_flag,
                                headless_flag,
                                cmd_opt,
-                               exec_file_opt);
+                               exec_file_opt,
+                               cmdline_opt);
     }
 
     auto is_mmode = argc >= 2 && strcmp(argv[1], "-m") == 0;
@@ -2411,13 +2511,13 @@ SELECT tbl_name FROM sqlite_master WHERE sql LIKE 'CREATE VIRTUAL TABLE%'
         }
     } catch (const CLI::CallForHelp& e) {
         if (is_mmode) {
-            fmt::print("{}\n", app.help());
+            fmt::print(FMT_STRING("{}\n"), app.help());
         } else {
             usage();
         }
         return EXIT_SUCCESS;
     } catch (const CLI::CallForVersion& e) {
-        fmt::print("{}\n", VCS_PACKAGE_STRING);
+        fmt::print(FMT_STRING("{}\n"), VCS_PACKAGE_STRING);
         return EXIT_SUCCESS;
     } catch (const CLI::ParseError& e) {
         if (!arg_errors.empty()) {
@@ -2444,7 +2544,9 @@ SELECT tbl_name FROM sqlite_master WHERE sql LIKE 'CREATE VIRTUAL TABLE%'
     }
 
     lnav_log_file = make_optional_from_nullable(
-        fopen(lnav_data.ld_debug_log_name.c_str(), "a"));
+        fopen(lnav_data.ld_debug_log_name.c_str(), "ae"));
+    lnav_log_file |
+        [](auto* file) { fcntl(fileno(file), F_SETFD, FD_CLOEXEC); };
     log_info("lnav started");
 
     {
@@ -2459,6 +2561,7 @@ SELECT tbl_name FROM sqlite_master WHERE sql LIKE 'CREATE VIRTUAL TABLE%'
     }
 
     load_config(lnav_data.ld_config_paths, config_errors);
+
     if (!config_errors.empty()) {
         if (print_user_msgs(config_errors, mode_flags) != EXIT_SUCCESS) {
             return EXIT_FAILURE;
@@ -2484,15 +2587,20 @@ SELECT tbl_name FROM sqlite_master WHERE sql LIKE 'CREATE VIRTUAL TABLE%'
                 = attr_line_t("the ")
                       .append("-i"_symbol)
                       .append(
-                          " option expects one or more log format definition "
-                          "files to install in your lnav configuration "
+                          " option expects one or more log format "
+                          "definition "
+                          "files to install in your lnav "
+                          "configuration "
                           "directory");
             const auto install_help
                 = attr_line_t(
-                      "log format definitions are JSON files that tell lnav "
+                      "log format definitions are JSON files that "
+                      "tell lnav "
                       "how to understand log files\n")
                       .append(
-                          "See: https://docs.lnav.org/en/latest/formats.html");
+                          "See: "
+                          "https://docs.lnav.org/en/latest/"
+                          "formats.html");
 
             lnav::console::print(stderr,
                                  lnav::console::user_message::error(
@@ -2502,7 +2610,7 @@ SELECT tbl_name FROM sqlite_master WHERE sql LIKE 'CREATE VIRTUAL TABLE%'
             return EXIT_FAILURE;
         }
 
-        for (auto& file_path : file_args) {
+        for (const auto& file_path : file_args) {
             if (endswith(file_path, ".git")) {
                 if (!install_from_git(file_path)) {
                     return EXIT_FAILURE;
@@ -2592,44 +2700,64 @@ SELECT tbl_name FROM sqlite_master WHERE sql LIKE 'CREATE VIRTUAL TABLE%'
                                  ? configs_installed_path
                                  : formats_installed_path)
                 / dst_name;
-            auto_fd in_fd, out_fd;
 
-            if ((in_fd = open(file_path.c_str(), O_RDONLY)) == -1) {
-                perror("unable to open file to install");
-            } else if ((out_fd = lnav::filesystem::openp(
-                            dst_path, O_WRONLY | O_CREAT | O_TRUNC, 0644))
-                       == -1)
-            {
-                fprintf(stderr,
-                        "error: unable to open destination: %s -- %s\n",
-                        dst_path.c_str(),
-                        strerror(errno));
-            } else {
-                char buffer[2048];
-                ssize_t rc;
-
-                while ((rc = read(in_fd, buffer, sizeof(buffer))) > 0) {
-                    ssize_t remaining = rc, written;
-
-                    while (remaining > 0) {
-                        written = write(out_fd, buffer, rc);
-                        if (written == -1) {
-                            fprintf(stderr,
-                                    "error: unable to install file -- %s\n",
-                                    strerror(errno));
-                            exit(EXIT_FAILURE);
-                        }
+            auto read_res = lnav::filesystem::read_file(file_path);
+            if (read_res.isErr()) {
+                auto um = lnav::console::user_message::error(
+                              attr_line_t("cannot read file to install -- ")
+                                  .append(lnav::roles::file(file_path)))
+                              .with_reason(read_res.unwrap());
 
-                        remaining -= written;
-                    }
+                lnav::console::print(stderr, um);
+                return EXIT_FAILURE;
+            }
+
+            auto file_content = read_res.unwrap();
+
+            auto read_dst_res = lnav::filesystem::read_file(dst_path);
+            if (read_dst_res.isOk()) {
+                auto dst_content = read_dst_res.unwrap();
+
+                if (dst_content == file_content) {
+                    auto um = lnav::console::user_message::info(
+                        attr_line_t("file is already installed at -- ")
+                            .append(lnav::roles::file(dst_path)));
+
+                    lnav::console::print(stdout, um);
+
+                    return EXIT_SUCCESS;
                 }
+            }
 
-                lnav::console::print(
-                    stderr,
-                    lnav::console::user_message::ok(
-                        attr_line_t("installed -- ")
-                            .append(lnav::roles::file(dst_path))));
+            auto write_res = lnav::filesystem::write_file(
+                dst_path,
+                file_content,
+                {lnav::filesystem::write_file_options::backup_existing});
+            if (write_res.isErr()) {
+                auto um = lnav::console::user_message::error(
+                              attr_line_t("failed to install file to -- ")
+                                  .append(lnav::roles::file(dst_path)))
+                              .with_reason(write_res.unwrapErr());
+
+                lnav::console::print(stderr, um);
+                return EXIT_FAILURE;
             }
+
+            auto write_file_res = write_res.unwrap();
+            auto um = lnav::console::user_message::ok(
+                attr_line_t("installed -- ")
+                    .append(lnav::roles::file(dst_path)));
+            if (write_file_res.wfr_backup_path) {
+                um.with_note(
+                    attr_line_t("the previously installed ")
+                        .append_quoted(
+                            lnav::roles::file(dst_path.filename().string()))
+                        .append(" was backed up to -- ")
+                        .append(lnav::roles::file(
+                            write_file_res.wfr_backup_path.value().string())));
+            }
+
+            lnav::console::print(stdout, um);
         }
         return EXIT_SUCCESS;
     }
@@ -2644,9 +2772,9 @@ SELECT tbl_name FROM sqlite_master WHERE sql LIKE 'CREATE VIRTUAL TABLE%'
         }
     }
 
-    /* If we statically linked against an ncurses library that had a non-
-     * standard path to the terminfo database, we need to set this variable
-     * so that it will try the default path.
+    /* If we statically linked against an ncurses library that had a
+     * non-standard path to the terminfo database, we need to set this
+     * variable so that it will try the default path.
      */
     setenv("TERMINFO_DIRS",
            "/usr/share/terminfo:/lib/terminfo:/usr/share/lib/terminfo",
@@ -2662,12 +2790,10 @@ SELECT tbl_name FROM sqlite_master WHERE sql LIKE 'CREATE VIRTUAL TABLE%'
         .set_word_wrap(false);
     auto log_fos = new field_overlay_source(lnav_data.ld_log_source,
                                             lnav_data.ld_text_source);
-    if (lnav_data.ld_flags & LNF_HEADLESS) {
-        log_fos->fos_show_status = false;
-    }
-    log_fos->fos_contexts.emplace("", false, true);
+    log_fos->fos_contexts.emplace("", false, true, true);
     lnav_data.ld_views[LNV_LOG]
         .set_sub_source(&lnav_data.ld_log_source)
+#if 0
         .set_delegate(std::make_shared<action_delegate>(
             lnav_data.ld_log_source,
             [](auto child_pid) { lnav_data.ld_children.push_back(child_pid); },
@@ -2677,17 +2803,23 @@ SELECT tbl_name FROM sqlite_master WHERE sql LIKE 'CREATE VIRTUAL TABLE%'
                     pp->get_fd());
                 lnav_data.ld_files_to_front.template emplace_back(desc, 0_vl);
             }))
+#endif
         .add_input_delegate(lnav_data.ld_log_source)
         .set_tail_space(2_vl)
         .set_overlay_source(log_fos);
     auto sel_reload_delegate = [](textview_curses& tc) {
-        if (lnav_config.lc_ui_movement.mode == config_movement_mode::CURSOR) {
+        if (!(lnav_data.ld_flags & LNF_HEADLESS)
+            && lnav_config.lc_ui_movement.mode == config_movement_mode::CURSOR)
+        {
             tc.set_selectable(true);
         }
     };
     lnav_data.ld_views[LNV_LOG].set_reload_config_delegate(sel_reload_delegate);
     lnav_data.ld_views[LNV_PRETTY].set_reload_config_delegate(
         sel_reload_delegate);
+    auto text_header_source
+        = std::make_shared<textfile_header_overlay>(&lnav_data.ld_text_source);
+    lnav_data.ld_views[LNV_TEXT].set_overlay_source(text_header_source.get());
     lnav_data.ld_views[LNV_TEXT].set_sub_source(&lnav_data.ld_text_source);
     lnav_data.ld_views[LNV_TEXT].set_reload_config_delegate(
         sel_reload_delegate);
@@ -2696,6 +2828,10 @@ SELECT tbl_name FROM sqlite_master WHERE sql LIKE 'CREATE VIRTUAL TABLE%'
         .set_sub_source(&lnav_data.ld_hist_source2);
     lnav_data.ld_views[LNV_DB].set_sub_source(&lnav_data.ld_db_row_source);
     lnav_data.ld_db_overlay.dos_labels = &lnav_data.ld_db_row_source;
+    lnav_data.ld_db_preview_overlay_source[0].dos_labels
+        = &lnav_data.ld_db_preview_source[0];
+    lnav_data.ld_db_preview_overlay_source[1].dos_labels
+        = &lnav_data.ld_db_preview_source[1];
     lnav_data.ld_views[LNV_DB]
         .set_reload_config_delegate(sel_reload_delegate)
         .set_overlay_source(&lnav_data.ld_db_overlay);
@@ -2707,15 +2843,32 @@ SELECT tbl_name FROM sqlite_master WHERE sql LIKE 'CREATE VIRTUAL TABLE%'
         .add_input_delegate(*lnav_data.ld_spectro_source)
         .set_tail_space(4_vl);
     lnav_data.ld_views[LNV_SPECTRO].set_selectable(true);
+    auto gantt_view_source
+        = std::make_shared<gantt_source>(lnav_data.ld_views[LNV_LOG],
+                                         lnav_data.ld_log_source,
+                                         lnav_data.ld_gantt_details_source,
+                                         lnav_data.ld_gantt_status_source);
+    gantt_view_source->gs_exec_context = &lnav_data.ld_exec_context;
+    auto gantt_header_source
+        = std::make_shared<gantt_header_overlay>(gantt_view_source);
+    lnav_data.ld_views[LNV_GANTT]
+        .set_sub_source(gantt_view_source.get())
+        .set_overlay_source(gantt_header_source.get())
+        .set_tail_space(4_vl);
+    lnav_data.ld_views[LNV_GANTT].set_selectable(true);
+
+    auto _gantt_cleanup = finally([] {
+        lnav_data.ld_views[LNV_GANTT].set_sub_source(nullptr);
+        lnav_data.ld_views[LNV_GANTT].set_overlay_source(nullptr);
+    });
 
     lnav_data.ld_doc_view.set_sub_source(&lnav_data.ld_doc_source);
     lnav_data.ld_example_view.set_sub_source(&lnav_data.ld_example_source);
     lnav_data.ld_match_view.set_sub_source(&lnav_data.ld_match_source);
-    lnav_data.ld_preview_view.set_sub_source(&lnav_data.ld_preview_source);
+    lnav_data.ld_preview_view[0].set_sub_source(
+        &lnav_data.ld_preview_source[0]);
     lnav_data.ld_filter_view.set_sub_source(filter_source)
-        .add_input_delegate(*filter_source)
-        .add_child_view(&filter_source->fss_match_view)
-        .add_child_view(filter_source->fss_editor.get());
+        .add_input_delegate(*filter_source);
     lnav_data.ld_files_view.set_sub_source(&lnav_data.ld_files_source)
         .add_input_delegate(lnav_data.ld_files_source);
     lnav_data.ld_user_message_view.set_sub_source(
@@ -2761,6 +2914,9 @@ SELECT tbl_name FROM sqlite_master WHERE sql LIKE 'CREATE VIRTUAL TABLE%'
     lnav_data.ld_vtab_manager->register_vtab(
         std::make_shared<log_format_vtab_impl>(
             *log_format::find_root_format("generic_log")));
+    lnav_data.ld_vtab_manager->register_vtab(
+        std::make_shared<log_format_vtab_impl>(
+            *log_format::find_root_format("lnav_piper_log")));
 
     for (auto& iter : log_format::get_root_formats()) {
         auto lvi = iter->get_vtab_impl();
@@ -2803,6 +2959,7 @@ SELECT tbl_name FROM sqlite_master WHERE sql LIKE 'CREATE VIRTUAL TABLE%'
     lnav_data.ld_mode = ln_mode_t::PAGING;
 
     if ((isatty(STDIN_FILENO) || is_dev_null(STDIN_FILENO)) && file_args.empty()
+        && lnav_data.ld_active_files.fc_file_names.empty()
         && !mode_flags.mf_no_default)
     {
         char start_dir[FILENAME_MAX];
@@ -2837,56 +2994,63 @@ SELECT tbl_name FROM sqlite_master WHERE sql LIKE 'CREATE VIRTUAL TABLE%'
         load_stdin = true;
     }
 
-    for (auto& file_path : file_args) {
-        auto file_path_without_trailer = file_path;
+    for (const auto& file_path_str : file_args) {
+        auto file_path_without_trailer = file_path_str;
         auto file_loc = file_location_t{mapbox::util::no_init{}};
         auto_mem<char> abspath;
         struct stat st;
 
-        auto colon_index = file_path.rfind(':');
+        auto colon_index = file_path_str.rfind(':');
         if (colon_index != std::string::npos) {
-            auto top_range = scn::string_view{&file_path[colon_index + 1],
-                                              &(*file_path.cend())};
+            auto top_range = scn::string_view{&file_path_str[colon_index + 1],
+                                              &(*file_path_str.cend())};
             auto scan_res = scn::scan_value<int>(top_range);
 
             if (scan_res) {
-                file_path_without_trailer = file_path.substr(0, colon_index);
+                file_path_without_trailer
+                    = file_path_str.substr(0, colon_index);
                 file_loc = vis_line_t(scan_res.value());
             } else {
                 log_warning(
-                    "failed to parse line number from file path with colon: %s",
-                    file_path.c_str());
+                    "failed to parse line number from file path "
+                    "with colon: %s",
+                    file_path_str.c_str());
             }
         }
-        auto hash_index = file_path.rfind('#');
+        auto hash_index = file_path_str.rfind('#');
         if (hash_index != std::string::npos) {
-            file_loc = file_path.substr(hash_index);
-            file_path_without_trailer = file_path.substr(0, hash_index);
-        }
-        if (stat(file_path_without_trailer.c_str(), &st) == 0) {
-            file_path = file_path_without_trailer;
+            file_loc = file_path_str.substr(hash_index);
+            file_path_without_trailer = file_path_str.substr(0, hash_index);
         }
+        auto file_path = ghc::filesystem::path(
+            stat(file_path_without_trailer.c_str(), &st) == 0
+                ? file_path_without_trailer
+                : file_path_str);
 
-        if (file_path == "-") {
+        if (file_path_str == "-") {
             load_stdin = true;
         }
 #ifdef HAVE_LIBCURL
-        else if (is_url(file_path))
+        else if (is_url(file_path_str))
         {
-            auto ul = std::make_shared<url_loader>(file_path);
+            auto ul = std::make_shared<url_loader>(file_path_str);
 
-            lnav_data.ld_active_files.fc_file_names[file_path].with_fd(
-                ul->copy_fd());
+            lnav_data.ld_active_files.fc_file_names[ul->get_path()]
+                .with_filename(file_path);
             isc::to<curl_looper&, services::curl_streamer_t>().send(
                 [ul](auto& clooper) { clooper.add_request(ul); });
+        } else if (file_path_str.find("://") != std::string::npos) {
+            lnav_data.ld_commands.insert(
+                lnav_data.ld_commands.begin(),
+                fmt::format(FMT_STRING(":open {}"), file_path_str));
         }
 #endif
-        else if (is_glob(file_path))
+        else if (lnav::filesystem::is_glob(file_path))
         {
             lnav_data.ld_active_files.fc_file_names[file_path].with_tail(
                 !(lnav_data.ld_flags & LNF_HEADLESS));
-        } else if (stat(file_path.c_str(), &st) == -1) {
-            if (file_path.find(':') != std::string::npos) {
+        } else if (lnav::filesystem::statp(file_path, &st) == -1) {
+            if (file_path_str.find(':') != std::string::npos) {
                 lnav_data.ld_active_files.fc_file_names[file_path].with_tail(
                     !(lnav_data.ld_flags & LNF_HEADLESS));
             } else {
@@ -2909,7 +3073,8 @@ SELECT tbl_name FROM sqlite_master WHERE sql LIKE 'CREATE VIRTUAL TABLE%'
         } else if (S_ISFIFO(st.st_mode)) {
             auto_fd fifo_fd;
 
-            if ((fifo_fd = open(file_path.c_str(), O_RDONLY)) == -1) {
+            if ((fifo_fd = lnav::filesystem::openp(file_path, O_RDONLY)) == -1)
+            {
                 lnav::console::print(
                     stderr,
                     lnav::console::user_message::error(
@@ -2918,25 +3083,15 @@ SELECT tbl_name FROM sqlite_master WHERE sql LIKE 'CREATE VIRTUAL TABLE%'
                         .with_errno_reason());
                 retval = EXIT_FAILURE;
             } else {
-                auto fifo_tmp_fd
-                    = lnav::filesystem::open_temp_file(
-                          ghc::filesystem::temp_directory_path()
-                          / "lnav.fifo.XXXXXX")
-                          .map([](auto&& pair) {
-                              ghc::filesystem::remove(pair.first);
-
-                              return std::move(pair.second);
-                          })
-                          .expect("Cannot create temporary file for FIFO");
-                auto fifo_piper = std::make_shared<piper_proc>(
-                    std::move(fifo_fd), false, std::move(fifo_tmp_fd));
-                auto fifo_out_fd = fifo_piper->get_fd();
                 auto desc = fmt::format(FMT_STRING("FIFO [{}]"),
                                         lnav_data.ld_fifo_counter++);
+                auto create_piper_res = lnav::piper::create_looper(
+                    desc, std::move(fifo_fd), auto_fd{});
 
-                lnav_data.ld_active_files.fc_file_names[desc].with_fd(
-                    std::move(fifo_out_fd));
-                lnav_data.ld_pipers.push_back(fifo_piper);
+                if (create_piper_res.isOk()) {
+                    lnav_data.ld_active_files.fc_file_names[desc].with_piper(
+                        create_piper_res.unwrap());
+                }
             }
         } else if ((abspath = realpath(file_path.c_str(), nullptr)) == nullptr)
         {
@@ -2952,7 +3107,8 @@ SELECT tbl_name FROM sqlite_master WHERE sql LIKE 'CREATE VIRTUAL TABLE%'
                 dir_wild + "/*", logfile_open_options());
         } else {
             lnav_data.ld_active_files.fc_file_names.emplace(
-                abspath.in(), logfile_open_options());
+                abspath.in(),
+                logfile_open_options().with_init_location(file_loc));
             if (file_loc.valid()) {
                 lnav_data.ld_files_to_front.emplace_back(abspath.in(),
                                                          file_loc);
@@ -2997,7 +3153,8 @@ SELECT tbl_name FROM sqlite_master WHERE sql LIKE 'CREATE VIRTUAL TABLE%'
                     std::string partial_line(sbr.get_data(), partial_len);
 
                     fprintf(stderr,
-                            "error:%s:%ld:line did not match format %s\n",
+                            "error:%s:%ld:line did not match format "
+                            "%s\n",
                             lf->get_filename().c_str(),
                             line_number,
                             fmt->get_pattern_path(line_number).c_str());
@@ -3039,44 +3196,66 @@ SELECT tbl_name FROM sqlite_master WHERE sql LIKE 'CREATE VIRTUAL TABLE%'
         retval = EXIT_FAILURE;
     }
 
+    nonstd::optional<std::string> stdin_url;
+    ghc::filesystem::path stdin_dir;
     if (load_stdin && !isatty(STDIN_FILENO) && !is_dev_null(STDIN_FILENO)
         && !exec_stdin)
     {
-        if (stdin_opts.so_out.empty()) {
-            auto pattern
-                = lnav::paths::dotlnav() / "stdin-captures/stdin.XXXXXX";
+        static const std::string STDIN_NAME = "stdin";
+        struct stat stdin_st;
 
-            auto open_result = lnav::filesystem::open_temp_file(pattern);
-            if (open_result.isErr()) {
-                fprintf(stderr,
-                        "Unable to open temporary file for stdin: %s",
-                        open_result.unwrapErr().c_str());
-                return EXIT_FAILURE;
+        if (fstat(STDIN_FILENO, &stdin_st) == -1) {
+            lnav::console::print(
+                stderr,
+                lnav::console::user_message::error("unable to stat() stdin")
+                    .with_errno_reason());
+            retval = EXIT_FAILURE;
+        } else if (S_ISFIFO(stdin_st.st_mode)) {
+            struct pollfd pfd[1];
+
+            pfd[0].fd = STDIN_FILENO;
+            pfd[0].events = POLLIN;
+            pfd[0].revents = 0;
+            auto prc = poll(pfd, 1, 0);
+
+            if (prc == 0 || (pfd[0].revents & POLLIN)) {
+                auto stdin_piper_res = lnav::piper::create_looper(
+                    STDIN_NAME, auto_fd::dup_of(STDIN_FILENO), auto_fd{});
+                if (stdin_piper_res.isOk()) {
+                    auto stdin_piper = stdin_piper_res.unwrap();
+                    stdin_url = stdin_piper.get_url();
+                    stdin_dir = stdin_piper.get_out_dir();
+                    auto& loo = lnav_data.ld_active_files
+                                    .fc_file_names[stdin_piper.get_name()];
+                    loo.with_piper(stdin_piper).with_include_in_session(false);
+                    if (lnav_data.ld_treat_stdin_as_log) {
+                        loo.with_text_format(text_format_t::TF_LOG);
+                    }
+                }
             }
+        } else if (S_ISREG(stdin_st.st_mode)) {
+            // The shell connected a file directly, just open it up
+            // and add it in here.
+            auto loo = logfile_open_options{}
+                           .with_filename(STDIN_NAME)
+                           .with_include_in_session(false);
+
+            auto open_res
+                = logfile::open(STDIN_NAME, loo, auto_fd::dup_of(STDIN_FILENO));
 
-            auto temp_pair = open_result.unwrap();
-            stdin_tmp_path = temp_pair.first;
-            stdin_opts.so_out_fd = std::move(temp_pair.second);
-        } else {
-            auto open_res = lnav::filesystem::create_file(
-                stdin_opts.so_out, O_RDWR | O_TRUNC, 0600);
             if (open_res.isErr()) {
-                fmt::print(stderr, "error: {}\n", open_res.unwrapErr());
-                return EXIT_FAILURE;
-            }
+                lnav::console::print(
+                    stderr,
+                    lnav::console::user_message::error("unable to open stdin")
+                        .with_reason(open_res.unwrapErr()));
+                retval = EXIT_FAILURE;
+            } else {
+                file_collection fc;
 
-            stdin_opts.so_out_fd = open_res.unwrap();
+                fc.fc_files.emplace_back(open_res.unwrap());
+                update_active_files(fc);
+            }
         }
-
-        stdin_captured = true;
-        stdin_reader
-            = std::make_shared<piper_proc>(auto_fd(STDIN_FILENO),
-                                           stdin_opts.so_timestamp,
-                                           std::move(stdin_opts.so_out_fd));
-        lnav_data.ld_active_files.fc_file_names["stdin"]
-            .with_fd(stdin_reader->get_fd())
-            .with_include_in_session(false);
-        lnav_data.ld_pipers.push_back(stdin_reader);
     }
 
     if (!isatty(STDIN_FILENO) && isatty(STDOUT_FILENO)) {
@@ -3085,7 +3264,7 @@ SELECT tbl_name FROM sqlite_master WHERE sql LIKE 'CREATE VIRTUAL TABLE%'
         }
     }
 
-    if (retval == EXIT_SUCCESS
+    if (retval == EXIT_SUCCESS && lnav_data.ld_active_files.fc_files.empty()
         && lnav_data.ld_active_files.fc_file_names.empty()
         && lnav_data.ld_commands.empty()
         && !(lnav_data.ld_show_help_view || mode_flags.mf_no_default))
@@ -3094,11 +3273,10 @@ SELECT tbl_name FROM sqlite_master WHERE sql LIKE 'CREATE VIRTUAL TABLE%'
             stderr,
             lnav::console::user_message::error("nothing to do")
                 .with_reason("no files given or default files found")
-                .with_help(
-                    attr_line_t("use the ")
-                        .append_quoted(lnav::roles::keyword("-N"))
-                        .append(
-                            " option to open lnav without loading any files")));
+                .with_help(attr_line_t("use the ")
+                               .append_quoted(lnav::roles::keyword("-N"))
+                               .append(" option to open lnav without "
+                                       "loading any files")));
         retval = EXIT_FAILURE;
     }
 
@@ -3145,6 +3323,26 @@ SELECT tbl_name FROM sqlite_master WHERE sql LIKE 'CREATE VIRTUAL TABLE%'
                 log_info("    %s", file_iter->first.c_str());
             }
 
+            if (!(lnav_data.ld_flags & LNF_HEADLESS)
+                && verbosity == verbosity_t::quiet && load_stdin
+                && lnav_data.ld_active_files.fc_file_names.size() == 1)
+            {
+                rescan_files(true);
+                gather_pipers();
+                auto rebuild_res = rebuild_indexes(ui_clock::now() + 15ms);
+                if (rebuild_res.rir_completed
+                    && lnav_data.ld_child_pollers.empty())
+                {
+                    rebuild_indexes_repeatedly();
+                    if (lnav_data.ld_active_files.fc_files.empty()
+                        || lnav_data.ld_active_files.fc_files[0]->size() < 24)
+                    {
+                        lnav_data.ld_flags |= LNF_HEADLESS;
+                        verbosity = verbosity_t::standard;
+                    }
+                }
+            }
+
             if (lnav_data.ld_flags & LNF_HEADLESS) {
                 std::vector<
                     std::pair<Result<std::string, lnav::console::user_message>,
@@ -3153,21 +3351,29 @@ SELECT tbl_name FROM sqlite_master WHERE sql LIKE 'CREATE VIRTUAL TABLE%'
                 textview_curses *log_tc, *text_tc, *tc;
                 bool output_view = true;
 
+                log_fos->fos_contexts.top().c_show_applicable_annotations
+                    = false;
+
                 view_colors::init(true);
                 rescan_files(true);
-                if (!lnav_data.ld_active_files.fc_name_to_errors.empty()) {
-                    for (const auto& pair :
-                         lnav_data.ld_active_files.fc_name_to_errors)
-                    {
-                        lnav::console::print(
-                            stderr,
-                            lnav::console::user_message::error(
-                                attr_line_t("unable to open file: ")
-                                    .append(lnav::roles::file(pair.first)))
-                                .with_reason(pair.second.fei_description));
-                    }
+                wait_for_pipers();
+                rescan_files(true);
+                rebuild_indexes_repeatedly();
+                {
+                    safe::WriteAccess<safe_name_to_errors> errs(
+                        *lnav_data.ld_active_files.fc_name_to_errors);
+                    if (!errs->empty()) {
+                        for (const auto& pair : *errs) {
+                            lnav::console::print(
+                                stderr,
+                                lnav::console::user_message::error(
+                                    attr_line_t("unable to open file: ")
+                                        .append(lnav::roles::file(pair.first)))
+                                    .with_reason(pair.second.fei_description));
+                        }
 
-                    return EXIT_FAILURE;
+                        return EXIT_FAILURE;
+                    }
                 }
                 init_session();
                 lnav_data.ld_exec_context.set_output("stdout", stdout, nullptr);
@@ -3197,23 +3403,30 @@ SELECT tbl_name FROM sqlite_master WHERE sql LIKE 'CREATE VIRTUAL TABLE%'
                 archive_manager::cleanup_cache();
                 tailer::cleanup_cache();
                 line_buffer::cleanup_cache();
+                lnav::piper::cleanup();
+                file_converter_manager::cleanup();
                 wait_for_pipers();
+                rescan_files(true);
                 isc::to<curl_looper&, services::curl_streamer_t>()
                     .send_and_wait(
                         [](auto& clooper) { clooper.process_all(); });
                 rebuild_indexes_repeatedly();
                 wait_for_children();
-                if (!lnav_data.ld_active_files.fc_name_to_errors.empty()) {
-                    for (const auto& pair :
-                         lnav_data.ld_active_files.fc_name_to_errors)
-                    {
-                        fprintf(stderr,
-                                "error: unable to open file: %s -- %s\n",
-                                pair.first.c_str(),
-                                pair.second.fei_description.c_str());
-                    }
+                {
+                    safe::WriteAccess<safe_name_to_errors> errs(
+                        *lnav_data.ld_active_files.fc_name_to_errors);
+                    if (!errs->empty()) {
+                        for (const auto& pair : *errs) {
+                            lnav::console::print(
+                                stderr,
+                                lnav::console::user_message::error(
+                                    attr_line_t("unable to open file: ")
+                                        .append(lnav::roles::file(pair.first)))
+                                    .with_reason(pair.second.fei_description));
+                        }
 
-                    return EXIT_FAILURE;
+                        return EXIT_FAILURE;
+                    }
                 }
 
                 for (const auto& lf : lnav_data.ld_active_files.fc_files) {
@@ -3270,21 +3483,19 @@ SELECT tbl_name FROM sqlite_master WHERE sql LIKE 'CREATE VIRTUAL TABLE%'
                     }
 
                     auto* los = tc->get_overlay_source();
+                    attr_line_t ov_al;
+                    while (los != nullptr && tc->get_inner_height() > 0_vl
+                           && los->list_static_overlay(
+                               *tc, y, tc->get_inner_height(), ov_al))
+                    {
+                        write_line_to(stdout, ov_al);
+                        ov_al.clear();
+                        ++y;
+                    }
 
                     vis_line_t vl;
-                    for (vl = tc->get_top(); vl < tc->get_inner_height();
-                         ++vl, ++y)
+                    for (vl = tc->get_top(); vl < tc->get_inner_height(); ++vl)
                     {
-                        attr_line_t al;
-
-                        while (los != nullptr
-                               && los->list_value_for_overlay(
-                                   *tc, y, tc->get_inner_height(), vl, al))
-                        {
-                            write_line_to(stdout, al);
-                            ++y;
-                        }
-
                         std::vector<attr_line_t> rows(1);
                         tc->listview_value_for_rows(*tc, vl, rows);
                         if (suppress_empty_lines && rows[0].empty()) {
@@ -3292,17 +3503,14 @@ SELECT tbl_name FROM sqlite_master WHERE sql LIKE 'CREATE VIRTUAL TABLE%'
                         }
 
                         write_line_to(stdout, rows[0]);
-                    }
-                    {
-                        attr_line_t al;
 
-                        while (los != nullptr
-                               && los->list_value_for_overlay(
-                                   *tc, y, tc->get_inner_height(), vl, al)
-                               && !al.empty())
-                        {
-                            write_line_to(stdout, al);
-                            ++y;
+                        std::vector<attr_line_t> row_overlay_content;
+                        if (los != nullptr) {
+                            los->list_value_for_overlay(
+                                *tc, vl, row_overlay_content);
+                            for (const auto& ov_row : row_overlay_content) {
+                                write_line_to(stdout, ov_row);
+                            }
                         }
                     }
                 }
@@ -3330,56 +3538,30 @@ SELECT tbl_name FROM sqlite_master WHERE sql LIKE 'CREATE VIRTUAL TABLE%'
             fprintf(stderr, "error: %s\n", e.what());
         }
 
-        // When reading from stdin, tell the user where the capture file is
-        // stored so they can look at it later.
-        if (stdin_captured && stdin_opts.so_out.empty()
-            && !(lnav_data.ld_flags & LNF_HEADLESS))
+        // When reading from stdin, tell the user where the capture
+        // file is stored so they can look at it later.
+        if (stdin_url && !(lnav_data.ld_flags & LNF_HEADLESS)
+            && verbosity != verbosity_t::quiet)
         {
-            auto stdin_fd = stdin_reader->get_fd();
-            struct stat stdin_stat;
-            nonstd::optional<file_ssize_t> stdin_size;
-
-            // NB: the file can be deleted by the time we get here
-            fchmod(stdin_fd.get(), S_IRUSR);
-            if (fstat(stdin_fd.get(), &stdin_stat) != -1) {
-                stdin_size = stdin_stat.st_size;
-            }
-            if (!ghc::filesystem::exists(stdin_tmp_path)
-                || verbosity == verbosity_t::quiet || !stdin_size
-                || stdin_size.value() == 0
-                || stdin_size.value() > MAX_STDIN_CAPTURE_SIZE)
+            file_size_t stdin_size = 0;
+            for (const auto& ent :
+                 ghc::filesystem::directory_iterator(stdin_dir))
             {
-                std::error_code rm_err_code;
-
-                log_info("not saving stdin capture -- %s (size=%d)",
-                         stdin_tmp_path.c_str(),
-                         stdin_size.value_or(-1));
-                ghc::filesystem::remove(stdin_tmp_path, rm_err_code);
-            } else {
-                auto home = getenv_opt("HOME");
-                auto path_str = stdin_tmp_path.string();
-
-                if (home && startswith(path_str, home.value())) {
-                    path_str = path_str.substr(strlen(home.value()));
-                    if (path_str[0] != '/') {
-                        path_str.insert(0, 1, '/');
-                    }
-                    path_str.insert(0, 1, '~');
-                }
-
-                lnav::console::print(
-                    stderr,
-                    lnav::console::user_message::info(
-                        attr_line_t()
-                            .append(lnav::roles::number(humanize::file_size(
-                                stdin_size.value(), humanize::alignment::none)))
-                            .append(" of data from stdin was captured and "
-                                    "will be saved for one day.  You can "
-                                    "reopen it by running:\n")
-                            .appendf(FMT_STRING("   {} "),
-                                     lnav_data.ld_program_name)
-                            .append(lnav::roles::file(path_str))));
+                stdin_size += ent.file_size();
             }
+
+            lnav::console::print(
+                stderr,
+                lnav::console::user_message::info(
+                    attr_line_t()
+                        .append(lnav::roles::number(humanize::file_size(
+                            stdin_size, humanize::alignment::none)))
+                        .append(" of data from stdin was captured and "
+                                "will be saved for one day.  You can "
+                                "reopen it by running:\n")
+                        .appendf(FMT_STRING("   {} "),
+                                 lnav_data.ld_program_name)
+                        .append(lnav::roles::file(stdin_url.value()))));
         }
     }