summaryrefslogtreecommitdiffstats
path: root/docs/source
diff options
context:
space:
mode:
Diffstat (limited to 'docs/source')
-rw-r--r--docs/source/_ext/__init__.py0
-rw-r--r--docs/source/_ext/lnavlexer.py26
-rw-r--r--docs/source/_static/theme_overrides.css56
-rw-r--r--docs/source/cli.rst168
-rw-r--r--docs/source/commands.rst132
-rw-r--r--docs/source/conf.py460
-rw-r--r--docs/source/config.rst270
-rw-r--r--docs/source/cookbook.rst104
-rw-r--r--docs/source/data.rst193
-rw-r--r--docs/source/docutils.conf2
-rw-r--r--docs/source/events.rst56
-rw-r--r--docs/source/faq.rst57
-rw-r--r--docs/source/filter-out-preview.pngbin0 -> 690117 bytes
-rw-r--r--docs/source/formats.rst527
-rw-r--r--docs/source/group_concat-help.pngbin0 -> 60071 bytes
-rw-r--r--docs/source/hotkey-tips.pngbin0 -> 62825 bytes
-rw-r--r--docs/source/hotkeys.rst289
-rw-r--r--docs/source/howitworks.rst13
-rw-r--r--docs/source/index.rst35
-rw-r--r--docs/source/intro.rst131
-rw-r--r--docs/source/key-encoding-prompt.pngbin0 -> 88912 bytes
-rw-r--r--docs/source/lnav-breadcrumbs-help.pngbin0 -> 45782 bytes
-rw-r--r--docs/source/lnav-config-header.pngbin0 -> 35192 bytes
-rw-r--r--docs/source/lnav-files-panel.pngbin0 -> 70248 bytes
-rw-r--r--docs/source/lnav-filters-panel.pngbin0 -> 48867 bytes
-rw-r--r--docs/source/lnav-spectro-cpu-pct.pngbin0 -> 558558 bytes
-rw-r--r--docs/source/lnav-ui.pngbin0 -> 923959 bytes
-rw-r--r--docs/source/open-error.pngbin0 -> 21053 bytes
-rw-r--r--docs/source/open-help.pngbin0 -> 98785 bytes
-rw-r--r--docs/source/open-preview.pngbin0 -> 53409 bytes
-rw-r--r--docs/source/query-results.pngbin0 -> 35595 bytes
-rw-r--r--docs/source/sessions.rst23
-rw-r--r--docs/source/sql-help.pngbin0 -> 29721 bytes
-rw-r--r--docs/source/sqlext.rst184
-rw-r--r--docs/source/sqltab.rst217
-rw-r--r--docs/source/ui.rst280
-rw-r--r--docs/source/usage.rst291
37 files changed, 3514 insertions, 0 deletions
diff --git a/docs/source/_ext/__init__.py b/docs/source/_ext/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/docs/source/_ext/__init__.py
diff --git a/docs/source/_ext/lnavlexer.py b/docs/source/_ext/lnavlexer.py
new file mode 100644
index 0000000..2509a5f
--- /dev/null
+++ b/docs/source/_ext/lnavlexer.py
@@ -0,0 +1,26 @@
+
+__all__ = ['LnavCommandLexer']
+
+import re
+
+from pygments.token import Whitespace, Text, Keyword, Literal
+from pygments.lexers._mapping import LEXERS
+from pygments.lexers.python import RegexLexer
+
+class LnavCommandLexer(RegexLexer):
+ name = 'lnav'
+
+ flags = re.IGNORECASE
+ tokens = {
+ 'root': [
+ (r'\s+', Whitespace),
+ (r':[\w\-]+', Keyword),
+ (r'\<[\w\-]+\>', Literal.String.Doc),
+ (r'.', Text),
+ ]
+ }
+
+def setup(app):
+ LEXERS['LnavCommandLexer'] = (
+ '_ext.lnavlexer', 'lnav', ('lnav',), ('*.lnav',), ('text/lnav',))
+ app.add_lexer('lnav', LnavCommandLexer)
diff --git a/docs/source/_static/theme_overrides.css b/docs/source/_static/theme_overrides.css
new file mode 100644
index 0000000..4fc1c9f
--- /dev/null
+++ b/docs/source/_static/theme_overrides.css
@@ -0,0 +1,56 @@
+/* override table width restrictions */
+@media screen and (min-width: 767px) {
+
+ .wy-table-responsive table td {
+ /* !important prevents the common CSS stylesheets from overriding
+ this as on RTD they are loaded after this stylesheet */
+ white-space: normal !important;
+ }
+
+}
+
+th p {
+ margin-bottom: 0px;
+}
+
+.wy-nav-content {
+ max-width: 900px;
+}
+
+table.query-results p {
+ font-size: 0.9em !important;
+}
+
+DL DT:target, :target > H2, :target > H3, span:target + H2, span:target + H3 {
+ border: 0 !important;
+ border-bottom: 1px solid #d3d381 !important;
+ background: #ffc !important;
+ /* padding: 1em !important;*/
+}
+
+DL DT {
+ border: 0 !important;
+ background: inherit !important;
+ border-bottom: 1px solid #c3c0ee !important;
+ display: block !important;
+}
+
+kbd {
+ padding: 0.1em 0.6em;
+ border: 1px solid #ccc;
+ font-size: 11px;
+ font-family: Arial, Helvetica, sans-serif;
+ background-color: #f7f7f7;
+ color: #333;
+ -moz-box-shadow: 0 1px 0px rgba(0, 0, 0, 0.2), 0 0 0 2px #ffffff inset;
+ -webkit-box-shadow: 0 1px 0px rgba(0, 0, 0, 0.2), 0 0 0 2px #ffffff inset;
+ box-shadow: 0 1px 0px rgba(0, 0, 0, 0.2), 0 0 0 2px #ffffff inset;
+ -moz-border-radius: 3px;
+ -webkit-border-radius: 3px;
+ border-radius: 3px;
+ display: inline-block;
+ margin: 0 0.1em 0.1em 0.1em;
+ text-shadow: 0 1px 0 #fff;
+ line-height: 1.4;
+ white-space: nowrap;
+}
diff --git a/docs/source/cli.rst b/docs/source/cli.rst
new file mode 100644
index 0000000..565e54f
--- /dev/null
+++ b/docs/source/cli.rst
@@ -0,0 +1,168 @@
+.. _cli:
+
+Command Line Interface
+======================
+
+There are two command-line interfaces provided by **lnav**, one for viewing
+files and one for managing **lnav**'s configuration. The file viewing mode is
+the default and is all that most people will need. The management mode can
+be useful for those that are developing log file formats and is activated by
+passing the :option:`-m` option as the first argument.
+
+File Viewing Mode
+-----------------
+
+The following options can be used when starting **lnav**. There are not
+many flags because the majority of the functionality is accessed using
+the :option:`-c` option to execute :ref:`commands<commands>` or
+:ref:`SQL queries<sql-ext>`.
+
+Options
+^^^^^^^
+
+.. option:: -h
+
+ Print these command-line options and exit.
+
+.. option:: -H
+
+ Start lnav and switch to the help view.
+
+.. option:: -C
+
+ Check the given files against the configuration, report any errors, and
+ exit. This option can be helpful for validating that a log format is
+ well-formed.
+
+.. option:: -c <command>
+
+ Execute the given lnav command, SQL query, or lnav script. The
+ argument must be prefixed with the character used to enter the prompt
+ to distinguish between the different types (i.e. ':', ';', '|').
+ This option can be given multiple times.
+
+.. option:: -f <path>
+
+ Execute the given command file. This option can be given multiple times.
+
+.. option:: -I <path>
+
+ Add a configuration directory.
+
+.. option:: -i
+
+ Install the format files in the :file:`.lnav/formats/` directory.
+ Individual files will be installed in the :file:`installed`
+ directory and git repositories will be cloned with a directory
+ name based on their repository URI.
+
+.. option:: -u
+
+ Update formats installed from git repositories.
+
+.. option:: -d <path>
+
+ Write debug messages to the given file.
+
+.. option:: -n
+
+ Run without the curses UI (headless mode).
+
+.. option:: -N
+
+ Do not open the default syslog file if no files are given.
+
+.. option:: -r
+
+ Recursively load files from the given base directories.
+
+.. option:: -t
+
+ Prepend timestamps to the lines of data being read in on the standard input.
+
+.. option:: -w <path>
+
+ Write the contents of the standard input to this file.
+
+.. option:: -V
+
+ Print the version of lnav.
+
+.. option:: -q
+
+ Do not print the log messages after executing all of the commands.
+
+
+Management Mode (v0.11.0+)
+--------------------------
+
+The management CLI mode provides functionality for query **lnav**'s log
+format definitions.
+
+Options
+^^^^^^^
+
+.. option:: -m
+
+ Switch to management mode. This must be the first option passed on the
+ command-line.
+
+Subcommands
+^^^^^^^^^^^
+
+.. option:: regex101 import <regex101-url> <format-name> [<regex-name>]
+
+ Convert a regex101.com entry into a skeleton log format file.
+
+.. option:: format <format-name> regex <regex-name> push
+
+ Push a log format regular expression to regex101.com .
+
+.. option:: format <format-name> regex <regex-name> pull
+
+ Pull changes to a regex that was previously pushed to regex101.com .
+
+Environment Variables
+---------------------
+
+.. envvar:: XDG_CONFIG_HOME
+
+ If this variable is set, lnav will use this directory to store its
+ configuration in a sub-directory named :file:`lnav`.
+
+.. envvar:: HOME
+
+ If :envvar:`XDG_CONFIG_HOME` is not set, lnav will use this directory
+ to store its configuration in a sub-directory named :file:`.lnav`.
+
+.. envvar:: APPDATA
+
+ On Windows, lnav will use this directory instead of HOME
+ to store its configuration in a sub-directory named :file:`.lnav`.
+
+.. envvar:: TZ
+
+ The timezone setting is used in some log formats to convert UTC timestamps
+ to the local timezone.
+
+
+Examples
+--------
+
+ To load and follow the system syslog file:
+
+ .. prompt:: bash
+
+ lnav
+
+ To load all of the files in :file:`/var/log`:
+
+ .. prompt:: bash
+
+ lnav /var/log
+
+ To watch the output of make with timestamps prepended:
+
+ .. prompt:: bash
+
+ make 2>&1 | lnav -t
diff --git a/docs/source/commands.rst b/docs/source/commands.rst
new file mode 100644
index 0000000..d83a07e
--- /dev/null
+++ b/docs/source/commands.rst
@@ -0,0 +1,132 @@
+.. role:: lnavcmd(code)
+ :language: lnav
+ :class: highlight
+
+.. _commands:
+
+Commands
+========
+
+Commands provide access to some of the more advanced features in **lnav**, like
+:ref:`filtering<filtering>` and
+:ref:`"search tables"<search_tables>`. You can activate the command
+prompt by pressing the :kbd:`:` key. At the prompt, you can start typing
+in the desired command and/or double-tap :kbd:`TAB` to activate
+auto-completion and show the available commands. To guide you in the usage of
+the commands, a help window will appear above the command prompt with an
+explanation of the command and its parameters (if it has any). For example,
+the screenshot below shows the help for the :code:`:open` command:
+
+.. figure:: open-help.png
+ :align: center
+ :figwidth: 50%
+
+ Screenshot of the online help for the :code:`:open` command.
+
+In addition to online help, many commands provide a preview of the effects that
+the command will have. This preview will activate shortly after you have
+finished typing, but before you have pressed :kbd:`Enter` to execute the
+command. For example, the :code:`:open` command will show a preview of the
+first few lines of the file given as its argument:
+
+.. figure:: open-preview.png
+ :align: center
+
+ Screenshot of the preview shown for the :code:`:open` command.
+
+The :lnavcmd:`:filter-out pattern` command is another instance where the preview behavior
+can help you craft the correct command-line. This command takes a PCRE2 regular
+expression that specifies the log messages that should be filtered out of the
+view. The preview for this command will highlight the portion of the log
+messages that match the expression in red. Thus, you can be certain that the
+regular expression is matching the log messages you are interested in before
+committing the filter. The following screenshot shows an example of this
+preview behavior for the string "launchd":
+
+.. figure:: filter-out-preview.png
+ :align: center
+
+ Screenshot showing the preview for the :code:`:filter-out` command.
+
+Any errors detected during preview will be shown in the status bar right above
+the command prompt. For example, an attempt to open an unknown file will show
+an error message in the status bar, like so:
+
+.. figure:: open-error.png
+ :align: center
+
+ Screenshot of the error shown when trying to open a non-existent file.
+
+.. tip::
+
+ Note that almost all commands support TAB-completion for their arguments.
+ So, if you are in doubt as to what to type for an argument, you can double-
+ tap the :kbd:`TAB` key to get suggestions. For example, the
+ TAB-completion for the :code:`filter-in` command will suggest words that are
+ currently displayed in the view.
+
+.. note:: The following commands can be disabled by setting the ``LNAVSECURE``
+ environment variable before executing the **lnav** binary:
+
+ - :code:`:open`
+ - :code:`:pipe-to`
+ - :code:`:pipe-line-to`
+ - :code:`:write-*-to`
+
+ This makes it easier to run **lnav** in restricted environments without the
+ risk of privilege escalation.
+
+I/O Commands
+------------
+
+Anonymization
+^^^^^^^^^^^^^
+
+Anonymization is the process of removing identifying information from content
+to make it safer for sharing with others. For example, an IP address can
+often be used to uniquely identify an entity. Substituting all instances of
+a particular IP with the same dummy value would remove the identifying data
+without losing statistical accuracy. **lnav** has built-in support for
+anonymization through the :code:`--anonymize` flag on the :code:`:write-*`
+collection of commands. While the anonymization process should catch most
+
+ :IPv4 Addresses: Are replaced with addresses in the :code:`10.0.0.0/8` range.
+
+ :IPv6 Addresses: Are replaced with addresses in the :code:`2001:db8::/32` range.
+
+ :URL User Names: Are replaced with a random animal name.
+
+ :URL Passwords: Are replaced with a hash of the input password.
+
+ :URL Hosts: Are replaced with a random name under the example.com domain.
+
+ :URL Paths: Are recursively examined for substitution.
+
+ :URL Query Strings: Are recursively examined for substitution.
+
+ :URL Fragments: Are recursively examined for substitution.
+
+ :Paths: Are recursively examined for substitution.
+
+ :Credit Card Numbers: Are replaced with a 16 digit hash of the input number.
+
+ :MAC Addresses: Are replaced with addresses in the :code:`00:00:5E:00:53:00` range.
+
+ :Hex Dumps: Are replaced with a hash of the input replicated to the size of input.
+
+ :Email User Names: Are replaced with a random animal name.
+
+ :Email Host Names: Are replaced with a random name under the example.com domain.
+
+ :Words: Are replaced with a random word with a matching case style.
+
+ :Quoted Strings: Are recursively examined for substitution.
+
+ :UUID: Are replaced with a hash of the input.
+
+ :XML Attribute Values: Are recursively examined for substitution.
+
+Reference
+---------
+
+.. include:: ../../src/internals/cmd-ref.rst
diff --git a/docs/source/conf.py b/docs/source/conf.py
new file mode 100644
index 0000000..5f74b8c
--- /dev/null
+++ b/docs/source/conf.py
@@ -0,0 +1,460 @@
+# -*- coding: utf-8 -*-
+#
+# lnav documentation build configuration file, created by
+# sphinx-quickstart on Fri Jul 12 21:09:39 2013.
+#
+# This file is execfile()d with the current directory set to its containing dir.
+#
+# Note that not all possible configuration values are present in this
+# autogenerated file.
+#
+# All configuration values have a default; values that are commented out
+# serve to show the default.
+
+import sys, os
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+this_dir = os.path.abspath('.')
+src_dir = os.path.join(this_dir, "..", "..", "src")
+sys.path.insert(0, this_dir)
+sys.path.insert(0, src_dir)
+
+import format2csv
+
+format2csv.main(["",
+ os.path.join(this_dir, "format-table.csv"),
+ os.path.join(src_dir, "formats")])
+
+import re
+from pygments.lexer import RegexLexer, words
+from pygments.token import Punctuation, Whitespace, Text, Comment, Operator, \
+ Keyword, Name, String, Literal, Number, Generic
+from sphinx.highlighting import lexers
+
+
+class CustSqliteLexer(RegexLexer):
+ name = 'custsqlite'
+
+ flags = re.IGNORECASE
+ tokens = {
+ 'root': [
+ (r'\s+', Text),
+ (r'--.*\n?', Comment.Single),
+ (r'#.*\n?', Comment.Single),
+ (r'/\*', Comment.Multiline, 'multiline-comments'),
+ (words((
+ 'ABORT',
+ 'ACTION',
+ 'ADD',
+ 'AFTER',
+ 'ALL',
+ 'ALTER',
+ 'ALWAYS',
+ 'ANALYZE',
+ 'AND',
+ 'AS',
+ 'ASC',
+ 'ATTACH',
+ 'AUTOINCREMENT',
+ 'BEFORE',
+ 'BEGIN',
+ 'BETWEEN',
+ 'BY',
+ 'CASCADE',
+ 'CASE',
+ 'CAST',
+ 'CHECK',
+ 'COLLATE',
+ 'COLUMN',
+ 'COMMIT',
+ 'CONFLICT',
+ 'CONSTRAINT',
+ 'CREATE',
+ 'CROSS',
+ 'CURRENT',
+ 'CURRENT_DATE',
+ 'CURRENT_TIME',
+ 'CURRENT_TIMESTAMP',
+ 'DATABASE',
+ 'DEFAULT',
+ 'DEFERRABLE',
+ 'DEFERRED',
+ 'DELETE',
+ 'DESC',
+ 'DETACH',
+ 'DISTINCT',
+ 'DO',
+ 'DROP',
+ 'EACH',
+ 'ELSE',
+ 'END',
+ 'ESCAPE',
+ 'EXCEPT',
+ 'EXCLUDE',
+ 'EXCLUSIVE',
+ 'EXISTS',
+ 'EXPLAIN',
+ 'FAIL',
+ 'FILTER',
+ 'FIRST',
+ 'FOLLOWING',
+ 'FOR',
+ 'FOREIGN',
+ 'FROM',
+ 'FULL',
+ 'GENERATED',
+ 'GLOB',
+ 'GROUP',
+ 'GROUPS',
+ 'HAVING',
+ 'IF',
+ 'IGNORE',
+ 'IMMEDIATE',
+ 'IN',
+ 'INDEX',
+ 'INDEXED',
+ 'INITIALLY',
+ 'INNER',
+ 'INSERT',
+ 'INSTEAD',
+ 'INTERSECT',
+ 'INTO',
+ 'IS',
+ 'ISNULL',
+ 'JOIN',
+ 'KEY',
+ 'LAST',
+ 'LEFT',
+ 'LIKE',
+ 'LIMIT',
+ 'MATCH',
+ 'NATURAL',
+ 'NO',
+ 'NOT',
+ 'NOTHING',
+ 'NOTNULL',
+ 'NULL',
+ 'NULLS',
+ 'OF',
+ 'OFFSET',
+ 'ON',
+ 'OR',
+ 'ORDER',
+ 'OTHERS',
+ 'OUTER',
+ 'OVER',
+ 'PARTITION',
+ 'PLAN',
+ 'PRAGMA',
+ 'PRECEDING',
+ 'PRIMARY',
+ 'QUERY',
+ 'RAISE',
+ 'RANGE',
+ 'RECURSIVE',
+ 'REFERENCES',
+ 'REGEXP',
+ 'REINDEX',
+ 'RELEASE',
+ 'RENAME',
+ 'REPLACE',
+ 'RESTRICT',
+ 'RIGHT',
+ 'ROLLBACK',
+ 'ROW',
+ 'ROWS',
+ 'SAVEPOINT',
+ 'SELECT',
+ 'SET',
+ 'TABLE',
+ 'TEMP',
+ 'TEMPORARY',
+ 'THEN',
+ 'TIES',
+ 'TO',
+ 'TRANSACTION',
+ 'TRIGGER',
+ 'UNBOUNDED',
+ 'UNION',
+ 'UNIQUE',
+ 'UPDATE',
+ 'USING',
+ 'VACUUM',
+ 'VALUES',
+ 'VIEW',
+ 'VIRTUAL',
+ 'WHEN',
+ 'WHERE',
+ 'WINDOW',
+ 'WITH',
+ 'WITHOUT'), suffix=r'\b'),
+ Keyword),
+ (words((
+ 'ARRAY', 'BIGINT', 'BINARY', 'BIT', 'BLOB', 'BOOLEAN', 'CHAR',
+ 'CHARACTER', 'DATE', 'DEC', 'DECIMAL', 'FLOAT', 'INT', 'INTEGER',
+ 'INTERVAL', 'NUMBER', 'NUMERIC', 'REAL', 'SERIAL', 'SMALLINT',
+ 'VARCHAR', 'VARYING', 'INT8', 'SERIAL8', 'TEXT'), suffix=r'\b'),
+ Name.Builtin),
+ (r'[+*/<>=~!@#%^&|`?-]', Operator),
+ (r'[0-9]+', Number.Integer),
+ # TODO: Backslash escapes?
+ (r"'(''|[^'])*'", String.Single),
+ (r'"(""|[^"])*"', String.Symbol), # not a real string literal in ANSI SQL
+ (r'[a-z_][\w$]*', Name), # allow $s in strings for Oracle
+ (r'\$[a-z_]+', Name),
+ (r'[;:()\[\],.]', Punctuation)
+ ],
+ 'multiline-comments': [
+ (r'/\*', Comment.Multiline, 'multiline-comments'),
+ (r'\*/', Comment.Multiline, '#pop'),
+ (r'[^/*]+', Comment.Multiline),
+ (r'[/*]', Comment.Multiline)
+ ]
+ }
+
+ def analyse_text(text):
+ return 0.01
+
+
+lexers['custsqlite'] = CustSqliteLexer(startinline=True)
+
+# -- General configuration -----------------------------------------------------
+
+# If your documentation needs a minimal Sphinx version, state it here.
+# needs_sphinx = '1.0'
+
+# Add any Sphinx extension module names here, as strings. They can be extensions
+# coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
+extensions = [
+ 'sphinx.ext.coverage',
+ "sphinx_rtd_theme",
+ 'sphinx-jsonschema',
+ 'sphinx-prompt',
+ '_ext.lnavlexer',
+]
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ['_templates']
+
+# The suffix of source filenames.
+source_suffix = '.rst'
+
+# The encoding of source files.
+# source_encoding = 'utf-8-sig'
+
+# The master toctree document.
+master_doc = 'index'
+
+# General information about the project.
+project = u'lnav'
+copyright = u'2022, Tim Stack'
+
+# The version info for the project you're documenting, acts as replacement for
+# |version| and |release|, also used in various other places throughout the
+# built documents.
+#
+# The short X.Y version.
+version = '0.11'
+# The full version, including alpha/beta/rc tags.
+release = '0.11.1'
+
+# The language for content autogenerated by Sphinx. Refer to documentation
+# for a list of supported languages.
+# language = None
+
+# There are two options for replacing |today|: either, you set today to some
+# non-false value, then it is used:
+# today = ''
+# Else, today_fmt is used as the format for a strftime call.
+# today_fmt = '%B %d, %Y'
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+exclude_patterns = []
+
+# The reST default role (used for this markup: `text`) to use for all documents.
+# default_role = None
+
+# If true, '()' will be appended to :func: etc. cross-reference text.
+# add_function_parentheses = True
+
+# If true, the current module name will be prepended to all description
+# unit titles (such as .. function::).
+# add_module_names = True
+
+# If true, sectionauthor and moduleauthor directives will be shown in the
+# output. They are ignored by default.
+# show_authors = False
+
+# The name of the Pygments (syntax highlighting) style to use.
+pygments_style = 'sphinx'
+
+# A list of ignored prefixes for module index sorting.
+# modindex_common_prefix = []
+
+# -- Options for HTML output ---------------------------------------------------
+
+# The theme to use for HTML and HTML Help pages. See the documentation for
+# a list of builtin themes.
+html_theme = 'sphinx_rtd_theme'
+
+# Theme options are theme-specific and customize the look and feel of a theme
+# further. For a list of options available for each theme, see the
+# documentation.
+html_theme_options = {
+}
+
+# Add any paths that contain custom themes here, relative to this directory.
+# html_theme_path = []
+
+# The name for this set of Sphinx documents. If None, it defaults to
+# "<project> v<release> documentation".
+# html_title = None
+
+# A shorter title for the navigation bar. Default is the same as html_title.
+# html_short_title = None
+
+# The name of an image file (relative to this directory) to place at the top
+# of the sidebar.
+# html_logo = None
+
+# The name of an image file (within the static path) to use as favicon of the
+# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32
+# pixels large.
+# html_favicon = None
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = ['_static']
+
+
+def setup(app):
+ app.add_css_file('theme_overrides.css')
+
+
+# html_context = {
+# 'css_files': [
+# '_static/theme_overrides.css', # override wide tables in RTD theme
+# ],
+# }
+
+# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
+# using the given strftime format.
+# html_last_updated_fmt = '%b %d, %Y'
+
+# If true, SmartyPants will be used to convert quotes and dashes to
+# typographically correct entities.
+# html_use_smartypants = True
+
+# Custom sidebar templates, maps document names to template names.
+# html_sidebars = {}
+
+# Additional templates that should be rendered to pages, maps page names to
+# template names.
+# html_additional_pages = {}
+
+# If false, no module index is generated.
+# html_domain_indices = True
+
+# If false, no index is generated.
+# html_use_index = True
+
+# If true, the index is split into individual pages for each letter.
+# html_split_index = False
+
+# If true, links to the reST sources are added to the pages.
+# html_show_sourcelink = True
+
+# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
+# html_show_sphinx = True
+
+# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
+# html_show_copyright = True
+
+# If true, an OpenSearch description file will be output, and all pages will
+# contain a <link> tag referring to it. The value of this option must be the
+# base URL from which the finished HTML is served.
+# html_use_opensearch = ''
+
+# This is the file name suffix for HTML files (e.g. ".xhtml").
+# html_file_suffix = None
+
+# Output file base name for HTML help builder.
+htmlhelp_basename = 'lnavdoc'
+
+# -- Options for LaTeX output --------------------------------------------------
+
+latex_elements = {
+ # The paper size ('letterpaper' or 'a4paper').
+ # 'papersize': 'letterpaper',
+
+ # The font size ('10pt', '11pt' or '12pt').
+ # 'pointsize': '10pt',
+
+ # Additional stuff for the LaTeX preamble.
+ # 'preamble': '',
+}
+
+# Grouping the document tree into LaTeX files. List of tuples
+# (source start file, target name, title, author, documentclass [howto/manual]).
+latex_documents = [
+ ('index', 'lnav.tex', u'lnav Documentation',
+ u'Tim Stack', 'manual'),
+]
+
+# The name of an image file (relative to this directory) to place at the top of
+# the title page.
+# latex_logo = None
+
+# For "manual" documents, if this is true, then toplevel headings are parts,
+# not chapters.
+# latex_use_parts = False
+
+# If true, show page references after internal links.
+# latex_show_pagerefs = False
+
+# If true, show URL addresses after external links.
+# latex_show_urls = False
+
+# Documents to append as an appendix to all manuals.
+# latex_appendices = []
+
+# If false, no module index is generated.
+# latex_domain_indices = True
+
+
+# -- Options for manual page output --------------------------------------------
+
+# One entry per manual page. List of tuples
+# (source start file, name, description, authors, manual section).
+man_pages = [
+ ('index', 'lnav', u'lnav Documentation',
+ [u'Tim Stack'], 1)
+]
+
+# If true, show URL addresses after external links.
+# man_show_urls = False
+
+
+# -- Options for Texinfo output ------------------------------------------------
+
+# Grouping the document tree into Texinfo files. List of tuples
+# (source start file, target name, title, author,
+# dir menu entry, description, category)
+texinfo_documents = [
+ ('index', 'lnav', u'lnav Documentation',
+ u'Tim Stack', 'lnav', 'One line description of project.',
+ 'Miscellaneous'),
+]
+
+# Documents to append as an appendix to all manuals.
+# texinfo_appendices = []
+
+# If false, no module index is generated.
+# texinfo_domain_indices = True
+
+# How to display URL addresses: 'footnote', 'no', or 'inline'.
+# texinfo_show_urls = 'footnote'
diff --git a/docs/source/config.rst b/docs/source/config.rst
new file mode 100644
index 0000000..5294e52
--- /dev/null
+++ b/docs/source/config.rst
@@ -0,0 +1,270 @@
+
+.. _Configuration:
+
+Configuration
+=============
+
+The configuration for **lnav** is stored in the following JSON files in
+:file:`~/.lnav`:
+
+* :file:`config.json` -- Contains local customizations that were done using the
+ :code:`:config` command.
+* :file:`configs/default/*.json` -- The default configuration files that are
+ built into lnav are written to this directory with :file:`.sample` appended.
+ Removing the :file:`.sample` extension and editing the file will allow you to
+ do basic customizations.
+* :file:`configs/installed/*.json` -- Contains configuration files installed
+ using the :option:`-i` flag (e.g. :code:`$ lnav -i /path/to/config.json`).
+* :file:`configs/*/*.json` -- Other directories that contain :file:`*.json`
+ files will be loaded on startup. This structure is convenient for installing
+ **lnav** configurations, like from a git repository.
+
+A valid **lnav** configuration file must contain an object with the
+:code:`$schema` property, like so:
+
+.. code-block:: json
+
+ {
+ "$schema": "https://lnav.org/schemas/config-v1.schema.json"
+ }
+
+.. note::
+
+ Log format definitions are stored separately in the :file:`~/.lnav/formats`
+ directly. See the :ref:`Log Formats<log_formats>` chapter for more
+ information.
+
+
+Options
+-------
+
+The following configuration options can be used to customize the **lnav** UI to
+your liking. The options can be changed using the :code:`:config` command.
+
+.. jsonschema:: ../schemas/config-v1.schema.json#/properties/ui/properties/keymap
+
+.. jsonschema:: ../schemas/config-v1.schema.json#/properties/ui/properties/theme
+
+.. jsonschema:: ../schemas/config-v1.schema.json#/properties/ui/properties/clock-format
+
+.. jsonschema:: ../schemas/config-v1.schema.json#/properties/ui/properties/dim-text
+
+.. jsonschema:: ../schemas/config-v1.schema.json#/properties/ui/properties/default-colors
+
+
+.. _themes:
+
+Theme Definitions
+-----------------
+
+User Interface themes are defined in a JSON configuration file. A theme is
+made up of the style definitions for different types of text in the UI. A
+:ref:`definition<theme_style>` can include the foreground/background colors
+and the bold/underline attributes. The style definitions are broken up into
+multiple categories for the sake of organization. To make it easier to write
+a definition, a theme can define variables that can be referenced as color
+values.
+
+Variables
+^^^^^^^^^
+
+The :code:`vars` object in a theme definition contains the mapping of variable
+names to color values. These variables can be referenced in style definitions
+by prefixing them with a dollar-sign (e.g. :code:`$black`). The following
+variables can also be defined to control the values of the ANSI colors that
+are log messages or plain text:
+
+.. csv-table:: ANSI colors
+ :header: "Variable Name", "ANSI Escape"
+
+ "black", "ESC[30m"
+ "red", "ESC[31m"
+ "green", "ESC[32m"
+ "yellow", "ESC[33m"
+ "blue", "ESC[34m"
+ "magenta", "ESC[35m"
+ "cyan", "ESC[36m"
+ "white", "ESC[37m"
+
+Specifying Colors
+^^^^^^^^^^^^^^^^^
+
+Colors can be specified using hexadecimal notation by starting with a hash
+(e.g. :code:`#aabbcc`) or using a color name as found at
+http://jonasjacek.github.io/colors/. If colors are not specified for a style,
+the values from the :code:`styles/text` definition.
+
+.. note::
+
+ When specifying colors in hexadecimal notation, you do not need to have an
+ exact match in the XTerm 256 color palette. A best approximation will be
+ picked based on the `CIEDE2000 <https://en.wikipedia.org/wiki/Color_difference#CIEDE2000>`_
+ color difference algorithm.
+
+
+
+Example
+^^^^^^^
+
+The following example sets the black/background color for text to a dark grey
+using a variable and sets the foreground to an off-white. This theme is
+incomplete, but it works enough to give you an idea of how a theme is defined.
+You can copy the code block, save it to a file in
+:file:`~/.lnav/configs/installed/` and then activate it by executing
+:code:`:config /ui/theme example` in lnav. For a more complete theme
+definition, see one of the definitions built into **lnav**, like
+`monocai <https://github.com/tstack/lnav/blob/master/src/themes/monocai.json>`_.
+
+ .. code-block:: json
+
+ {
+ "$schema": "https://lnav.org/schemas/config-v1.schema.json",
+ "ui": {
+ "theme-defs": {
+ "example1": {
+ "vars": {
+ "black": "#2d2a2e"
+ },
+ "styles": {
+ "text": {
+ "color": "#f6f6f6",
+ "background-color": "$black"
+ }
+ }
+ }
+ }
+ }
+ }
+
+Reference
+^^^^^^^^^
+
+.. jsonschema:: ../schemas/config-v1.schema.json#/properties/ui/properties/theme-defs/patternProperties/([\w\-]+)/properties/vars
+
+.. jsonschema:: ../schemas/config-v1.schema.json#/properties/ui/properties/theme-defs/patternProperties/([\w\-]+)/properties/styles
+
+.. jsonschema:: ../schemas/config-v1.schema.json#/properties/ui/properties/theme-defs/patternProperties/([\w\-]+)/properties/syntax-styles
+
+.. jsonschema:: ../schemas/config-v1.schema.json#/properties/ui/properties/theme-defs/patternProperties/([\w\-]+)/properties/status-styles
+
+.. jsonschema:: ../schemas/config-v1.schema.json#/properties/ui/properties/theme-defs/patternProperties/([\w\-]+)/properties/log-level-styles
+
+.. _theme_style:
+
+.. jsonschema:: ../schemas/config-v1.schema.json#/definitions/style
+
+
+.. _keymaps:
+
+Keymap Definitions
+------------------
+
+Keymaps in **lnav** map a key sequence to a command to execute. When a key is
+pressed, it is converted into a hex-encoded string that is looked up in the
+keymap. The :code:`command` value associated with the entry in the keymap is
+then executed. Note that the "command" can be an **lnav**
+:ref:`command<commands>`, a :ref:`SQL statement/query<sql-ext>`, or an
+**lnav** script. If an :code:`alt-msg` value is included in the entry, the
+bottom-right section of the UI will be updated with the help text.
+
+.. note::
+
+ Not all functionality is available via commands or SQL at the moment. Also,
+ some hotkeys are not implemented via keymaps.
+
+Key Sequence Encoding
+^^^^^^^^^^^^^^^^^^^^^
+
+Key presses are converted into a hex-encoded string that is used to lookup an
+entry in the keymap. Each byte of the keypress value is formatted as an
+:code:`x` followed by the hex-encoding in lowercase. For example, the encoding
+for the £ key would be :code:`xc2xa3`. To make it easier to discover the
+encoding for unassigned keys, **lnav** will print in the command prompt the
+:code:`:config` command and
+`JSON-Pointer <https://tools.ietf.org/html/rfc6901>`_ for assigning a command
+to the key.
+
+.. figure:: key-encoding-prompt.png
+ :align: center
+
+ Screenshot of the command prompt when an unassigned key is pressed.
+
+.. note::
+
+ Since **lnav** is a terminal application, it can only receive keypresses that
+ can be represented as characters or escape sequences. For example, it cannot
+ handle the press of a modifier key.
+
+Reference
+^^^^^^^^^
+
+.. jsonschema:: ../schemas/config-v1.schema.json#/properties/ui/properties/keymap-defs/patternProperties/([\w\-]+)
+
+
+Log Handling
+------------
+
+The handling of logs is largely determined by the
+:ref:`log file formats<log_formats>`, this section covers options that are not
+specific to a particular format.
+
+Watch Expressions (v0.11.0+)
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Watch expressions can be used to fire an event when a log message matches a
+condition. You can then install a listener for these events and trigger an
+action to be performed. For example, to automate filtering based on
+identifiers, a watch expression can match messages that mention the ID and then
+a trigger can install a filter for that ID. Creating a watch expression is
+done by adding an entry into the :code:`/log/watch-expressions` configuration
+tree. For example, to create a watch named "dhcpdiscover" that matches
+DHCPDISCOVER messages from the :code:`dhclient` daemon, you would run the
+following:
+
+.. code-block:: lnav
+
+ :config /log/watch-expressions/dhcpdiscover/expr :log_procname = 'dhclient' AND startswith(:log_body, 'DHCPDISCOVER')
+
+The watch expression can refer to column names in the log message by prefixing
+them with a colon. The expression is evaluated by passing the log message
+fields as bound parameters and not against a table. The easiest way to test
+out an expression is with the :ref:`mark_expr` command, since it will behave
+similarly. After changing the configuration, you'll need to restart lnav
+for the effect to take place. You can then query the :code:`lnav_events`
+table to see any generated
+:code:`https://lnav.org/event-log-msg-detected-v1.schema.json` events from the
+logs that were loaded:
+
+.. code-block:: custsqlite
+
+ ;SELECT * FROM lnav_events
+
+From there, you can create a SQLite trigger on the :code:`lnav_events` table
+that will examine the event contents and perform an action. See the
+:ref:`Events` section for more information on handling events.
+
+Reference
+^^^^^^^^^
+
+.. jsonschema:: ../schemas/config-v1.schema.json#/properties/log/properties/watch-expressions/patternProperties/([\w\-]+)
+
+.. _tuning:
+
+Tuning
+------
+
+The following configuration options can be used to tune the internals of
+**lnav** to your liking. The options can be changed using the :code:`:config`
+command.
+
+.. jsonschema:: ../schemas/config-v1.schema.json#/properties/tuning/properties/archive-manager
+
+.. jsonschema:: ../schemas/config-v1.schema.json#/properties/tuning/properties/clipboard
+
+.. jsonschema:: ../schemas/config-v1.schema.json#/definitions/clip-commands
+
+.. jsonschema:: ../schemas/config-v1.schema.json#/properties/tuning/properties/file-vtab
+
+.. jsonschema:: ../schemas/config-v1.schema.json#/properties/tuning/properties/logfile
+
+.. jsonschema:: ../schemas/config-v1.schema.json#/properties/tuning/properties/remote/properties/ssh
diff --git a/docs/source/cookbook.rst b/docs/source/cookbook.rst
new file mode 100644
index 0000000..3eb0ffd
--- /dev/null
+++ b/docs/source/cookbook.rst
@@ -0,0 +1,104 @@
+
+.. _Cookbook:
+
+Cookbook
+========
+
+This chapter contains recipes for common tasks that can be done in **lnav**.
+These recipes can be used as a starting point for your own needs after some
+adaptation.
+
+
+Log Formats
+-----------
+
+TBD
+
+Defining a New Format
+^^^^^^^^^^^^^^^^^^^^^
+
+TBD
+
+
+Annotating Logs
+---------------
+
+Log messages can be annotated in a couple of different ways in **lnav** to help
+you get organized.
+
+Create partitions for Linux boots
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+When digging through logs that can be broken up into multiple sections,
+**lnav**'s :ref:`partitioning feature<taking_notes>` can be used to keep track
+of which section you are in. For example, if a collection of Linux logs
+covered multiple boots, the following script could be used to create partitions
+for each boot. After the partition name is set for the log messages, the
+current name will show up in the top status bar next to the current time.
+
+.. literalinclude:: ../../src/scripts/partition-by-boot.lnav
+ :language: custsqlite
+ :caption: partition-by-boot.lnav
+ :linenos:
+
+Tagging SSH log messages
+^^^^^^^^^^^^^^^^^^^^^^^^
+
+Log messages can be tagged interactively with the :ref:`:tag<tag>` command or
+programmatically using the :ref:`sql-ext`. This example uses a script to
+search for interesting SSH messages and automatically adds an appropriate tag.
+
+.. literalinclude:: ../../example-scripts/tag-ssh-msgs.lnav
+ :language: custsqlite
+ :caption: tag-ssh-msgs.lnav
+ :linenos:
+
+Log Analysis
+------------
+
+Most log analysis within **lnav** is done through the :ref:`sql-ext`. The
+following examples should give you some ideas to start leveraging this
+functionality. One thing to keep in mind is that if a query gets to be too
+large or multiple statements need to be executed, you can create a
+:code:`.lnav` script that contains the statements and execute it using the
+:kbd:`\|` command prompt.
+
+Count client IPs in web access logs
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+To count the occurrences of an IP in web access logs and order the results
+from highest to lowest:
+
+ .. code-block:: custsqlite
+
+ ;SELECT c_ip, count(*) as hits FROM access_log GROUP BY c_ip ORDER BY hits DESC
+
+
+Show only lines where a numeric field is in a range
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+The :ref:`:filter-expr<filter_expr>` command can be used to filter web access
+logs to only show lines where the number of bytes transferred to the client is
+between 10,000 and 40,000 bytes like so:
+
+ .. code-block:: custsqlite
+
+ :filter-expr :sc_bytes BETWEEN 10000 AND 40000
+
+
+Generating a Report
+^^^^^^^^^^^^^^^^^^^
+
+Reports can be generated by writing an **lnav** :ref:`script<scripts>` that
+uses SQL queries and commands to format a document. A basic script can simply
+execute a SQL query that is shown in the DB view. More sophisticated scripts
+can use the following commands to generate customized output for a report:
+
+* The :ref:`:echo<echo>` command to write plain text
+* :ref:`SQL queries<sql-ext>` followed by a "write" command, like
+ :ref:`:write-table-to<write_table_to>`.
+
+.. literalinclude:: ../../example-scripts/report-demo.lnav
+ :language: custsqlite
+ :caption: report-demo.lnav
+ :linenos:
diff --git a/docs/source/data.rst b/docs/source/data.rst
new file mode 100644
index 0000000..a7352e1
--- /dev/null
+++ b/docs/source/data.rst
@@ -0,0 +1,193 @@
+
+.. _data-ext:
+
+Extracting Data
+===============
+
+**Note**: This feature is still in **BETA**, you should expect bugs and
+incompatible changes in the future.
+
+Log messages contain a good deal of useful data, but it's not always easy to get
+at. The log parser built into **lnav** is able to extract data as described by
+:ref:`log_formats` as well as discovering data in plain text messages. This data
+can then be queried and processed using the SQLite front-end that is also
+incorporated into **lnav**. As an example, the following Syslog message from
+:code:`sudo` can be processed to extract several key/value pairs::
+
+ Jul 31 11:42:26 Example-MacBook-Pro.local sudo[87024]: testuser : TTY=ttys004 ; PWD=/Users/testuser/github/lbuild ; USER=root ; COMMAND=/usr/bin/make install
+
+The data that can be extracted by the parser is viewable directly in **lnav**
+by pressing the 'p' key. The results will be shown in an overlay like the
+following::
+
+ Current Time: 2013-07-31T11:42:26.000 Original Time: 2013-07-31T11:42:26.000 Offset: +0.000
+ Known message fields:
+ ├ log_hostname = Example-MacBook-Pro.local
+ ├ log_procname = sudo
+ ├ log_pid = 87024
+ Discovered message fields:
+ ├ col_0 = testuser
+ ├ TTY = ttys004
+ ├ PWD = /Users/testuser/github/lbuild
+ ├ USER = root
+ └ COMMAND = /usr/bin/make install
+
+Notice that the parser has detected pairs of the form '<key>=<value>'. The data
+parser will also look for pairs separated by a colon. If there are no clearly
+demarcated pairs, then the parser will extract anything that looks like data
+values and assign them keys of the form 'col_N'. For example, two data values,
+an IPv4 address and a symbol, will be extracted from the following log
+message::
+
+ Apr 29 08:13:43 sample-centos5 avahi-daemon[2467]: Registering new address record for 10.1.10.62 on eth0.
+
+Since there are no keys for the values in the message, the parser will assign
+'col_0' for the IP address and 'col_1' for the symbol, as seen here::
+
+ Current Time: 2013-04-29T08:13:43.000 Original Time: 2013-04-29T08:13:43.000 Offset: +0.000
+ Known message fields:
+ ├ log_hostname = sample-centos5
+ ├ log_procname = avahi-daemon
+ ├ log_pid = 2467
+ Discovered message fields:
+ ├ col_0 = 10.1.10.62
+ └ col_1 = eth0
+
+Now that you have an idea of how the parser works, you can begin to perform
+queries on the data that is being extracted. The SQLite database engine is
+embedded into **lnav** and its `Virtual Table
+<http://www.sqlite.org/vtab.html>`_ mechanism is used to provide a means to
+process this log data. Each log format has its own table that can be used to
+access all of the loaded messages that are in that format. For accessing log
+message content that is more free-form, like the examples given here, the
+**logline** table can be used. The **logline** table is recreated for each
+query and is based on the format and pairs discovered in the log message at
+the top of the display.
+
+Queries can be performed by pressing the semi-colon (;) key in **lnav**. After
+pressing the key, the overlay showing any known or discovered fields will be
+displayed to give you an idea of what data is available. The query can be any
+`SQL query <http://sqlite.org/lang.html>`_ supported by SQLite. To make
+analysis easier, **lnav** includes many extra functions for processing strings,
+paths, and IP addresses. See :ref:`sql-ext` for more information.
+
+As an example, the simplest query to perform initially would be a "select all",
+like so:
+
+.. code-block:: sql
+
+ SELECT * FROM logline
+
+When this query is run against the second example log message given above, the
+following results are received::
+
+ log_line log_part log_time log_idle_msecs log_level log_hostname log_procname log_pid col_0 col_1
+
+ 292 p.0 2013-04-11T16:42:51.000 0 info localhost avahi-daemon 2480 fe80::a00:27ff:fe98:7f6e eth0
+ 293 p.0 2013-04-11T16:42:51.000 0 info localhost avahi-daemon 2480 10.0.2.15 eth0
+ 330 p.0 2013-04-11T16:47:02.000 0 info localhost avahi-daemon 2480 fe80::a00:27ff:fe98:7f6e eth0
+ 336 p.0 2013-04-11T16:47:02.000 0 info localhost avahi-daemon 2480 10.1.10.75 eth0
+ 343 p.0 2013-04-11T16:47:02.000 0 info localhost avahi-daemon 2480 10.1.10.75 eth0
+ 370 p.0 2013-04-11T16:59:39.000 0 info localhost avahi-daemon 2480 10.1.10.75 eth0
+ 377 p.0 2013-04-11T16:59:39.000 0 info localhost avahi-daemon 2480 10.1.10.75 eth0
+ 382 p.0 2013-04-11T16:59:41.000 0 info localhost avahi-daemon 2480 fe80::a00:27ff:fe98:7f6e eth0
+ 401 p.0 2013-04-11T17:20:45.000 0 info localhost avahi-daemon 4247 fe80::a00:27ff:fe98:7f6e eth0
+ 402 p.0 2013-04-11T17:20:45.000 0 info localhost avahi-daemon 4247 10.1.10.75 eth0
+
+ 735 p.0 2013-04-11T17:41:46.000 0 info sample-centos5 avahi-daemon 2465 fe80::a00:27ff:fe98:7f6e eth0
+ 736 p.0 2013-04-11T17:41:46.000 0 info sample-centos5 avahi-daemon 2465 10.1.10.75 eth0
+ 781 p.0 2013-04-12T03:32:30.000 0 info sample-centos5 avahi-daemon 2465 10.1.10.64 eth0
+ 788 p.0 2013-04-12T03:32:30.000 0 info sample-centos5 avahi-daemon 2465 10.1.10.64 eth0
+ 1166 p.0 2013-04-25T10:56:00.000 0 info sample-centos5 avahi-daemon 2467 fe80::a00:27ff:fe98:7f6e eth0
+ 1167 p.0 2013-04-25T10:56:00.000 0 info sample-centos5 avahi-daemon 2467 10.1.10.111 eth0
+ 1246 p.0 2013-04-26T06:06:25.000 0 info sample-centos5 avahi-daemon 2467 10.1.10.49 eth0
+ 1253 p.0 2013-04-26T06:06:25.000 0 info sample-centos5 avahi-daemon 2467 10.1.10.49 eth0
+ 1454 p.0 2013-04-28T06:53:55.000 0 info sample-centos5 avahi-daemon 2467 10.1.10.103 eth0
+ 1461 p.0 2013-04-28T06:53:55.000 0 info sample-centos5 avahi-daemon 2467 10.1.10.103 eth0
+
+ 1497 p.0 2013-04-29T08:13:43.000 0 info sample-centos5 avahi-daemon 2467 10.1.10.62 eth0
+ 1504 p.0 2013-04-29T08:13:43.000 0 info sample-centos5 avahi-daemon 2467 10.1.10.62 eth0
+
+Note that **lnav** is not returning results for all messages that are in this
+syslog file. Rather, it searches for messages that match the format for the
+given line and returns only those messages in results. In this case, that
+format is "Registering new address record for <IP> on <symbol>", which
+corresponds to the parts of the message that were not recognized as data.
+
+More sophisticated queries can be done, of course. For example, to find out the
+frequency of IP addresses mentioned in these messages, you can run:
+
+.. code-block:: sql
+
+ SELECT col_0,count(*) FROM logline GROUP BY col_0
+
+The results for this query are::
+
+ col_0 count(*)
+
+ 10.0.2.15 1
+ 10.1.10.49 2
+ 10.1.10.62 2
+ 10.1.10.64 2
+ 10.1.10.75 6
+ 10.1.10.103 2
+ 10.1.10.111 1
+ fe80::a00:27ff:fe98:7f6e 6
+
+Since this type of query is fairly common, **lnav** includes a "summarize"
+command that will compute the frequencies of identifiers as well as min, max,
+average, median, and standard deviation for number columns. In this case, you
+can run the following to compute the frequencies and return an ordered set of
+results::
+
+ :summarize col_0
+
+
+Recognized Data Types
+---------------------
+
+When searching for data to extract from log messages, **lnav** looks for the
+following set of patterns:
+
+
+Strings
+ Single and double-quoted strings. Example: "The quick brown fox."
+
+URLs
+ URLs that contain the '://' separator. Example: http://example.com
+
+Paths
+ File system paths. Examples: /path/to/file, ./relative/path
+
+MAC Address
+ Ethernet MAC addresses. Example: c4:2c:03:0e:e4:4a
+
+Hex Dumps
+ A colon-separated string of hex numbers. Example: e8:06:88:ff
+
+Date/Time
+ Date and time stamps of the form "YYYY-mm-DD" and "HH:MM:SS".
+
+IP Addresses
+ IPv4 and IPv6 addresses. Examples: 127.0.0.1, fe80::c62c:3ff:fe0e:e44a%en0
+
+UUID
+ The common formatting for 128-bit UUIDs. Example:
+ 0E305E39-F1E9-4DE4-B10B-5829E5DF54D0
+
+Version Numbers
+ Dot-separated version numbers. Example: 3.7.17
+
+Numbers
+ Numbers in base ten, hex, and octal formats. Examples: 1234, 0xbeef, 0777
+
+E-Mail Address
+ Strings that look close to an e-mail address. Example: gary@example.com
+
+Constants
+ Common constants in languages, like: true, false, null, None.
+
+Symbols
+ Words that follow the common conventions for symbols in programming
+ languages. For example, containing all capital letters, or separated
+ by colons. Example: SOME_CONSTANT_VALUE, namespace::value
diff --git a/docs/source/docutils.conf b/docs/source/docutils.conf
new file mode 100644
index 0000000..1bf4d83
--- /dev/null
+++ b/docs/source/docutils.conf
@@ -0,0 +1,2 @@
+[restructuredtext parser]
+syntax_highlight = short
diff --git a/docs/source/events.rst b/docs/source/events.rst
new file mode 100644
index 0000000..8318abf
--- /dev/null
+++ b/docs/source/events.rst
@@ -0,0 +1,56 @@
+.. _Events:
+
+Events (v0.11.0+)
+=================
+
+The events mechanism allows **lnav** to be automated based on events that
+occur during processing. For example, filters could be added only when a
+particular log file format is detected instead of always installing them.
+Events are published through the :ref:`lnav_events<table_lnav_events>` SQLite
+table. Reacting to events can be done by creating a SQLite trigger on the
+table and inspecting the content of the event.
+
+Trigger Example
+---------------
+
+The following is an example of a trigger that adds an out filter when a
+syslog file is loaded. You can copy the code into an :file:`.sql` file and
+install it by running :code:`lnav -i my_trigger.sql`.
+
+.. code-block:: sql
+ :caption: my_trigger.sql
+ :linenos:
+
+ CREATE TRIGGER IF NOT EXISTS add_format_specific_filters
+ AFTER INSERT ON lnav_events WHEN
+ -- Check the event type
+ jget(NEW.content, '/$schema') =
+ 'https://lnav.org/event-file-format-detected-v1.schema.json' AND
+ -- Only create the filter when a given format is seen
+ jget(NEW.content, '/format') = 'syslog_log' AND
+ -- Don't create the filter if it's already there
+ NOT EXISTS (
+ SELECT 1 FROM lnav_view_filters WHERE pattern = 'noisy message')
+ BEGIN
+ INSERT INTO lnav_view_filters (view_name, enabled, type, pattern) VALUES
+ ('log', 1, 'OUT', 'noisy message');
+ END;
+
+.. _event_reference:
+
+Reference
+---------
+
+The following tables describe the schema of the event JSON objects.
+
+.. jsonschema:: ../schemas/event-file-open-v1.schema.json#
+ :lift_description:
+
+.. jsonschema:: ../schemas/event-file-format-detected-v1.schema.json#
+ :lift_description:
+
+.. jsonschema:: ../schemas/event-log-msg-detected-v1.schema.json#
+ :lift_description:
+
+.. jsonschema:: ../schemas/event-session-loaded-v1.schema.json#
+ :lift_description:
diff --git a/docs/source/faq.rst b/docs/source/faq.rst
new file mode 100644
index 0000000..6ca1d92
--- /dev/null
+++ b/docs/source/faq.rst
@@ -0,0 +1,57 @@
+
+.. _faq:
+
+Frequently Asked Questions
+==========================
+
+Q: How can I copy & paste without decorations?
+----------------------------------------------
+
+:Answer: There are a few ways to do this:
+
+ * Use the :ref:`bookmark<hotkeys_bookmarks>` hotkeys to mark lines and then
+ press :kbd:`c` to copy to the local system keyboard.
+
+ * Press :kbd:`CTRL` + :kbd:`l` to temporarily switch to "lo-fi"
+ mode where the contents of the current view are printed to the terminal.
+ This option is useful when you are logged into a remote host.
+
+
+Q: How can I force a format for a file?
+---------------------------------------
+
+:Answer: The log format for a file is automatically detected and cannot be
+ forced.
+
+:Solution: Add some of the log file lines to the :ref:`sample<format_sample>`
+ array and then startup lnav to get a detailed explanation of where the format
+ patterns are not matching the sample lines.
+
+:Details: The first lines of the file are matched against the
+ :ref:`regular expressions defined in the format definitions<format_regex>`.
+ The order of the formats is automatically determined so that more specific
+ formats are tried before more generic ones. Therefore, if the expected
+ format is not being chosen for a file, then it means the regular expressions
+ defined by that format are not matching the first few lines of the file.
+
+ See :ref:`format_order` for more information.
+
+Q: Why isn't my log file highlighted correctly?
+-----------------------------------------------
+
+TBD
+
+Q: Why isn't a file being displayed?
+------------------------------------
+
+:Answer: Plaintext files are displayed separately from log files in the TEXT
+ view.
+
+:Solution: Press the :kbd:`t` key to switch to the text view. Or, open the
+ files configuration panel by pressing :kbd:`TAB` to cycle through the
+ panels, and then press :kbd:`/` to search for the file you're interested in.
+ If the file is a log, a new :ref:`log format<log_formats>` will need to be
+ created or an existing one modified.
+
+:Details: If a file being monitored by lnav does not match a known log file
+ format, it is treated as plaintext and will be displayed in the TEXT view.
diff --git a/docs/source/filter-out-preview.png b/docs/source/filter-out-preview.png
new file mode 100644
index 0000000..8f9816f
--- /dev/null
+++ b/docs/source/filter-out-preview.png
Binary files differ
diff --git a/docs/source/formats.rst b/docs/source/formats.rst
new file mode 100644
index 0000000..8960761
--- /dev/null
+++ b/docs/source/formats.rst
@@ -0,0 +1,527 @@
+.. _log_formats:
+
+Log Formats
+===========
+
+Log files loaded into **lnav** are parsed based on formats defined in
+configuration files. Many
+formats are already built in to the **lnav** binary and you can define your own
+using a JSON file. When loading files, each format is checked to see if it can
+parse the first few lines in the file. Once a match is found, that format will
+be considered that files format and used to parse the remaining lines in the
+file. If no match is found, the file is considered to be plain text and can
+be viewed in the "text" view that is accessed with the **t** key.
+
+The following log formats are built into **lnav**:
+
+.. csv-table::
+ :header: "Name", "Table Name", "Description"
+ :widths: 8 5 20
+ :file: format-table.csv
+
+In addition to the above formats, the following self-describing formats are
+supported:
+
+* The
+ `Bro Network Security Monitor <https://www.bro.org/sphinx/script-reference/log-files.html>`_
+ TSV log format is supported in lnav versions v0.8.3+. The Bro log format is
+ self-describing, so **lnav** will read the header to determine the shape of
+ the file.
+* The
+ `W3C Extend Log File Format <https://www.w3.org/TR/WD-logfile.html>`_
+ is supported in lnav versions v0.10.0+. The W3C log format is
+ self-describing, so **lnav** will read the header to determine the shape of
+ the file.
+
+There is also basic support for the `logfmt <https://brandur.org/logfmt>`_
+convention for formatting log messages. Files that use this format must
+have the entire line be key/value pairs and the timestamp contained in a
+field named :code:`time` or :code:`ts`. If the file you're using does not
+quite follow this formatting, but wraps logfmt data with another recognized
+format, you can use the :ref:`logfmt2json` SQL function to convert the data
+into JSON for further analysis.
+
+
+Defining a New Format
+---------------------
+
+New log formats can be defined by placing JSON configuration files in
+subdirectories of the :file:`~/.lnav/formats/` directory. The directories and
+files can be named anything you like, but the files must have the '.json' suffix. A
+sample file containing the builtin configuration will be written to this
+directory when **lnav** starts up. You can consult that file when writing your
+own formats or if you need to modify existing ones. Format directories can
+also contain '.sql' and '.lnav' script files that can be used automate log file
+analysis.
+
+Creating a Format Using Regex101.com (v0.11.0+)
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+For plain-text log files, the easiest way to create a log format definition is
+to create the regular expression that recognizes log messages using
+https://regex101.com . Simply copy a log line into the test string input box
+on the site and then start editing the regular expression. When building the
+regular expression, you'll want to use named captures for the structured parts
+of the log message. Any raw message text should be matched by a captured named
+"body". Once you have a regex that matches the whole log message, you can use
+**lnav**'s "management CLI" to create a skeleton format file. The skeleton
+will be populated with the regular expression from the site and the test
+string, along with any unit tests, will be added to the "samples" list. The
+"regex101 import" management command is used to create the skeleton and has
+the following form:
+
+.. prompt:: bash
+
+ lnav -m regex101 import <regex101-url> <format-name> [<regex-name>]
+
+If the import was successful, the path to the new format file should be
+printed out. The skeleton will most likely need some changes to make it
+fully functional. For example, the :code:`kind` properties for captured values
+default to :code:`string`, but you'll want to change them to the appropriate
+type.
+
+Format File Reference
+^^^^^^^^^^^^^^^^^^^^^
+
+An **lnav** format file must contain a single JSON object, preferably with a
+:code:`$schema` property that refers to the
+`format-v1.schema <https://lnav.org/schemas/format-v1.schema.json>`_,
+like so:
+
+.. code-block:: json
+
+ {
+ "$schema": "https://lnav.org/schemas/format-v1.schema.json"
+ }
+
+Each format to be defined in the file should be a separate field in the top-level
+object. The field name should be the symbolic name of the format. This value
+will also be used as the SQL table name for the log. The value for each field
+should be another object with the following fields:
+
+ :title: The short and human-readable name for the format.
+ :description: A longer description of the format.
+ :url: A URL to the definition of the format.
+
+ :file-pattern: A regular expression used to match log file paths. Typically,
+ every file format will be tried during the detection process. This field
+ can be used to limit which files a format is applied to in case there is
+ a potential for conflicts.
+
+ .. _format_regex:
+
+ :regex: This object contains sub-objects that describe the message formats
+ to match in a plain-text log file. Each :code:`regex` MUST only match one
+ type of log message. It must not match log messages that are matched by
+ other regexes in this format. This uniqueness requirement is necessary
+ because **lnav** will "lock-on" to a regex and use it to match against
+ the next line in a file. So, if the regexes do not uniquely match each
+ type of log message, messages can be matched by the wrong regex. The
+ "lock-on" behavior is needed to avoid the performance hit of having to
+ try too many different regexes.
+
+ .. note:: Log files that contain JSON messages should not specify this field.
+
+ :pattern: The regular expression that should be used to match log messages.
+ The `PCRE2 <http://www.pcre.org>`_ library is used by **lnav** to do all
+ regular expression matching.
+
+ :module-format: If true, this regex will only be used to parse message
+ bodies for formats that can act as containers, such as syslog. Default:
+ false.
+
+ :json: True if each log line is JSON-encoded.
+
+ :line-format: An array that specifies the text format for JSON-encoded
+ log messages. Log files that are JSON-encoded will have each message
+ converted from the raw JSON encoding into this format. Each element
+ is either an object that defines which fields should be inserted into
+ the final message string and or a string constant that should be
+ inserted. For example, the following configuration will tranform each
+ log message object into a string that contains the timestamp, followed
+ by a space, and then the message body:
+
+ .. code-block:: json
+
+ [ { "field": "ts" }, " ", { "field": "msg" } ]
+
+ :field: The name or `JSON-Pointer <https://tools.ietf.org/html/rfc6901>`_
+ of the message field that should be inserted at this point in the
+ message. The special :code:`__timestamp__` field name can be used to
+ insert a human-readable timestamp. The :code:`__level__` field can be
+ used to insert the level name as defined by lnav.
+
+ .. tip::
+
+ Use a JSON-Pointer to reference nested fields. For example, to include
+ a "procname" property that is nested in a "details" object, you would
+ write the field reference as :code:`/details/procname`.
+
+ :min-width: The minimum width for the field. If the value for the field
+ in a given log message is shorter, padding will be added as needed to
+ meet the minimum-width requirement. (v0.8.2+)
+ :max-width: The maximum width for the field. If the value for the field
+ in a given log message is longer, the overflow algorithm will be applied
+ to try and shorten the field. (v0.8.2+)
+ :align: Specifies the alignment for the field, either "left" or "right".
+ If "left", padding to meet the minimum-width will be added on the right.
+ If "right", padding will be added on the left. (v0.8.2+)
+ :overflow: The algorithm used to shorten a field that is longer than
+ "max-width". The following algorithms are supported:
+
+ :abbrev: Removes all but the first letter in dotted text. For example,
+ "com.example.foo" would be shortened to "c.e.foo".
+ :truncate: Truncates any text past the maximum width.
+ :dot-dot: Cuts out the middle of the text and replaces it with two
+ dots (i.e. '..').
+
+ (v0.8.2+)
+ :timestamp-format: The timestamp format to use when displaying the time
+ for this log message. (v0.8.2+)
+ :default-value: The default value to use if the field could not be found
+ in the current log message. The built-in default is "-".
+ :text-transform: Transform the text in the field. Supported options are:
+ none, uppercase, lowercase, capitalize
+
+ :timestamp-field: The name of the field that contains the log message
+ timestamp. Defaults to "timestamp".
+
+ :timestamp-format: An array of timestamp formats using a subset of the
+ strftime conversion specification. The following conversions are
+ supported: %a, %b, %L, %M, %H, %I, %d, %e, %k, %l, %m, %p, %y, %Y, %S, %s,
+ %Z, %z. In addition, you can also use the following:
+
+ :%L: Milliseconds as a decimal number (range 000 to 999).
+ :%f: Microseconds as a decimal number (range 000000 to 999999).
+ :%N: Nanoseconds as a decimal number (range 000000000 to 999999999).
+ :%q: Seconds from the epoch as a hexidecimal number.
+ :%i: Milliseconds from the epoch.
+ :%6: Microseconds from the epoch.
+
+ :timestamp-divisor: For JSON logs with numeric timestamps, this value is used
+ to divide the timestamp by to get the number of seconds and fractional
+ seconds.
+
+ :subsecond-field: (v0.11.1+) The path to the property in a JSON-lines log
+ message that contains the sub-second time value
+
+ :subsecond-units: (v0.11.1+) The units of the subsecond-field property value.
+ The following values are supported:
+
+ :milli: for milliseconds
+ :micro: for microseconds
+ :nano: for nanoseconds
+
+ :ordered-by-time: (v0.8.3+) Indicates that the order of messages in the file
+ is time-based. Files that are not naturally ordered by time will be sorted
+ in order to display them in the correct order. Note that this sorting can
+ incur a performance penalty when tailing logs.
+
+ :level-field: The name of the regex capture group that contains the log
+ message level. Defaults to "level".
+
+ :body-field: The name of the field that contains the main body of the
+ message. Defaults to "body".
+
+ :opid-field: The name of the field that contains the "operation ID" of the
+ message. An "operation ID" establishes a thread of messages that might
+ correspond to a particular operation/request/transaction. The user can
+ press the 'o' or 'Shift+O' hotkeys to move forward/backward through the
+ list of messages that have the same operation ID. Note: For JSON-encoded
+ logs, the opid field can be a path (e.g. "foo/bar/opid") if the field is
+ nested in an object and it MUST be included in the "line-format" for the
+ 'o' hotkeys to work.
+
+ :module-field: The name of the field that contains the module identifier
+ that distinguishes messages from one log source from another. This field
+ should be used if this message format can act as a container for other
+ types of log messages. For example, an Apache access log can be sent to
+ syslog instead of written to a file. In this case, **lnav** will parse
+ the syslog message and then separately parse the body of the message to
+ determine the "sub" format. This module identifier is used to help
+ **lnav** quickly identify the format to use when parsing message bodies.
+
+ :hide-extra: A boolean for JSON logs that indicates whether fields not
+ present in the line-format should be displayed on their own lines.
+
+ :level: A mapping of error levels to regular expressions. During scanning
+ the contents of the capture group specified by *level-field* will be
+ checked against each of these regexes. Once a match is found, the log
+ message level will set to the corresponding level. The available levels,
+ in order of severity, are: **fatal**, **critical**, **error**,
+ **warning**, **stats**, **info**, **debug**, **debug2-5**, **trace**.
+ For JSON logs with exact numeric levels, the number for the corresponding
+ level can be supplied. If the JSON log format uses numeric ranges instead
+ of exact numbers, you can supply a pattern and the number found in the log
+ will be converted to a string for pattern-matching.
+
+ :multiline: If false, **lnav** will consider any log lines that do not
+ match one of the message patterns to be in error when checking files with
+ the '-C' option. This flag will not affect normal viewing operation.
+ Default: true.
+
+ :value: This object contains the definitions for the values captured by the
+ regexes.
+
+ :kind: The type of data that was captured **string**, **integer**,
+ **float**, **json**, **quoted**.
+ :collate: The name of the SQLite collation function for this value.
+ The standard SQLite collation functions can be used as well as the
+ ones defined by lnav, as described in :ref:`collators`.
+ :identifier: A boolean that indicates whether or not this field represents
+ an identifier and should be syntax colored.
+ :foreign-key: A boolean that indicates that this field is a key and should
+ not be graphed. This should only need to be set for integer fields.
+ :hidden: A boolean for log fields that indicates whether they should
+ be displayed. The behavior is slightly different for JSON logs and text
+ logs. For a JSON log, this property determines whether an extra line
+ will be added with the key/value pair. For text logs, this property
+ controls whether the value should be displayed by default or replaced
+ with an ellipsis.
+ :rewriter: A command to rewrite this field when pretty-printing log
+ messages containing this value. The command must start with ':', ';',
+ or '|' to signify whether it is a regular command, SQL query, or a script
+ to be executed. The other fields in the line are accessible in SQL by
+ using the ':' prefix. The text value of this field will then be replaced
+ with the result of the command when pretty-printing. For example, the
+ HTTP access log format will rewrite the status code field to include the
+ textual version (e.g. 200 (OK)) using the following SQL query:
+
+ .. code-block:: sql
+
+ ;SELECT :sc_status || ' (' || (
+ SELECT message FROM http_status_codes
+ WHERE status = :sc_status) || ') '
+
+ :tags: This object contains the tags that should automatically be added to
+ log messages.
+
+ :pattern: The regular expression evaluated over a line in the log file as
+ it is read in. If there is a match, the log message the line is a part
+ of will have this tag added to it.
+ :paths: This array contains objects that define restrictions on the file
+ paths that the tags will be applied to. The objects in this array can
+ contain:
+
+ :glob: A glob pattern to check against the log files read by lnav.
+
+ .. _format_sample:
+
+ :sample: A list of objects that contain sample log messages. All formats
+ must include at least one sample and it must be matched by one of the
+ included regexes. Each object must contain the following field:
+
+ :line: The sample message.
+ :level: The expected error level. An error will be raised if this level
+ does not match the level parsed by lnav for this sample message.
+
+ :highlights: This object contains the definitions for patterns to be
+ highlighted in a log message. Each entry should have a name and a
+ definition with the following fields:
+
+ :pattern: The regular expression to match in the log message body.
+ :color: The foreground color to use when highlighting the part of the
+ message that matched the pattern. If no color is specified, one will be
+ picked automatically. Colors can be specified using hexadecimal notation
+ by starting with a hash (e.g. #aabbcc) or using a color name as found
+ at http://jonasjacek.github.io/colors/.
+ :background-color: The background color to use when highlighting the part
+ of the message that matched the pattern. If no background color is
+ specified, black will be used. The background color is only considered
+ if a foreground color is specified.
+ :underline: If true, underline the part of the message that matched the
+ pattern.
+ :blink: If true, blink the part of the message that matched the pattern.
+
+Example format:
+
+.. code-block:: json
+
+ {
+ "$schema": "https://lnav.org/schemas/format-v1.schema.json",
+ "example_log" : {
+ "title" : "Example Log Format",
+ "description" : "Log format used in the documentation example.",
+ "url" : "http://example.com/log-format.html",
+ "regex" : {
+ "basic" : {
+ "pattern" : "^(?<timestamp>\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}Z)>>(?<level>\\w+)>>(?<component>\\w+)>>(?<body>.*)$"
+ }
+ },
+ "level-field" : "level",
+ "level" : {
+ "error" : "ERROR",
+ "warning" : "WARNING"
+ },
+ "value" : {
+ "component" : {
+ "kind" : "string",
+ "identifier" : true
+ }
+ },
+ "sample" : [
+ {
+ "line" : "2011-04-01T15:14:34.203Z>>ERROR>>core>>Shit's on fire yo!"
+ }
+ ]
+ }
+ }
+
+Patching an Existing Format
+---------------------------
+
+When loading log formats from files, **lnav** will overlay any new data over
+previously loaded data. This feature allows you to override existing value or
+append new ones to the format configurations. For example, you can separately
+add a new regex to the example log format given above by creating another file
+with the following contents:
+
+.. code-block:: json
+
+ {
+ "$schema": "https://lnav.org/schemas/format-v1.schema.json",
+ "example_log" : {
+ "regex" : {
+ "custom1" : {
+ "pattern" : "^(?<timestamp>\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}Z)<<(?<level>\\w+)--(?<component>\\w+)>>(?<body>.*)$"
+ }
+ },
+ "sample" : [
+ {
+ "line" : "2011-04-01T15:14:34.203Z<<ERROR--core>>Shit's on fire yo!"
+ }
+ ]
+ }
+ }
+
+.. _scripts:
+
+Scripts
+-------
+
+Format directories may also contain :file:`.sql` and :file:`.lnav` files to help automate
+log file analysis. The SQL files are executed on startup to create any helper
+tables or views and the '.lnav' script files can be executed using the pipe
+hotkey :kbd:`|`. For example, **lnav** includes a "partition-by-boot" script that
+partitions the log view based on boot messages from the Linux kernel. A script
+can have a mix of SQL and **lnav** commands, as well as include other scripts.
+The type of statement to execute is determined by the leading character on a
+line: a semi-colon begins a SQL statement; a colon starts an **lnav** command;
+and a pipe :code:`|` denotes another script to be executed. Lines beginning with a
+hash are treated as comments. The following variables are defined in a script:
+
+.. envvar:: #
+
+ The number of arguments passed to the script.
+
+.. envvar:: __all__
+
+ A string containing all the arguments joined by a single space.
+
+.. envvar:: 0
+
+ The path to the script being executed.
+
+.. envvar:: 1-N
+
+ The arguments passed to the script.
+
+Remember that you need to use the :ref:`:eval<eval>` command when referencing
+variables in most **lnav** commands. Scripts can provide help text to be
+displayed during interactive usage by adding the following tags in a comment
+header:
+
+ :@synopsis: The synopsis should contain the name of the script and any
+ parameters to be passed. For example::
+
+ # @synopsis: hello-world <name1> [<name2> ... <nameN>]
+
+ :@description: A one-line description of what the script does. For example::
+
+ # @description: Say hello to the given names.
+
+
+
+.. tip::
+
+ The :ref:`:eval<eval>` command can be used to do variable substitution for
+ commands that do not natively support it. For example, to substitute the
+ variable, :code:`pattern`, in a :ref:`:filter-out<filter_out>` command:
+
+ .. code-block:: lnav
+
+ :eval :filter-out ${pattern}
+
+VSCode Extension
+^^^^^^^^^^^^^^^^
+
+The `lnav VSCode Extension <https://marketplace.visualstudio.com/items?itemName=lnav.lnav>`_
+can be installed to add syntax highlighting to lnav scripts.
+
+Installing Formats
+------------------
+
+File formats are loaded from subdirectories in :file:`/etc/lnav/formats` and
+:file:`~/.lnav/formats/`. You can manually create these subdirectories and
+copy the format files into there. Or, you can pass the '-i' option to **lnav**
+to automatically install formats from the command-line. For example:
+
+.. code-block:: bash
+
+ $ lnav -i myformat.json
+ info: installed: /home/example/.lnav/formats/installed/myformat_log.json
+
+Format files installed using this method will be placed in the :file:`installed`
+subdirectory and named based on the first format name found in the file.
+
+You can also install formats from git repositories by passing the repository's
+clone URL. A standard set of repositories is maintained at
+(https://github.com/tstack/lnav-config) and can be installed by passing 'extra'
+on the command line, like so:
+
+.. prompt:: bash
+
+ lnav -i extra
+
+These repositories can be updated by running **lnav** with the '-u' flag.
+
+Format files can also be made executable by adding a shebang (#!) line to the
+top of the file, like so::
+
+ #! /usr/bin/env lnav -i
+ {
+ "myformat_log" : ...
+ }
+
+Executing the format file should then install it automatically:
+
+.. code-block:: bash
+
+ $ chmod ugo+rx myformat.json
+ $ ./myformat.json
+ info: installed: /home/example/.lnav/formats/installed/myformat_log.json
+
+.. _format_order:
+
+Format Order When Scanning a File
+---------------------------------
+
+When **lnav** loads a file, it tries each log format against the first 15,000
+lines [#]_ of the file trying to find a match. When a match is found, that log
+format will be locked in and used for the rest of the lines in that file.
+Since there may be overlap between formats, **lnav** performs a test on
+startup to determine which formats match each others sample lines. Using
+this information it will create an ordering of the formats so that the more
+specific formats are tried before the more generic ones. For example, a
+format that matches certain syslog messages will match its own sample lines,
+but not the ones in the syslog samples. On the other hand, the syslog format
+will match its own samples and those in the more specific format. You can
+see the order of the format by enabling debugging and checking the **lnav**
+log file for the "Format order" message:
+
+.. prompt:: bash
+
+ lnav -d /tmp/lnav.log
+
+.. [#] The maximum number of lines to check can be configured. See the
+ :ref:`tuning` section for more details.
diff --git a/docs/source/group_concat-help.png b/docs/source/group_concat-help.png
new file mode 100644
index 0000000..f7b6f08
--- /dev/null
+++ b/docs/source/group_concat-help.png
Binary files differ
diff --git a/docs/source/hotkey-tips.png b/docs/source/hotkey-tips.png
new file mode 100644
index 0000000..5b78686
--- /dev/null
+++ b/docs/source/hotkey-tips.png
Binary files differ
diff --git a/docs/source/hotkeys.rst b/docs/source/hotkeys.rst
new file mode 100644
index 0000000..f91c1e8
--- /dev/null
+++ b/docs/source/hotkeys.rst
@@ -0,0 +1,289 @@
+.. _hotkeys:
+
+Hotkey Reference
+================
+
+This reference covers the keys used to control **lnav**. Consult the `built-in
+help <https://github.com/tstack/lnav/blob/master/src/help.txt>`_ in **lnav** for
+a more detailed explanation of each key.
+
+Spatial Navigation
+------------------
+
+The majority of these hotkeys should be available in all views.
+
+.. list-table::
+ :header-rows: 1
+ :widths: 6 6 6 20
+
+ * - Keypress
+ -
+ -
+ - Command
+ * - :kbd:`Space`
+ - :kbd:`PgDn`
+ -
+ - Down a page
+ * - :kbd:`b`
+ - :kbd:`Backspace`
+ - :kbd:`PgUp`
+ - Up a page
+ * - :kbd:`j`
+ - :kbd:`↓`
+ -
+ - Down a line
+ * - :kbd:`k`
+ - :kbd:`↑`
+ -
+ - Up a line
+ * - :kbd:`h`
+ - :kbd:`←`
+ -
+ - Left half a page. In the log view, pressing left while at the start of
+ the message text will reveal the source file name for each line.
+ Pressing again will reveal the full path.
+ * - :kbd:`Shift` + :kbd:`h`
+ - :kbd:`Shift` + :kbd:`←`
+ -
+ - Left ten columns
+ * - :kbd:`l`
+ - :kbd:`→`
+ -
+ - Right half a page
+ * - :kbd:`Shift` + :kbd:`l`
+ - :kbd:`Shift` + :kbd:`→`
+ -
+ - Right ten columns
+ * - :kbd:`Home`
+ - :kbd:`g`
+ -
+ - Top of the view
+ * - :kbd:`End`
+ - :kbd:`G`
+ -
+ - Bottom of the view
+ * - :kbd:`e`
+ - :kbd:`Shift` + :kbd:`e`
+ -
+ - Next/previous error
+ * - :kbd:`w`
+ - :kbd:`Shift` + :kbd:`w`
+ -
+ - Next/previous warning
+ * - :kbd:`n`
+ - :kbd:`Shift` + :kbd:`n`
+ -
+ - Next/previous search hit
+ * - :kbd:`>`
+ - :kbd:`<`
+ -
+ - Next/previous search hit (horizontal)
+ * - :kbd:`f`
+ - :kbd:`Shift` + :kbd:`f`
+ -
+ - Next/previous file
+ * - :kbd:`u`
+ - :kbd:`Shift` + :kbd:`u`
+ -
+ - Next/previous bookmark
+ * - :kbd:`o`
+ - :kbd:`Shift` + :kbd:`o`
+ -
+ - Forward/backward through log messages with a matching "opid" field
+ * - :kbd:`s`
+ - :kbd:`Shift` + :kbd:`s`
+ -
+ - Next/previous slow down in the log message rate
+ * - :kbd:`{`
+ - :kbd:`}`
+ -
+ - Previous/next location in history
+
+Chronological Navigation
+------------------------
+
+These hotkeys are only functional on views that are time-based, like the log
+view or the histogram view.
+
+.. list-table::
+ :header-rows: 1
+ :widths: 5 5 20
+
+ * - Keypress
+ -
+ - Command
+ * - :kbd:`d`
+ - :kbd:`Shift` + :kbd:`d`
+ - Forward/backward 24 hours
+ * - :kbd:`1` - :kbd:`6`
+ - :kbd:`Shift` + :kbd:`1` - :kbd:`6`
+ - Next/previous n'th ten minute of the hour
+ * - :kbd:`7`
+ - :kbd:`8`
+ - Previous/next minute
+ * - :kbd:`0`
+ - :kbd:`Shift` + :kbd:`0`
+ - Next/previous day
+ * - :kbd:`r`
+ - :kbd:`Shift` + :kbd:`r`
+ - Forward/backward by the relative time that was last used with the goto command.
+
+Breadcrumb Navigation
+---------------------
+
+The following hotkeys are related to the breadcrumb bar that is below the top
+status bar.
+
+.. list-table::
+ :header-rows: 1
+ :widths: 5 20
+
+ * - Keypress
+ - Description
+ * - :kbd:`ENTER`
+ - Focus on the breadcrumb bar. Or, if the bar is currently focused,
+ accept the selected value and drop focus.
+ * - :kbd:`Escape`
+ - Drop focus on the breadcrumb bar.
+ * - :kbd:`←`
+ - Select the crumb to the left. If the first crumb is selected, the
+ selection will wrap around to the last crumb.
+ * - :kbd:`→`
+ - Accept the current value, which might mean navigating to the value in
+ the view, then selecting the crumb to the right.
+ * - :kbd:`Ctrl` + :kbd:`a`
+ - Select the first crumb.
+ * - :kbd:`Ctrl` + :kbd:`e`
+ - Select the last crumb.
+ * - :kbd:`↓`
+ - Select the next value in the crumb dropdown.
+ * - :kbd:`↑`
+ - Select the previous value in the crumb dropdown.
+ * - :kbd:`Home`
+ - Select the first value in the crumb dropdown.
+ * - :kbd:`End`
+ - Select the last value in the crumb dropdown.
+
+While a crumb is selected, you can perform a fuzzy search on the possible
+values by typing in the value you are interested in.
+
+.. _hotkeys_bookmarks:
+
+Bookmarks
+---------
+
+.. list-table::
+ :header-rows: 1
+ :widths: 5 20
+
+ * - Keypress
+ - Command
+ * - :kbd:`m`
+ - Mark/unmark the top line
+ * - :kbd:`Shift` + :kbd:`m`
+ - Mark/unmark the range of lines from the last marked to the top
+ * - :kbd:`Shift` + :kbd:`j`
+ - Mark/unmark the next line after the previously marked
+ * - :kbd:`Shift` + :kbd:`k`
+ - Mark/unmark the previous line
+ * - :kbd:`c`
+ - Copy marked lines to the clipboard
+ * - :kbd:`Shift` + :kbd:`c`
+ - Clear marked lines
+
+.. _hotkeys_display:
+
+Display
+-------
+
+.. list-table::
+ :header-rows: 1
+ :widths: 5 20
+
+ * - Keypress
+ - Command
+ * - :kbd:`?`
+ - View/leave builtin help
+ * - :kbd:`q`
+ - Return to the previous view/quit
+ * - :kbd:`Shift` + :kbd:`q`
+ - Return to the previous view/quit while matching the top times of the two views
+ * - :kbd:`a`
+ - Restore the view that was previously popped with 'q/Q'
+ * - :kbd:`Shift` + :kbd:`a`
+ - Restore the view that was previously popped with 'q/Q' and match the top times of the views
+ * - :kbd:`Shift` + :kbd:`p`
+ - Switch to/from the pretty-printed view of the displayed log or text files
+ * - :kbd:`Shift` + :kbd:`t`
+ - Display elapsed time between lines
+ * - :kbd:`t`
+ - Switch to/from the text file view
+ * - :kbd:`i`
+ - Switch to/from the histogram view
+ * - :kbd:`Shift` + :kbd:`i`
+ - Switch to/from the histogram view
+ * - :kbd:`v`
+ - Switch to/from the SQL result view
+ * - :kbd:`Shift` + :kbd:`v`
+ - Switch to/from the SQL result view and move to the corresponding in the
+ log_line column
+ * - :kbd:`p`
+ - Toggle the display of the log parser results
+ * - :kbd:`Tab`
+ - In the log/text views, focus on the configuration panel for editing
+ filters and examining the list of loaded files. In the SQL result view,
+ cycle through columns to display as bar graphs
+ * - :kbd:`Ctrl` + :kbd:`l`
+ - Switch to lo-fi mode. The displayed log lines will be dumped to the
+ terminal without any decorations so they can be copied easily.
+ * - :kbd:`Ctrl` + :kbd:`w`
+ - Toggle word-wrap.
+ * - :kbd:`Ctrl` + :kbd:`p`
+ - Show/hide the data preview panel that may be opened when entering
+ commands or SQL queries.
+ * - :kbd:`Ctrl` + :kbd:`f`
+ - Toggle the enabled/disabled state of all filters in the current view.
+ * - :kbd:`x`
+ - Toggle the hiding of log message fields. The hidden fields will be
+ replaced with three bullets and highlighted in yellow.
+ * - :kbd:`=`
+ - Pause/unpause loading of new file data.
+
+Session
+-------
+
+.. list-table::
+ :header-rows: 1
+ :widths: 5 20
+
+ * - Keypress
+ - Command
+ * - :kbd:`Ctrl` + :kbd:`R`
+ - Reset the current :ref:`session<sessions>` state. The session state
+ includes things like filters, bookmarks, and hidden fields.
+
+Query Prompts
+-------------
+
+.. list-table::
+ :header-rows: 1
+ :widths: 5 20
+
+ * - Keypress
+ - Command
+ * - :kbd:`/`
+ - Search for lines matching a regular expression
+ * - :kbd:`;`
+ - Open the :ref:`sql-ext` to execute SQL statements/queries
+ * - :kbd:`:`
+ - Execute an internal command, see :ref:`commands` for more information
+ * - :kbd:`\|`
+ - Execute an lnav script located in a format directory
+ * - :kbd:`Ctrl` + :kbd:`]`
+ - Abort the prompt
+
+Customizing
+-----------
+
+You can customize the behavior of hotkeys by defining your own keymaps.
+Consult the :ref:`Keymaps<keymaps>` configuration section for more information.
diff --git a/docs/source/howitworks.rst b/docs/source/howitworks.rst
new file mode 100644
index 0000000..d111fd0
--- /dev/null
+++ b/docs/source/howitworks.rst
@@ -0,0 +1,13 @@
+
+.. _howitworks:
+
+How It Works
+============
+
+"Magic"
+
+Internal Architecture
+---------------------
+
+The `ARCHITECTURE.md <https://github.com/tstack/lnav/blob/master/ARCHITECTURE.md>`_
+file in the source tree contains some information about lnav's internals.
diff --git a/docs/source/index.rst b/docs/source/index.rst
new file mode 100644
index 0000000..d94c0eb
--- /dev/null
+++ b/docs/source/index.rst
@@ -0,0 +1,35 @@
+Welcome to lnav's documentation!
+================================
+
+The `Log File Navigator <http://lnav.org>`_ (**lnav**) is an advanced log file
+viewer for the console.
+
+Contents:
+
+.. toctree::
+ :maxdepth: 2
+
+ intro
+ ui
+ hotkeys
+ cli
+ usage
+ cookbook
+ config
+ formats
+ sessions
+ commands
+ sqlext
+ sqltab
+ events
+ data
+ howitworks
+ faq
+
+
+Indices and tables
+==================
+
+* :ref:`genindex`
+* :ref:`modindex`
+* :ref:`search`
diff --git a/docs/source/intro.rst b/docs/source/intro.rst
new file mode 100644
index 0000000..ae3164c
--- /dev/null
+++ b/docs/source/intro.rst
@@ -0,0 +1,131 @@
+Introduction
+============
+
+The Log File Navigator, **lnav**, is an advanced log file viewer for the
+terminal. It provides an :ref:`easy-to-use interface<ui>` for monitoring and
+analyzing your log files with little to no setup. Simply point **lnav** at
+your log files and it will automatically detect the :ref:`log_formats`, index
+their contents, and display a combined view of all log messages. You can
+navigate through your logs using a variety of :ref:`hotkeys<hotkeys>`.
+:ref:`Commands<commands>` give you additional control over **lnav**'s behavior
+for doing things like applying filters, tagging messages, and more. You can
+then analyze your log messages using the :ref:`sql-ext`.
+
+Dependencies
+------------
+
+When compiling from source, the following dependencies are required:
+
+* `NCurses <http://www.gnu.org/s/ncurses/>`_
+* `PCRE2 <http://www.pcre.org>`_
+* `SQLite <http://www.sqlite.org>`_
+* `ZLib <http://wwww.zlib.net>`_
+* `Bzip2 <http://www.bzip.org>`_
+* `Readline <http://www.gnu.org/s/readline>`_
+* `libcurl <https://curl.haxx.se>`_
+* `libarchive <https://libarchive.org>`_
+
+Installation
+------------
+
+Check the `downloads page <http://lnav.org/downloads>`_ to see if there are
+packages for your operating system. To compile from source, use the following
+commands:
+
+.. prompt:: bash
+
+ ./configure
+ make
+ sudo make install
+
+
+Viewing Logs
+------------
+
+The arguments to **lnav** are the log files, directories, or URLs to be viewed.
+For example, to view all of the CUPS logs on your system:
+
+.. prompt:: bash
+
+ lnav /var/log/cups
+
+The formats of the logs are determined automatically and indexed on-the-fly.
+See :ref:`log_formats` for a listing of the predefined formats and how to
+define your own.
+
+If no arguments are given, **lnav** will try to open the syslog file on your
+system:
+
+.. prompt:: bash
+
+ lnav
+
+
+Setup
+-----
+
+After starting **lnav**, you might want to set the
+:ref:`configuration options<Configuration>` mentioned below. Configuration in
+**lnav** is done using the :code:`:config` command. To change a configuration
+option, start by pressing :kbd:`:` to enter the command prompt. Then,
+type "config" followed by the option name and value.
+
+.. note::
+
+ Tab-completion is available for these configuration options and, in some
+ cases, their values as well.
+
+
+Keymap
+^^^^^^
+
+The keymap defines the mapping from :ref:`hotkeys<hotkeys>` to commands to
+execute. The default mapping is for "U.S." keyboards. The following command
+can be used to change the keymap:
+
+.. code-block:: lnav
+
+ :config /ui/keymap <keymap-name>
+
+The builtin keymaps are:
+
+ :de: `German <https://github.com/tstack/lnav/blob/master/src/keymaps/de-keymap.json>`_
+ :fr: `French <https://github.com/tstack/lnav/blob/master/src/keymaps/fr-keymap.json>`_
+ :sv: `Swedish <https://github.com/tstack/lnav/blob/master/src/keymaps/sv-keymap.json>`_
+ :uk: `United Kingdom <https://github.com/tstack/lnav/blob/master/src/keymaps/uk-keymap.json>`_
+ :us: `United States <https://github.com/tstack/lnav/blob/master/src/keymaps/us-keymap.json>`_
+
+To create or customize a keymap, consult the :ref:`keymaps` section.
+
+
+Theme
+^^^^^
+
+The visual styling of **lnav** can be customized using a theme. The following
+command can be used to the change the theme:
+
+.. code-block:: lnav
+
+ :config /ui/theme <theme-name>
+
+The builtin themes are:
+`default <https://github.com/tstack/lnav/blob/master/src/themes/default-theme.json>`_,
+`eldar <https://github.com/tstack/lnav/blob/master/src/themes/eldar.json>`_,
+`grayscale <https://github.com/tstack/lnav/blob/master/src/themes/grayscale.json>`_,
+`monocai <https://github.com/tstack/lnav/blob/master/src/themes/monocai.json>`_,
+`night-owl <https://github.com/tstack/lnav/blob/master/src/themes/night-owl.json>`_,
+`solarized-dark <https://github.com/tstack/lnav/blob/master/src/themes/solarized-dark.json>`_,
+and
+`solarized-light <https://github.com/tstack/lnav/blob/master/src/themes/default-theme.json>`_.
+
+To create or customize a theme, consult the :ref:`themes` section.
+
+
+Log Formats
+^^^^^^^^^^^
+
+In order for **lnav** to understand your log files, it needs to told how to
+parse the log messages using a log format definition. There are many log
+formats builtin and **lnav** will automatically determine the best format to
+use. In case your log file is not recognized, consult the :ref:`log_formats`
+section for information on how to create a format.
diff --git a/docs/source/key-encoding-prompt.png b/docs/source/key-encoding-prompt.png
new file mode 100644
index 0000000..fe63412
--- /dev/null
+++ b/docs/source/key-encoding-prompt.png
Binary files differ
diff --git a/docs/source/lnav-breadcrumbs-help.png b/docs/source/lnav-breadcrumbs-help.png
new file mode 100644
index 0000000..b3b80d0
--- /dev/null
+++ b/docs/source/lnav-breadcrumbs-help.png
Binary files differ
diff --git a/docs/source/lnav-config-header.png b/docs/source/lnav-config-header.png
new file mode 100644
index 0000000..079b9c2
--- /dev/null
+++ b/docs/source/lnav-config-header.png
Binary files differ
diff --git a/docs/source/lnav-files-panel.png b/docs/source/lnav-files-panel.png
new file mode 100644
index 0000000..7aed793
--- /dev/null
+++ b/docs/source/lnav-files-panel.png
Binary files differ
diff --git a/docs/source/lnav-filters-panel.png b/docs/source/lnav-filters-panel.png
new file mode 100644
index 0000000..788e63f
--- /dev/null
+++ b/docs/source/lnav-filters-panel.png
Binary files differ
diff --git a/docs/source/lnav-spectro-cpu-pct.png b/docs/source/lnav-spectro-cpu-pct.png
new file mode 100644
index 0000000..a9df2b6
--- /dev/null
+++ b/docs/source/lnav-spectro-cpu-pct.png
Binary files differ
diff --git a/docs/source/lnav-ui.png b/docs/source/lnav-ui.png
new file mode 100644
index 0000000..2dcad66
--- /dev/null
+++ b/docs/source/lnav-ui.png
Binary files differ
diff --git a/docs/source/open-error.png b/docs/source/open-error.png
new file mode 100644
index 0000000..6454b2d
--- /dev/null
+++ b/docs/source/open-error.png
Binary files differ
diff --git a/docs/source/open-help.png b/docs/source/open-help.png
new file mode 100644
index 0000000..a11656a
--- /dev/null
+++ b/docs/source/open-help.png
Binary files differ
diff --git a/docs/source/open-preview.png b/docs/source/open-preview.png
new file mode 100644
index 0000000..b888376
--- /dev/null
+++ b/docs/source/open-preview.png
Binary files differ
diff --git a/docs/source/query-results.png b/docs/source/query-results.png
new file mode 100644
index 0000000..14ae5b8
--- /dev/null
+++ b/docs/source/query-results.png
Binary files differ
diff --git a/docs/source/sessions.rst b/docs/source/sessions.rst
new file mode 100644
index 0000000..e97e598
--- /dev/null
+++ b/docs/source/sessions.rst
@@ -0,0 +1,23 @@
+
+.. _sessions:
+
+Sessions
+========
+
+Session information is stored automatically for the set of files that were
+passed in on the command-line and reloaded the next time **lnav** is executed.
+The information currently stored is:
+
+* Position within the files being viewed.
+* Active searches for each view.
+* :ref:`Log filters<filtering>`.
+* :ref:`Highlights<highlight>`.
+* :ref:`Hidden files<hide_file>`.
+* :ref:`Hidden fields<hide_fields>`.
+
+Bookmarks and log-time adjustments are stored separately on a per-file basis.
+Note that the bookmarks are associated with files based on the content of the
+first line of the file so that they are preserved even if the file has been
+moved from its current location.
+
+Session data is stored in the :file:`~/.lnav` directory.
diff --git a/docs/source/sql-help.png b/docs/source/sql-help.png
new file mode 100644
index 0000000..44901a4
--- /dev/null
+++ b/docs/source/sql-help.png
Binary files differ
diff --git a/docs/source/sqlext.rst b/docs/source/sqlext.rst
new file mode 100644
index 0000000..e0d7543
--- /dev/null
+++ b/docs/source/sqlext.rst
@@ -0,0 +1,184 @@
+
+.. _sql-ext:
+
+SQLite Interface
+================
+
+Log analysis in **lnav** can be done using the SQLite interface. Log messages
+can be accessed via `virtual tables <https://www.sqlite.org/vtab.html>`_ that
+are created for each file format. The tables have the same name as the log
+format and each message is its own row in the table. For example, given the
+following log message from an Apache access log::
+
+ 127.0.0.1 - frank [10/Oct/2000:13:55:36 -0700] "GET /apache_pb.gif HTTP/1.0" 200 2326
+
+These columns would be available for its row in the :code:`access_log` table:
+
+.. csv-table::
+ :class: query-results
+ :header-rows: 1
+
+ log_line,log_part,log_time,log_idle_msecs,log_level,log_mark,log_comment,log_tags,log_filters,c_ip,cs_method,cs_referer,cs_uri_query,cs_uri_stem,cs_user_agent,cs_username,cs_version,sc_bytes,sc_status
+ 0,<NULL>,2000-10-10 13:55:36.000,0,info,1,<NULL>,<NULL>,<NULL>,127.0.0.1,GET,<NULL>,<NULL>,/apache_pb.gif,<NULL>,frank,HTTP/1.0,2326,200
+
+.. note:: Some columns are hidden by default to reduce the amount of noise in
+ results, but they can still be accessed when explicitly used. The hidden
+ columns are: :code:`log_path`, :code:`log_text`, :code:`log_body`, and
+ :code:`log_raw_text`.
+
+You can activate the SQL prompt by pressing the :kbd:`;` key. At the
+prompt, you can start typing in the desired SQL statement and/or double-tap
+:kbd:`TAB` to activate auto-completion. A help window will appear above
+the prompt to guide you in the usage of SQL keywords and functions.
+
+.. figure:: sql-help.png
+ :align: center
+
+ Screenshot of the online help for the SQL prompt.
+
+.. figure:: group_concat-help.png
+ :align: center
+
+ Screenshot of the online help for the :code:`group_concat()` function.
+
+A simple query to perform on an Apache access log might be to get the average
+and maximum number of bytes returned by the server, grouped by IP address:
+
+.. code-block:: custsqlite
+
+ ;SELECT c_ip, avg(sc_bytes), max(sc_bytes) FROM access_log GROUP BY c_ip
+
+After pressing :kbd:`Enter`, SQLite will execute the query using **lnav**'s
+virtual table implementation to extract the data directly from the log files.
+Once the query has finished, the main window will switch to the DB view to
+show the results. Press :kbd:`q` to return to the log view and press :kbd:`v`
+to return to the log view. If the SQL results contain a
+:code:`log_line` column, you can press to :kbd:`Shift` + :kbd:`V` to
+switch between the DB view and the log
+
+.. figure:: query-results.png
+ :align: center
+
+ Screenshot of the SQL results view.
+
+The DB view has the following display features:
+
+* Column headers stick to the top of the view when scrolling.
+* A stacked bar chart of the numeric column values is displayed underneath the
+ rows. Pressing :kbd:`TAB` will cycle through displaying no columns, each
+ individual column, or all columns.
+* JSON columns in the top row can be pretty-printed by pressing :kbd:`p`.
+ The display will show the value and JSON-Pointer path that can be passed to
+ the `jget`_ function.
+
+
+Log Tables
+----------
+
+Each log format has its own database table that can be used to access log
+messages that match that format. The table name is the same as the format
+name, for example, the :code:`syslog_log` format will have a table that is
+also named :code:`syslog_log`. There is also an :code:`all_logs` table
+that provides access to all messages from all formats.
+
+.. note:: Only the displayed log messages are reflected in the SQLite
+ interface. Any log messages that have been filtered out are not
+ accessible.
+
+The columns in the log tables are made up of several builtins along with
+the values captured by the log format specification. Use the :code:`.schema`
+command in the SQL prompt to examine a dump of the current database schema.
+
+The following columns are builtin and included in a :code:`SELECT *`:
+
+ :log_line: The line number for the message in the log view.
+ :log_part: The partition the message is in. This column can be changed by
+ an :code:`UPDATE` or the :ref:`:parition-name<partition_name>` command.
+ :log_time: The adjusted timestamp for the log message. This time can differ
+ from the log message's time stamp if it arrived out-of-order and the log
+ format expects log files to be time-ordered.
+ :log_actual_time: The log messages original timestamp in the file.
+ :log_idle_msecs: The difference in time between this messages and the
+ previous. The unit of time is milliseconds.
+ :log_level: The log message level.
+ :log_mark: True if the log message was marked by the user.
+ :log_comment: The comment for the message. This column can be changed by
+ an :code:`UPDATE` or the :ref:`:comment<comment>` command.
+ :log_tags: A JSON list of tags for the message. This column can be changed
+ by an :code:`UPDATE` or the :ref:`:tag<tag>` command.
+ :log_filters: A JSON list of filter IDs that matched this message
+
+The following columns are builtin and are hidden, so they will *not* be
+included in a :code:`SELECT *`:
+
+ :log_time_msecs: The adjusted timestamp for the log message as the number of
+ milliseconds from the epoch. This column can be more efficient to use for
+ time-related operations, like :ref:`timeslice()<timeslice>`.
+ :log_path: The path to the log file this message is from.
+ :log_text: The full text of the log message.
+ :log_body: The body of the log message.
+ :log_raw_text: The raw text of this message from the log file. In this case
+ of JSON and CSV logs, this will be the exact line of JSON-Line and CSV
+ text from the file.
+
+Extensions
+----------
+
+To make it easier to analyze log data from within **lnav**, there are several
+built-in extensions that provide extra functions and collators beyond those
+`provided by SQLite <http://www.sqlite.org/lang_corefunc.html>`_. The majority
+of the functions are from the
+`extensions-functions.c <http://www.sqlite.org/contrib>`_ file available from
+the `sqlite.org <http://sqlite.org>`_ web site.
+
+.. tip:: You can include a SQLite database file on the command-line and use
+ **lnav**'s interface to perform queries. The database will be attached with
+ a name based on the database file name.
+
+Commands
+--------
+
+A SQL command is an internal macro implemented by lnav.
+
+* .schema - Open the schema view. This view contains a dump of the schema
+ for the internal tables and any tables in attached databases.
+* .msgformats - Executes a canned query that groups and counts log messages by
+ the format of their message bodies. This command can be useful for quickly
+ finding out the types of messages that are most common in a log file.
+
+Variables
+---------
+
+The following variables are available in SQL statements:
+
+* $LINES - The number of lines in the terminal window.
+* $COLS - The number of columns in the terminal window.
+
+Environment
+-----------
+
+Environment variables can be accessed in queries using the usual syntax of
+:code:`$VAR_NAME`. For example, to read the value of the "USER" variable, you
+can write:
+
+.. code-block:: custsqlite
+
+ ;SELECT $USER
+
+.. _collators:
+
+Collators
+---------
+
+* **naturalcase** - Compare strings "naturally" so that number values in the
+ string are compared based on their numeric value and not their character
+ values. For example, "foo10" would be considered greater than "foo2".
+* **naturalnocase** - The same as naturalcase, but case-insensitive.
+* **ipaddress** - Compare IPv4/IPv6 addresses.
+
+Reference
+---------
+
+The following is a reference of the SQL syntax and functions that are available:
+
+.. include:: ../../src/internals/sql-ref.rst
diff --git a/docs/source/sqltab.rst b/docs/source/sqltab.rst
new file mode 100644
index 0000000..ca2fefb
--- /dev/null
+++ b/docs/source/sqltab.rst
@@ -0,0 +1,217 @@
+.. _sql-tab:
+
+SQLite Tables Reference
+=======================
+
+In addition to the tables generated for each log format, **lnav** includes
+the following tables/views:
+
+* `environ`_
+* `lnav_events`_
+* `lnav_file`_
+* `lnav_user_notifications`_
+* `lnav_views`_
+* `lnav_views_echo`_
+* `lnav_view_stack`_
+* `lnav_view_filters`_
+* `lnav_view_filter_stats`_
+* `lnav_view_filters_and_stats`_
+* `all_logs`_
+* `http_status_codes`_
+* `regexp_capture(<string>, <regex>)`_
+
+These extra tables provide useful information and can let you manipulate
+**lnav**'s internal state. You can get a dump of the entire database schema
+by executing the '.schema' SQL command, like so::
+
+ ;.schema
+
+environ
+-------
+
+The **environ** table gives you access to the **lnav** process' environment
+variables. You can SELECT, INSERT, and UPDATE environment variables, like
+so:
+
+.. code-block:: custsqlite
+
+ ;SELECT * FROM environ WHERE name = 'SHELL'
+ name value
+ SHELL /bin/tcsh
+
+ ;UPDATE environ SET value = '/bin/sh' WHERE name = 'SHELL'
+
+Environment variables can be used to store simple values or pass values
+from **lnav**'s SQL environment to **lnav**'s commands. For example, the
+"open" command will do variable substitution, so you can insert a variable
+named "FILENAME" and then open it in **lnav** by referencing it with
+"$FILENAME":
+
+.. code-block:: custsqlite
+
+ ;INSERT INTO environ VALUES ('FILENAME', '/path/to/file')
+ :open $FILENAME
+
+
+.. _table_lnav_events:
+
+lnav_events
+-----------
+
+The **lnav_events** table allows you to react to events that occur while
+**lnav** is running using SQLite triggers. For example, when a file is
+opened, a row is inserted into the :code:`lnav_events` table that contains
+a timestamp and a JSON object with the event ID and the path of the file.
+The following columns are available in this table:
+
+ :ts: The timestamp of the event.
+ :content: A JSON object that contains the event information. See the
+ :ref:`event_reference` for more information about the types
+ of events that are available.
+
+lnav_file
+---------
+
+The **lnav_file** table allows you to examine and perform limited updates to
+the metadata for the files that are currently loaded into **lnav**. The
+following columns are available in this table:
+
+ :device: The device the file is stored on.
+ :inode: The inode for the file on the device.
+ :filepath: If this is a real file, it will be the absolute path. Otherwise,
+ it is a symbolic name. If it is a symbolic name, it can be UPDATEd so that
+ this file will be considered when saving and loading session information.
+ :format: The log file format for the file.
+ :lines: The number of lines in the file.
+ :time_offset: The millisecond offset for timestamps. This column can be
+ UPDATEd to change the offset of timestamps in the file.
+
+.. _table_lnav_user_notifications:
+
+lnav_user_notifications
+-----------------------
+
+The :code:`lnav_user_notifications` table allows you to display a custom message
+in the top-right corner of the UI. For example, to display "Hello, World!",
+you can enter:
+
+.. code-block:: custsqlite
+
+ ;REPLACE INTO lnav_user_notifications (message) VALUES ('Hello, World!')
+
+There are additional columns to have finer control of what is displayed and
+when:
+
+ :id: The unique ID for the message, defaults to "org.lnav.user". This is
+ the primary key for the table, so more than one type of message is not
+ allowed.
+ :priority: The priority of the message. Higher priority messages will be
+ displayed until they are cleared or are expired.
+ :created: The time the message was created.
+ :expiration: The time when the message should expire or NULL if it should
+ not automatically expire.
+ :views: A JSON array of view names where the message is applicable or NULL
+ if the message should be shown in all views.
+ :message: The message itself.
+
+This table will most likely be used in combination with :ref:`Events` and the
+`lnav_views_echo`_ table.
+
+lnav_views
+----------
+
+The **lnav_views** table allows you to SELECT and UPDATE information related
+to **lnav**'s "views" (e.g. log, text, ...). The following columns are
+available in this table:
+
+ :name: The name of the view.
+ :top: The line number at the top of the view. This value can be UPDATEd to
+ move the view to the given line.
+ :left: The left-most column number to display. This value can be UPDATEd to
+ move the view left or right.
+ :height: The number of lines that are displayed on the screen.
+ :inner_height: The number of lines of content being displayed.
+ :top_time: The timestamp of the top line in the view or NULL if the view is
+ not time-based. This value can be UPDATEd to move the view to the given
+ time.
+ :paused: Indicates if the view is paused and will not load new data.
+ :search: The search string for this view. This value can be UPDATEd to
+ initiate a text search in this view.
+
+lnav_views_echo
+---------------
+
+The :code:`lnav_views_echo` table is a real SQLite table that you can create
+TRIGGERs on in order to react to users moving around in a view.
+
+.. note::
+
+ The table is periodically updated to reflect the current state of the views.
+ The changes are *not* performed immediately after the user action.
+
+lnav_view_stack
+---------------
+
+The **lnav_view_stack** table allows you to SELECT and DELETE from the stack of
+**lnav** "views" (e.g. log, text, ...). The following columns are available in
+this table:
+
+ :name: The name of the view.
+
+.. _table_lnav_view_filters:
+
+lnav_view_filters
+-----------------
+
+The **lnav_view_filters** table allows you to manipulate the filters in the
+**lnav** views. The following columns are available in this table:
+
+ :view_name: The name of the view the filter is applied to.
+ :filter_id: The filter identifier. This will be assigned on insertion.
+ :enabled: Indicates whether this filter is enabled or disabled.
+ :type: The type of filter, either 'in' or 'out'.
+ :pattern: The regular expression to filter on.
+
+This table supports SELECT, INSERT, UPDATE, and DELETE on the table rows to
+read, create, update, and delete filters for the views.
+
+lnav_view_filter_stats
+----------------------
+
+The **lnav_view_filter_stats** table allows you to get information about how
+many lines matched a given filter. The following columns are available in
+this table:
+
+ :view_name: The name of the view.
+ :filter_id: The filter identifier.
+ :hits: The number of lines that matched this filter.
+
+This table is read-only.
+
+lnav_view_filters_and_stats
+---------------------------
+
+The **lnav_view_filters_and_stats** view joins the **lnav_view_filters** table
+with the **lnav_view_filter_stats** table into a single view for ease of use.
+
+all_logs
+--------
+
+.. f0:sql.tables.all_logs
+
+The **all_logs** table lets you query the format derived from the **lnav**
+log message parser that is used to automatically extract data, see
+:ref:`data-ext` for more details.
+
+http_status_codes
+-----------------
+
+The **http_status_codes** table is a handy reference that can be used to turn
+HTTP status codes into human-readable messages.
+
+regexp_capture(<string>, <regex>)
+---------------------------------
+
+The **regexp_capture()** table-valued function applies the regular expression
+to the given string and returns detailed results for the captured portions of
+the string.
diff --git a/docs/source/ui.rst b/docs/source/ui.rst
new file mode 100644
index 0000000..c7d7ffe
--- /dev/null
+++ b/docs/source/ui.rst
@@ -0,0 +1,280 @@
+.. _ui:
+
+User Interface
+==============
+
+The **lnav** TUI displays the content of the current "view" in the middle,
+with status bars above and below, and the interactive prompt as the last line.
+
+.. figure:: lnav-ui.png
+ :align: center
+ :alt: Screenshot of lnav showing a mix of syslog and web access_log messages.
+
+ Screenshot of **lnav** viewing syslog and web access_log messages.
+
+The default view shows the log messages from the log files that have been
+loaded. There are other views for displaying content like plaintext files
+and SQL results. The :ref:`ui_views` section describes the characteristics of
+each view in more detail. You can switch to the different views using the
+hotkeys described in the :ref:`hotkeys_display` section or by pressing
+:kbd:`ENTER` to activate the breadcrumb bar, moving to the first crumb, and
+then selecting the desired view. You can switch back to the previous view by
+pressing :kbd:`q`. You can switch forward to the new view by pressing
+:kbd:`a`. If the views are time-based (e.g. log and histogram), pressing
+:kbd:`Shift` + :kbd:`q` and :kbd:`Shift` + :kbd:`a` will synchronize the top
+times in the views.
+
+The right side of the display has a proportionally sized 'scrollbar' that
+shows:
+
+* the current position in the file;
+* the locations of errors/warnings in the log files by using red or yellow
+ coloring;
+* the locations of search hits by using a tick-mark pointing to the left;
+* the locations of bookmarks by using a tick-mark pointing to the right.
+
+Top Status Bar
+--------------
+
+The top status bar shows the current time and messages stored in the
+:ref:`table_lnav_user_notifications` table.
+
+Below the top status bar is the breadcrumb bar that displays the semantic
+location of the top line in the main view. For example, within a
+pretty-printed JSON document, it will show the path to property at the top
+of the view. The actual content of the bar depends on the current view and
+will be updated as you navigate around the main view. The bar can also be
+used to navigate around the document by focusing on it.
+
+Breadcrumb Bar
+--------------
+
+.. figure:: lnav-breadcrumbs-help.png
+ :align: center
+ :figwidth: 90%
+
+ Screenshot of the breadcrumb bar focused and navigating the help text
+
+To focus on the breadcrumb bar, press :kbd:`ENTER`. The :kbd:`←`/:kbd:`→`
+cursor keys can be used to select a crumb and the :kbd:`↑`/:kbd:`↓` keys can
+be used select a value of that crumb. To accept a value and drop focus on the
+bar, press :kbd:`ENTER`. To accept a value and move to the next crumb, press
+:kbd:`→`. Using :kbd:`→` makes it quicker to drill down into a document
+without having to constantly switch focus. To drop focus on the bar without
+accepting anything, press :kbd:`Escape`.
+
+There are three types of crumbs:
+
+* a dropdown where one of a limited set of values can be selected;
+* a combobox where a value can be entered directly or selected;
+* a numeric input for entering array indexes.
+
+When a dropdown or combobox is selected, you can type part of the desired value
+to filter the list of values. For example, the first crumb is always the
+current view, typing in "hi" will filter the list down to the "HIST" value.
+
+Configuration Panels
+--------------------
+
+.. figure:: lnav-config-header.png
+ :align: center
+ :figwidth: 90%
+
+ Screenshot of the header for the configuration panels when they are hidden.
+
+After the main view content, there is a header bar for two configuration
+panels: Files and Filters. These panels provide visual access to parts of
+lnav's configuration. To access the panels, press the :kbd:`TAB` key.
+To hide the panels again, press :kbd:`q`.
+
+.. figure:: lnav-files-panel.png
+ :align: center
+ :figwidth: 90%
+
+ Screenshot of the files panel showing the loaded files.
+
+The Files panel is open initially to display progress in loading files.
+The following information can be displayed for each file:
+
+* the "unique" portion of the path relative to the other files;
+* the amount of data that has been indexed;
+* the date range of log messages contained in the file;
+* the errors that were encountered while trying to index the file;
+* the notes recorded for files where some automatic action was taken,
+ like hiding the file if it was seen as a duplicate of another file.
+
+.. figure:: lnav-filters-panel.png
+ :align: center
+ :figwidth: 90%
+
+ Screenshot of the filters panel showing an OUT and a disabled IN filter.
+
+If the view supports filtering, there will be a status line showing the
+following:
+
+* the number of enabled filters and the total number of filters;
+* the number of lines that are **not** displayed because of filtering.
+
+To edit the filters, you can press TAB to change the focus from the main
+view to the filter editor. The editor allows you to create, enable/disable,
+and delete filters easily.
+
+Bottom Status Bar
+-----------------
+
+The second to last line is the bottom status bar, which shows the following:
+
+* the line number of the top line, starting from zero;
+* the location within the view, as a percentage;
+* the current search hit, the total number of hits, and the search term;
+* the loading indicator.
+
+When the interactive prompt is active, this bar can show the prompt
+description, help text, or error message.
+
+Prompt
+------
+
+Finally, the last line on the display is where you can enter search
+patterns and execute internal commands, such as converting a
+unix-timestamp into a human-readable date. The following key-presses
+will activate a corresponding prompt:
+
+* :kbd:`/` - The search prompt. You can enter a PCRE2-flavored regular
+ expression to search for in the current view.
+* :kbd:`:` - The command prompt. Commands are used to perform common
+ operations.
+* :kbd:`;` - The SQL prompt. SQL queries can be used for log analysis
+ and manipulating **lnav**'s state.
+* :kbd:`|` - The script prompt. Enter a path to the lnav script to
+ execute, along with the arguments to pass in.
+
+The command-line is by the readline library, so the usual set of keyboard
+shortcuts can be used for editing and moving within the command-line.
+
+.. _ui_views:
+
+Views
+-----
+
+The accessible content within lnav is separated into the following views.
+
+LOG
+^^^
+
+The log view displays the log messages from any loaded log files in time
+order. This view will be shown by default if any log messages are available.
+
+On color displays, the log messages will be highlighted as follows:
+
+* Errors will be colored in red;
+* warnings will be yellow;
+* search hits are reverse video;
+* various color highlights will be applied to: IP addresses, SQL keywords,
+ XML tags, file and line numbers in Java backtraces, and quoted strings;
+* "identifiers" in the messages will be randomly assigned colors based on their
+ content (works best on "xterm-256color" terminals).
+
+.. note::
+
+ If the coloring is too much for your tastes, you can change to the
+ "grayscale" theme by entering the following command:
+
+ .. code-block:: lnav
+
+ :config /ui/theme grayscale
+
+.. note::
+
+ If a log message has a timestamp that is out-of-order with its neighboring
+ messages, the timestamp will be highlighted in yellow. When one of these
+ messages is at the top of the log view, an overlay will display the
+ difference between the "actual time" and the "received time". The "actual
+ time" is the original textual timestamp. The "received time" is the time
+ of an earlier message that is larger than this log message's time.
+
+The breadcrumb bar will show the following crumbs:
+
+* the timestamp for the top line;
+* the log format for the top line;
+* the name of the file the top line was pulled from;
+* the "operation ID" of the top log message, if it is supported by the log
+ format.
+
+These crumbs are interactive and can be used to navigate to different parts
+of the log view. For example, selecting a different value in the log format
+crumb will jump to the first message with that format.
+
+TEXT
+^^^^
+
+The text view displays files for which lnav could not detect any log messages.
+
+Markdown
+""""""""
+
+Files with an :code:`.md` (or :code:`.markdown`) extension will be treated as
+Markdown files and rendered separately.
+
+DB
+^^
+
+The DB view shows the results of queries done through the SQLite interface.
+You can execute a query by pressing :kbd:`;` and then entering a SQL statement.
+You can switch to the SQL view by pressing :kbd:`v`.
+
+HELP
+^^^^
+
+The help view displays the builtin help text. Press :kbd:`?` to switch to the
+help view at any time. While in the help view, the breadcrumb bar can be used
+to navigate to different sections of the document.
+
+HIST
+^^^^
+
+The histogram view displays a stacked bar chart of messages over time
+classified by their log level and whether they've been bookmarked. Press
+:kbd:`i` to switch back and forth to the histogram view. You can also press
+:kbd:`Shift`+:kbd:`i` to toggle the histogram view while synchronizing the top
+time. While in the histogram view, pressing :kbd:`z`/:kbd:`Shift`+:kbd:`z`
+will zoom in/out.
+
+PRETTY
+^^^^^^
+
+The pretty-print view takes the text displayed in the current view and shows
+the result of a pretty-printer run on that text. For example, if a log
+message contained an XML message on a single line, the pretty-printer would
+break the XML across multiple lines with appropriate indentation.
+
+SCHEMA
+^^^^^^
+
+The schema view displays the current schema of the builtin SQLite database.
+
+SPECTRO
+^^^^^^^
+
+The spectrogram view is a "three"-dimensional display of data points of a log
+field or a SQL query column. The dimensions are time on the Y axis, the range
+of data point values on the X axis, and the number of data points as a color.
+For example, if you were to visualize process CPU usage over time, the range
+of values on the X axis would be CPU percentages and there would be colored
+blocks at each point on the line where a process had that CPU percentage, like
+so
+
+.. figure:: lnav-spectro-cpu-pct.png
+ :align: center
+
+ Screenshot of the **lnav** spectrogram view showing CPU usage of processes.
+
+The colors correspond to the relative number of data points in a bucket.
+The legend overlaid at the top line in the view shows the counts of data
+points that are in a particular color, with green having the fewest number of
+data points, yellow the middle, and red the most. You can select a particular
+bucket using the cursor keys to see the exact number of data points and the
+range of values. The panel at the bottom of the view shows the data points
+themselves from the original source, the log file or the SQL query results.
+You can press :kbd:`TAB` to focus on the details panel so you can scroll
+around and get a closer look at the values.
diff --git a/docs/source/usage.rst b/docs/source/usage.rst
new file mode 100644
index 0000000..d43ed52
--- /dev/null
+++ b/docs/source/usage.rst
@@ -0,0 +1,291 @@
+.. _usage:
+
+Usage
+=====
+
+This chapter contains an overview of how to use **lnav**.
+
+
+Basic Controls
+--------------
+
+Like most file viewers, scrolling through files can be done with the usual
+:ref:`hotkeys<hotkeys>`. For non-trivial operations, you can enter the
+:ref:`command<commands>` prompt by pressing :kbd:`:`. To analyze data in a
+log file, you can enter the :ref:`SQL prompt<sql-ext>` by pressing :kbd:`;`.
+
+.. tip::
+
+ Check the bottom right corner of the screen for tips on hotkeys that might
+ be useful in the current context.
+
+ .. figure:: hotkey-tips.png
+ :align: center
+
+ When **lnav** is first open, it suggests using :kbd:`e` and
+ :kbd:`Shift` + :kbd:`e` to jump to error messages.
+
+
+Viewing Files
+-------------
+
+The files to view in **lnav** can be given on the command-line or passed to the
+:ref:`:open<open>` command. A
+`glob pattern <https://en.wikipedia.org/wiki/Glob_(programming)>`_ can be given
+to watch for files with a common name. If the path is a directory, all of the
+files in the directory will be opened and the directory will be monitored for
+files to be added or removed from the view. If the path is an archive or
+compressed file (and lnav was built with libarchive), the archive will be
+extracted to a temporary location and the files within will be loaded. The
+files that are found will be scanned to identify their file format. Files
+that match a log format will be collated by time and displayed in the LOG
+view. Plain text files can be viewed in the TEXT view, which can be accessed
+by pressing :kbd:`t`.
+
+
+Archive Support
+^^^^^^^^^^^^^^^
+
+.. f0:archive
+
+If **lnav** is compiled with `libarchive <https://www.libarchive.org>`_,
+any files to be opened will be examined to see if they are a supported archive
+type. If so, the contents of the archive will be extracted to the
+:code:`$TMPDIR/lnav-user-${UID}-work/archives/` directory. Once extracted, the
+files within will be loaded into lnav. To speed up opening large amounts of
+files, any file that meets the following conditions will be automatically
+hidden and not indexed:
+
+* Binary files
+* Plain text files that are larger than 128KB
+* Duplicate log files
+
+The unpacked files will be left in the temporary directory after exiting
+**lnav** so that opening the same archive again will be faster. Unpacked
+archives that have not been accessed in the past two days will be automatically
+deleted the next time **lnav** is started.
+
+
+.. _remote:
+
+Remote Files
+^^^^^^^^^^^^
+
+Files on remote machines can be viewed and tailed if you have access to the
+machines via SSH. First, make sure you can SSH into the remote machine without
+any interaction by: 1) accepting the host key as known and 2) copying your
+identity's public key to the :file:`.ssh/authorized_keys` file on the remote
+machine. Once the setup is complete, you can open a file on a remote host
+using the same syntax as :command:`scp(1)` where the username and host are
+given, followed by a colon, and then the path to the files, like so::
+
+ [user@]host:/path/to/logs
+
+For example, to open :file:`/var/log/syslog.log` on "host1.example.com" as the
+user "dean", you would write:
+
+.. prompt:: bash
+
+ lnav dean@host1.example.com:/var/log/syslog.log
+
+Remote files can also be opened using the :ref:`:open<open>` command. Opening
+a remote file in the TUI has the advantage that the file path can be
+:kbd:`TAB`-completed and a preview is shown of the first few lines of the
+file.
+
+.. note::
+
+ If lnav is installed from the `snap <https://snapcraft.io/lnav>`_, you will
+ need to connect it to the
+ `ssh-keys plug <https://snapcraft.io/docs/ssh-keys-interface>`_ using the
+ following command:
+
+ .. prompt:: bash
+
+ sudo snap connect lnav:ssh-keys
+
+.. note::
+
+ Remote file access is implemented by transferring an
+ `αcτµαlly pδrταblε εxεcµταblε <https://justine.lol/ape.html>`_ to the
+ destination and invoking it. An APE binary can run on most any x86_64
+ machine and OS (i.e. MacOS, Linux, FreeBSD, Windows). The binary is
+ baked into the lnav executable itself, so there is no extra setup that
+ needs to be done on the remote machine.
+
+ The binary file is named ``tailer.bin.XXXXXX`` where *XXXXXX* is 6 random digits.
+ The file is, under normal circumstancies, deleted immediately.
+
+Searching
+---------
+
+Any log messages that are loaded into **lnav** are indexed by time and log
+level (e.g. error, warning) to make searching quick and easy with
+:ref:`hotkeys<hotkeys>`. For example, pressing :kbd:`e` will jump to the
+next error in the file and pressing :kbd:`Shift` + :kbd:`e` will jump to
+the previous error. Plain text searches can be done by pressing :kbd:`/`
+to enter the search prompt. A regular expression can be entered into the
+prompt to start a search through the current view.
+
+
+.. _filtering:
+
+Filtering
+---------
+
+To reduce the amount of noise in a log file, **lnav** can hide log messages
+that match certain criteria. The following sub-sections explain ways to go
+about that.
+
+
+Regular Expression Match
+^^^^^^^^^^^^^^^^^^^^^^^^
+
+If there are log messages that you are not interested in, you can do a
+"filter out" to hide messages that match a pattern. A filter can be created
+using the interactive editor, the :ref:`:filter-out<filter_out>` command, or
+by doing an :code:`INSERT` into the
+:ref:`lnav_view_filters<table_lnav_view_filters>` table.
+
+If there are log messages that you are only interested in, you can do a
+"filter in" to only show messages that match a pattern. The filter can be
+created using the interactive editor, the :ref:`:filter-in<filter_in>` command,
+or by doing an :code:`INSERT` into the
+:ref:`lnav_view_filters<table_lnav_view_filters>` table.
+
+
+SQLite Expression
+^^^^^^^^^^^^^^^^^
+
+Complex filtering can be done by passing a SQLite expression to the
+:ref:`:filter-expr<filter_expr>` command. The expression will be executed for
+every log message and if it returns true, the line will be shown in the log
+view.
+
+
+Time
+^^^^
+
+To limit log messages to a given time frame, the
+:ref:`:hide-lines-before<hide_lines_before>` and
+:ref:`:hide-lines-after<hide_lines_after>` commands can be used to specify
+the beginning and end of the time frame.
+
+
+Log level
+^^^^^^^^^
+
+To hide messages below a certain log level, you can use the
+:ref:`:set-min-log-level<set_min_log_level>` command.
+
+.. _search_tables:
+
+Search Tables
+-------------
+
+Search tables allow you to access arbitrary data in log messages through
+SQLite virtual tables. If there is some data in a log message that you can
+match with a regular expression, you can create a search-table that matches
+that data and any capture groups will be plumbed through as columns in the
+search table.
+
+Creating a search table can be done interactively using the
+:ref:`:create-search-table<create_search_table>` command or by adding it to
+a :ref:`log format definition<log_formats>`. The main difference between
+the two is that tables defined as part of a format will only search messages
+from log files with that format and the tables will include log message
+columns defined in that format. Whereas a table created with the command
+will search messages from all different formats and no format-specific
+columns will be included in the table.
+
+.. _taking_notes:
+
+Taking Notes
+------------
+
+A few of the columns in the log tables can be updated on a row-by-row basis to
+allow you to take notes. The majority of the columns in a log table are
+read-only since they are backed by the log files themselves. However, the
+following columns can be changed by an :code:`UPDATE` statement:
+
+* **log_part** - The "partition" the log message belongs to. This column can
+ also be changed by the :ref:`:partition-name<partition_name>` command.
+* **log_mark** - Indicates whether the line has been bookmarked.
+* **log_comment** - A free-form text field for storing commentary. This
+ column can also be changed by the :ref:`:comment<comment>` command.
+* **log_tags** - A JSON list of tags associated with the log message. This
+ column can also be changed by the :ref:`:tag<tag>` command.
+
+While these columns can be updated by through other means, using the SQL
+interface allows you to make changes automatically and en masse. For example,
+to bookmark all lines that have the text "something interesting" in the log
+message body, you can execute:
+
+.. code-block:: custsqlite
+
+ ;UPDATE all_logs SET log_mark = 1 WHERE log_body LIKE '%something interesting%'
+
+As a more advanced example of the power afforded by SQL and **lnav**'s virtual
+tables, we will tag log messages where the IP address bound by dhclient has
+changed. For example, if dhclient reports "bound to 10.0.0.1" initially and
+then reports "bound to 10.0.0.2", we want to tag only the messages where the
+IP address was different from the previous message. While this can be done
+with a single SQL statement [#]_, we will break things down into a few steps for
+this example. First, we will use the :ref:`:create-search-table<create_search_table>`
+command to match the dhclient message and extract the IP address:
+
+.. code-block:: lnav
+
+ :create-search-table dhclient_ip bound to (?<ip>[^ ]+)
+
+The above command will create a new table named :code:`dhclient_ip` with the
+standard log columns and an :code:`ip` column that contains the IP address.
+Next, we will create a view over the :code:`dhclient_ip` table that returns
+the log message line number, the IP address from the current row and the IP
+address from the previous row:
+
+.. code-block:: custsqlite
+
+ ;CREATE VIEW IF NOT EXISTS dhclient_ip_changes AS SELECT log_line, ip, lag(ip) OVER (ORDER BY log_line) AS prev_ip FROM dhclient_ip
+
+Finally, the following :code:`UPDATE` statement will concatenate the tag
+"#ipchanged" onto the :code:`log_tags` column for any rows in the view where
+the current IP is different from the previous IP:
+
+.. code-block:: custsqlite
+
+ ;UPDATE syslog_log SET log_tags = json_concat(log_tags, '#ipchanged') WHERE log_line IN (SELECT log_line FROM dhclient_ip_changes WHERE ip != prev_ip)
+
+Since the above can be a lot to type out interactively, you can put these
+commands into a :ref:`script<scripts>` and execute that script with the
+:kbd:`\|` hotkey.
+
+.. [#] The expression :code:`regexp_match('bound to ([^ ]+)', log_body) as ip`
+ can be used to extract the IP address from the log message body.
+
+Sharing Sessions With Others
+----------------------------
+
+After setting up filters, bookmarks, and making notes, you might want to share
+your work with others. If they have access to the same log files, you can
+use the :ref:`:export-session-to<export_session_to>` command to write an
+executable **lnav** script that will recreate the current session state. The
+script contains various SQL statements and **lnav** commands that capture the
+current state. So, you should feel free to modify the script or use it as a
+reference to learn about more advanced uses of lnav.
+
+The script will capture the file paths that were explicitly specified and
+not the files that were actually opened. For example, if you specified
+"/var/log" on the command line, the script will include
+:code:`:open /var/log/*` and not an individual open for each file in that
+directory.
+
+Also, in order to support archives of log files, lnav will try to find the
+directory where the archive was unpacked and use that as the base for the
+:code:`:open` command. Currently, this is done by searching for the top
+"README" file in the directory hierarchy containing the files [1]_. The
+consumer of the session script can then set the :code:`LOG_DIR_0` (or 1, 2,
+...) environment variable to change where the log files will be loaded from.
+
+.. [1] It is assumed a log archive would have a descriptive README file.
+ Other heuristics may be added in the future.