diff options
Diffstat (limited to 'docs/source')
37 files changed, 3514 insertions, 0 deletions
diff --git a/docs/source/_ext/__init__.py b/docs/source/_ext/__init__.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/docs/source/_ext/__init__.py diff --git a/docs/source/_ext/lnavlexer.py b/docs/source/_ext/lnavlexer.py new file mode 100644 index 0000000..2509a5f --- /dev/null +++ b/docs/source/_ext/lnavlexer.py @@ -0,0 +1,26 @@ + +__all__ = ['LnavCommandLexer'] + +import re + +from pygments.token import Whitespace, Text, Keyword, Literal +from pygments.lexers._mapping import LEXERS +from pygments.lexers.python import RegexLexer + +class LnavCommandLexer(RegexLexer): + name = 'lnav' + + flags = re.IGNORECASE + tokens = { + 'root': [ + (r'\s+', Whitespace), + (r':[\w\-]+', Keyword), + (r'\<[\w\-]+\>', Literal.String.Doc), + (r'.', Text), + ] + } + +def setup(app): + LEXERS['LnavCommandLexer'] = ( + '_ext.lnavlexer', 'lnav', ('lnav',), ('*.lnav',), ('text/lnav',)) + app.add_lexer('lnav', LnavCommandLexer) diff --git a/docs/source/_static/theme_overrides.css b/docs/source/_static/theme_overrides.css new file mode 100644 index 0000000..4fc1c9f --- /dev/null +++ b/docs/source/_static/theme_overrides.css @@ -0,0 +1,56 @@ +/* override table width restrictions */ +@media screen and (min-width: 767px) { + + .wy-table-responsive table td { + /* !important prevents the common CSS stylesheets from overriding + this as on RTD they are loaded after this stylesheet */ + white-space: normal !important; + } + +} + +th p { + margin-bottom: 0px; +} + +.wy-nav-content { + max-width: 900px; +} + +table.query-results p { + font-size: 0.9em !important; +} + +DL DT:target, :target > H2, :target > H3, span:target + H2, span:target + H3 { + border: 0 !important; + border-bottom: 1px solid #d3d381 !important; + background: #ffc !important; + /* padding: 1em !important;*/ +} + +DL DT { + border: 0 !important; + background: inherit !important; + border-bottom: 1px solid #c3c0ee !important; + display: block !important; +} + +kbd { + padding: 0.1em 0.6em; + border: 1px solid #ccc; + font-size: 11px; + font-family: Arial, Helvetica, sans-serif; + background-color: #f7f7f7; + color: #333; + -moz-box-shadow: 0 1px 0px rgba(0, 0, 0, 0.2), 0 0 0 2px #ffffff inset; + -webkit-box-shadow: 0 1px 0px rgba(0, 0, 0, 0.2), 0 0 0 2px #ffffff inset; + box-shadow: 0 1px 0px rgba(0, 0, 0, 0.2), 0 0 0 2px #ffffff inset; + -moz-border-radius: 3px; + -webkit-border-radius: 3px; + border-radius: 3px; + display: inline-block; + margin: 0 0.1em 0.1em 0.1em; + text-shadow: 0 1px 0 #fff; + line-height: 1.4; + white-space: nowrap; +} diff --git a/docs/source/cli.rst b/docs/source/cli.rst new file mode 100644 index 0000000..565e54f --- /dev/null +++ b/docs/source/cli.rst @@ -0,0 +1,168 @@ +.. _cli: + +Command Line Interface +====================== + +There are two command-line interfaces provided by **lnav**, one for viewing +files and one for managing **lnav**'s configuration. The file viewing mode is +the default and is all that most people will need. The management mode can +be useful for those that are developing log file formats and is activated by +passing the :option:`-m` option as the first argument. + +File Viewing Mode +----------------- + +The following options can be used when starting **lnav**. There are not +many flags because the majority of the functionality is accessed using +the :option:`-c` option to execute :ref:`commands<commands>` or +:ref:`SQL queries<sql-ext>`. + +Options +^^^^^^^ + +.. option:: -h + + Print these command-line options and exit. + +.. option:: -H + + Start lnav and switch to the help view. + +.. option:: -C + + Check the given files against the configuration, report any errors, and + exit. This option can be helpful for validating that a log format is + well-formed. + +.. option:: -c <command> + + Execute the given lnav command, SQL query, or lnav script. The + argument must be prefixed with the character used to enter the prompt + to distinguish between the different types (i.e. ':', ';', '|'). + This option can be given multiple times. + +.. option:: -f <path> + + Execute the given command file. This option can be given multiple times. + +.. option:: -I <path> + + Add a configuration directory. + +.. option:: -i + + Install the format files in the :file:`.lnav/formats/` directory. + Individual files will be installed in the :file:`installed` + directory and git repositories will be cloned with a directory + name based on their repository URI. + +.. option:: -u + + Update formats installed from git repositories. + +.. option:: -d <path> + + Write debug messages to the given file. + +.. option:: -n + + Run without the curses UI (headless mode). + +.. option:: -N + + Do not open the default syslog file if no files are given. + +.. option:: -r + + Recursively load files from the given base directories. + +.. option:: -t + + Prepend timestamps to the lines of data being read in on the standard input. + +.. option:: -w <path> + + Write the contents of the standard input to this file. + +.. option:: -V + + Print the version of lnav. + +.. option:: -q + + Do not print the log messages after executing all of the commands. + + +Management Mode (v0.11.0+) +-------------------------- + +The management CLI mode provides functionality for query **lnav**'s log +format definitions. + +Options +^^^^^^^ + +.. option:: -m + + Switch to management mode. This must be the first option passed on the + command-line. + +Subcommands +^^^^^^^^^^^ + +.. option:: regex101 import <regex101-url> <format-name> [<regex-name>] + + Convert a regex101.com entry into a skeleton log format file. + +.. option:: format <format-name> regex <regex-name> push + + Push a log format regular expression to regex101.com . + +.. option:: format <format-name> regex <regex-name> pull + + Pull changes to a regex that was previously pushed to regex101.com . + +Environment Variables +--------------------- + +.. envvar:: XDG_CONFIG_HOME + + If this variable is set, lnav will use this directory to store its + configuration in a sub-directory named :file:`lnav`. + +.. envvar:: HOME + + If :envvar:`XDG_CONFIG_HOME` is not set, lnav will use this directory + to store its configuration in a sub-directory named :file:`.lnav`. + +.. envvar:: APPDATA + + On Windows, lnav will use this directory instead of HOME + to store its configuration in a sub-directory named :file:`.lnav`. + +.. envvar:: TZ + + The timezone setting is used in some log formats to convert UTC timestamps + to the local timezone. + + +Examples +-------- + + To load and follow the system syslog file: + + .. prompt:: bash + + lnav + + To load all of the files in :file:`/var/log`: + + .. prompt:: bash + + lnav /var/log + + To watch the output of make with timestamps prepended: + + .. prompt:: bash + + make 2>&1 | lnav -t diff --git a/docs/source/commands.rst b/docs/source/commands.rst new file mode 100644 index 0000000..d83a07e --- /dev/null +++ b/docs/source/commands.rst @@ -0,0 +1,132 @@ +.. role:: lnavcmd(code) + :language: lnav + :class: highlight + +.. _commands: + +Commands +======== + +Commands provide access to some of the more advanced features in **lnav**, like +:ref:`filtering<filtering>` and +:ref:`"search tables"<search_tables>`. You can activate the command +prompt by pressing the :kbd:`:` key. At the prompt, you can start typing +in the desired command and/or double-tap :kbd:`TAB` to activate +auto-completion and show the available commands. To guide you in the usage of +the commands, a help window will appear above the command prompt with an +explanation of the command and its parameters (if it has any). For example, +the screenshot below shows the help for the :code:`:open` command: + +.. figure:: open-help.png + :align: center + :figwidth: 50% + + Screenshot of the online help for the :code:`:open` command. + +In addition to online help, many commands provide a preview of the effects that +the command will have. This preview will activate shortly after you have +finished typing, but before you have pressed :kbd:`Enter` to execute the +command. For example, the :code:`:open` command will show a preview of the +first few lines of the file given as its argument: + +.. figure:: open-preview.png + :align: center + + Screenshot of the preview shown for the :code:`:open` command. + +The :lnavcmd:`:filter-out pattern` command is another instance where the preview behavior +can help you craft the correct command-line. This command takes a PCRE2 regular +expression that specifies the log messages that should be filtered out of the +view. The preview for this command will highlight the portion of the log +messages that match the expression in red. Thus, you can be certain that the +regular expression is matching the log messages you are interested in before +committing the filter. The following screenshot shows an example of this +preview behavior for the string "launchd": + +.. figure:: filter-out-preview.png + :align: center + + Screenshot showing the preview for the :code:`:filter-out` command. + +Any errors detected during preview will be shown in the status bar right above +the command prompt. For example, an attempt to open an unknown file will show +an error message in the status bar, like so: + +.. figure:: open-error.png + :align: center + + Screenshot of the error shown when trying to open a non-existent file. + +.. tip:: + + Note that almost all commands support TAB-completion for their arguments. + So, if you are in doubt as to what to type for an argument, you can double- + tap the :kbd:`TAB` key to get suggestions. For example, the + TAB-completion for the :code:`filter-in` command will suggest words that are + currently displayed in the view. + +.. note:: The following commands can be disabled by setting the ``LNAVSECURE`` + environment variable before executing the **lnav** binary: + + - :code:`:open` + - :code:`:pipe-to` + - :code:`:pipe-line-to` + - :code:`:write-*-to` + + This makes it easier to run **lnav** in restricted environments without the + risk of privilege escalation. + +I/O Commands +------------ + +Anonymization +^^^^^^^^^^^^^ + +Anonymization is the process of removing identifying information from content +to make it safer for sharing with others. For example, an IP address can +often be used to uniquely identify an entity. Substituting all instances of +a particular IP with the same dummy value would remove the identifying data +without losing statistical accuracy. **lnav** has built-in support for +anonymization through the :code:`--anonymize` flag on the :code:`:write-*` +collection of commands. While the anonymization process should catch most + + :IPv4 Addresses: Are replaced with addresses in the :code:`10.0.0.0/8` range. + + :IPv6 Addresses: Are replaced with addresses in the :code:`2001:db8::/32` range. + + :URL User Names: Are replaced with a random animal name. + + :URL Passwords: Are replaced with a hash of the input password. + + :URL Hosts: Are replaced with a random name under the example.com domain. + + :URL Paths: Are recursively examined for substitution. + + :URL Query Strings: Are recursively examined for substitution. + + :URL Fragments: Are recursively examined for substitution. + + :Paths: Are recursively examined for substitution. + + :Credit Card Numbers: Are replaced with a 16 digit hash of the input number. + + :MAC Addresses: Are replaced with addresses in the :code:`00:00:5E:00:53:00` range. + + :Hex Dumps: Are replaced with a hash of the input replicated to the size of input. + + :Email User Names: Are replaced with a random animal name. + + :Email Host Names: Are replaced with a random name under the example.com domain. + + :Words: Are replaced with a random word with a matching case style. + + :Quoted Strings: Are recursively examined for substitution. + + :UUID: Are replaced with a hash of the input. + + :XML Attribute Values: Are recursively examined for substitution. + +Reference +--------- + +.. include:: ../../src/internals/cmd-ref.rst diff --git a/docs/source/conf.py b/docs/source/conf.py new file mode 100644 index 0000000..5f74b8c --- /dev/null +++ b/docs/source/conf.py @@ -0,0 +1,460 @@ +# -*- coding: utf-8 -*- +# +# lnav documentation build configuration file, created by +# sphinx-quickstart on Fri Jul 12 21:09:39 2013. +# +# This file is execfile()d with the current directory set to its containing dir. +# +# Note that not all possible configuration values are present in this +# autogenerated file. +# +# All configuration values have a default; values that are commented out +# serve to show the default. + +import sys, os + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +this_dir = os.path.abspath('.') +src_dir = os.path.join(this_dir, "..", "..", "src") +sys.path.insert(0, this_dir) +sys.path.insert(0, src_dir) + +import format2csv + +format2csv.main(["", + os.path.join(this_dir, "format-table.csv"), + os.path.join(src_dir, "formats")]) + +import re +from pygments.lexer import RegexLexer, words +from pygments.token import Punctuation, Whitespace, Text, Comment, Operator, \ + Keyword, Name, String, Literal, Number, Generic +from sphinx.highlighting import lexers + + +class CustSqliteLexer(RegexLexer): + name = 'custsqlite' + + flags = re.IGNORECASE + tokens = { + 'root': [ + (r'\s+', Text), + (r'--.*\n?', Comment.Single), + (r'#.*\n?', Comment.Single), + (r'/\*', Comment.Multiline, 'multiline-comments'), + (words(( + 'ABORT', + 'ACTION', + 'ADD', + 'AFTER', + 'ALL', + 'ALTER', + 'ALWAYS', + 'ANALYZE', + 'AND', + 'AS', + 'ASC', + 'ATTACH', + 'AUTOINCREMENT', + 'BEFORE', + 'BEGIN', + 'BETWEEN', + 'BY', + 'CASCADE', + 'CASE', + 'CAST', + 'CHECK', + 'COLLATE', + 'COLUMN', + 'COMMIT', + 'CONFLICT', + 'CONSTRAINT', + 'CREATE', + 'CROSS', + 'CURRENT', + 'CURRENT_DATE', + 'CURRENT_TIME', + 'CURRENT_TIMESTAMP', + 'DATABASE', + 'DEFAULT', + 'DEFERRABLE', + 'DEFERRED', + 'DELETE', + 'DESC', + 'DETACH', + 'DISTINCT', + 'DO', + 'DROP', + 'EACH', + 'ELSE', + 'END', + 'ESCAPE', + 'EXCEPT', + 'EXCLUDE', + 'EXCLUSIVE', + 'EXISTS', + 'EXPLAIN', + 'FAIL', + 'FILTER', + 'FIRST', + 'FOLLOWING', + 'FOR', + 'FOREIGN', + 'FROM', + 'FULL', + 'GENERATED', + 'GLOB', + 'GROUP', + 'GROUPS', + 'HAVING', + 'IF', + 'IGNORE', + 'IMMEDIATE', + 'IN', + 'INDEX', + 'INDEXED', + 'INITIALLY', + 'INNER', + 'INSERT', + 'INSTEAD', + 'INTERSECT', + 'INTO', + 'IS', + 'ISNULL', + 'JOIN', + 'KEY', + 'LAST', + 'LEFT', + 'LIKE', + 'LIMIT', + 'MATCH', + 'NATURAL', + 'NO', + 'NOT', + 'NOTHING', + 'NOTNULL', + 'NULL', + 'NULLS', + 'OF', + 'OFFSET', + 'ON', + 'OR', + 'ORDER', + 'OTHERS', + 'OUTER', + 'OVER', + 'PARTITION', + 'PLAN', + 'PRAGMA', + 'PRECEDING', + 'PRIMARY', + 'QUERY', + 'RAISE', + 'RANGE', + 'RECURSIVE', + 'REFERENCES', + 'REGEXP', + 'REINDEX', + 'RELEASE', + 'RENAME', + 'REPLACE', + 'RESTRICT', + 'RIGHT', + 'ROLLBACK', + 'ROW', + 'ROWS', + 'SAVEPOINT', + 'SELECT', + 'SET', + 'TABLE', + 'TEMP', + 'TEMPORARY', + 'THEN', + 'TIES', + 'TO', + 'TRANSACTION', + 'TRIGGER', + 'UNBOUNDED', + 'UNION', + 'UNIQUE', + 'UPDATE', + 'USING', + 'VACUUM', + 'VALUES', + 'VIEW', + 'VIRTUAL', + 'WHEN', + 'WHERE', + 'WINDOW', + 'WITH', + 'WITHOUT'), suffix=r'\b'), + Keyword), + (words(( + 'ARRAY', 'BIGINT', 'BINARY', 'BIT', 'BLOB', 'BOOLEAN', 'CHAR', + 'CHARACTER', 'DATE', 'DEC', 'DECIMAL', 'FLOAT', 'INT', 'INTEGER', + 'INTERVAL', 'NUMBER', 'NUMERIC', 'REAL', 'SERIAL', 'SMALLINT', + 'VARCHAR', 'VARYING', 'INT8', 'SERIAL8', 'TEXT'), suffix=r'\b'), + Name.Builtin), + (r'[+*/<>=~!@#%^&|`?-]', Operator), + (r'[0-9]+', Number.Integer), + # TODO: Backslash escapes? + (r"'(''|[^'])*'", String.Single), + (r'"(""|[^"])*"', String.Symbol), # not a real string literal in ANSI SQL + (r'[a-z_][\w$]*', Name), # allow $s in strings for Oracle + (r'\$[a-z_]+', Name), + (r'[;:()\[\],.]', Punctuation) + ], + 'multiline-comments': [ + (r'/\*', Comment.Multiline, 'multiline-comments'), + (r'\*/', Comment.Multiline, '#pop'), + (r'[^/*]+', Comment.Multiline), + (r'[/*]', Comment.Multiline) + ] + } + + def analyse_text(text): + return 0.01 + + +lexers['custsqlite'] = CustSqliteLexer(startinline=True) + +# -- General configuration ----------------------------------------------------- + +# If your documentation needs a minimal Sphinx version, state it here. +# needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be extensions +# coming with Sphinx (named 'sphinx.ext.*') or your custom ones. +extensions = [ + 'sphinx.ext.coverage', + "sphinx_rtd_theme", + 'sphinx-jsonschema', + 'sphinx-prompt', + '_ext.lnavlexer', +] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# The suffix of source filenames. +source_suffix = '.rst' + +# The encoding of source files. +# source_encoding = 'utf-8-sig' + +# The master toctree document. +master_doc = 'index' + +# General information about the project. +project = u'lnav' +copyright = u'2022, Tim Stack' + +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. +# +# The short X.Y version. +version = '0.11' +# The full version, including alpha/beta/rc tags. +release = '0.11.1' + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# language = None + +# There are two options for replacing |today|: either, you set today to some +# non-false value, then it is used: +# today = '' +# Else, today_fmt is used as the format for a strftime call. +# today_fmt = '%B %d, %Y' + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +exclude_patterns = [] + +# The reST default role (used for this markup: `text`) to use for all documents. +# default_role = None + +# If true, '()' will be appended to :func: etc. cross-reference text. +# add_function_parentheses = True + +# If true, the current module name will be prepended to all description +# unit titles (such as .. function::). +# add_module_names = True + +# If true, sectionauthor and moduleauthor directives will be shown in the +# output. They are ignored by default. +# show_authors = False + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + +# A list of ignored prefixes for module index sorting. +# modindex_common_prefix = [] + +# -- Options for HTML output --------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +html_theme = 'sphinx_rtd_theme' + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +html_theme_options = { +} + +# Add any paths that contain custom themes here, relative to this directory. +# html_theme_path = [] + +# The name for this set of Sphinx documents. If None, it defaults to +# "<project> v<release> documentation". +# html_title = None + +# A shorter title for the navigation bar. Default is the same as html_title. +# html_short_title = None + +# The name of an image file (relative to this directory) to place at the top +# of the sidebar. +# html_logo = None + +# The name of an image file (within the static path) to use as favicon of the +# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 +# pixels large. +# html_favicon = None + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] + + +def setup(app): + app.add_css_file('theme_overrides.css') + + +# html_context = { +# 'css_files': [ +# '_static/theme_overrides.css', # override wide tables in RTD theme +# ], +# } + +# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, +# using the given strftime format. +# html_last_updated_fmt = '%b %d, %Y' + +# If true, SmartyPants will be used to convert quotes and dashes to +# typographically correct entities. +# html_use_smartypants = True + +# Custom sidebar templates, maps document names to template names. +# html_sidebars = {} + +# Additional templates that should be rendered to pages, maps page names to +# template names. +# html_additional_pages = {} + +# If false, no module index is generated. +# html_domain_indices = True + +# If false, no index is generated. +# html_use_index = True + +# If true, the index is split into individual pages for each letter. +# html_split_index = False + +# If true, links to the reST sources are added to the pages. +# html_show_sourcelink = True + +# If true, "Created using Sphinx" is shown in the HTML footer. Default is True. +# html_show_sphinx = True + +# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. +# html_show_copyright = True + +# If true, an OpenSearch description file will be output, and all pages will +# contain a <link> tag referring to it. The value of this option must be the +# base URL from which the finished HTML is served. +# html_use_opensearch = '' + +# This is the file name suffix for HTML files (e.g. ".xhtml"). +# html_file_suffix = None + +# Output file base name for HTML help builder. +htmlhelp_basename = 'lnavdoc' + +# -- Options for LaTeX output -------------------------------------------------- + +latex_elements = { + # The paper size ('letterpaper' or 'a4paper'). + # 'papersize': 'letterpaper', + + # The font size ('10pt', '11pt' or '12pt'). + # 'pointsize': '10pt', + + # Additional stuff for the LaTeX preamble. + # 'preamble': '', +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, author, documentclass [howto/manual]). +latex_documents = [ + ('index', 'lnav.tex', u'lnav Documentation', + u'Tim Stack', 'manual'), +] + +# The name of an image file (relative to this directory) to place at the top of +# the title page. +# latex_logo = None + +# For "manual" documents, if this is true, then toplevel headings are parts, +# not chapters. +# latex_use_parts = False + +# If true, show page references after internal links. +# latex_show_pagerefs = False + +# If true, show URL addresses after external links. +# latex_show_urls = False + +# Documents to append as an appendix to all manuals. +# latex_appendices = [] + +# If false, no module index is generated. +# latex_domain_indices = True + + +# -- Options for manual page output -------------------------------------------- + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [ + ('index', 'lnav', u'lnav Documentation', + [u'Tim Stack'], 1) +] + +# If true, show URL addresses after external links. +# man_show_urls = False + + +# -- Options for Texinfo output ------------------------------------------------ + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [ + ('index', 'lnav', u'lnav Documentation', + u'Tim Stack', 'lnav', 'One line description of project.', + 'Miscellaneous'), +] + +# Documents to append as an appendix to all manuals. +# texinfo_appendices = [] + +# If false, no module index is generated. +# texinfo_domain_indices = True + +# How to display URL addresses: 'footnote', 'no', or 'inline'. +# texinfo_show_urls = 'footnote' diff --git a/docs/source/config.rst b/docs/source/config.rst new file mode 100644 index 0000000..5294e52 --- /dev/null +++ b/docs/source/config.rst @@ -0,0 +1,270 @@ + +.. _Configuration: + +Configuration +============= + +The configuration for **lnav** is stored in the following JSON files in +:file:`~/.lnav`: + +* :file:`config.json` -- Contains local customizations that were done using the + :code:`:config` command. +* :file:`configs/default/*.json` -- The default configuration files that are + built into lnav are written to this directory with :file:`.sample` appended. + Removing the :file:`.sample` extension and editing the file will allow you to + do basic customizations. +* :file:`configs/installed/*.json` -- Contains configuration files installed + using the :option:`-i` flag (e.g. :code:`$ lnav -i /path/to/config.json`). +* :file:`configs/*/*.json` -- Other directories that contain :file:`*.json` + files will be loaded on startup. This structure is convenient for installing + **lnav** configurations, like from a git repository. + +A valid **lnav** configuration file must contain an object with the +:code:`$schema` property, like so: + +.. code-block:: json + + { + "$schema": "https://lnav.org/schemas/config-v1.schema.json" + } + +.. note:: + + Log format definitions are stored separately in the :file:`~/.lnav/formats` + directly. See the :ref:`Log Formats<log_formats>` chapter for more + information. + + +Options +------- + +The following configuration options can be used to customize the **lnav** UI to +your liking. The options can be changed using the :code:`:config` command. + +.. jsonschema:: ../schemas/config-v1.schema.json#/properties/ui/properties/keymap + +.. jsonschema:: ../schemas/config-v1.schema.json#/properties/ui/properties/theme + +.. jsonschema:: ../schemas/config-v1.schema.json#/properties/ui/properties/clock-format + +.. jsonschema:: ../schemas/config-v1.schema.json#/properties/ui/properties/dim-text + +.. jsonschema:: ../schemas/config-v1.schema.json#/properties/ui/properties/default-colors + + +.. _themes: + +Theme Definitions +----------------- + +User Interface themes are defined in a JSON configuration file. A theme is +made up of the style definitions for different types of text in the UI. A +:ref:`definition<theme_style>` can include the foreground/background colors +and the bold/underline attributes. The style definitions are broken up into +multiple categories for the sake of organization. To make it easier to write +a definition, a theme can define variables that can be referenced as color +values. + +Variables +^^^^^^^^^ + +The :code:`vars` object in a theme definition contains the mapping of variable +names to color values. These variables can be referenced in style definitions +by prefixing them with a dollar-sign (e.g. :code:`$black`). The following +variables can also be defined to control the values of the ANSI colors that +are log messages or plain text: + +.. csv-table:: ANSI colors + :header: "Variable Name", "ANSI Escape" + + "black", "ESC[30m" + "red", "ESC[31m" + "green", "ESC[32m" + "yellow", "ESC[33m" + "blue", "ESC[34m" + "magenta", "ESC[35m" + "cyan", "ESC[36m" + "white", "ESC[37m" + +Specifying Colors +^^^^^^^^^^^^^^^^^ + +Colors can be specified using hexadecimal notation by starting with a hash +(e.g. :code:`#aabbcc`) or using a color name as found at +http://jonasjacek.github.io/colors/. If colors are not specified for a style, +the values from the :code:`styles/text` definition. + +.. note:: + + When specifying colors in hexadecimal notation, you do not need to have an + exact match in the XTerm 256 color palette. A best approximation will be + picked based on the `CIEDE2000 <https://en.wikipedia.org/wiki/Color_difference#CIEDE2000>`_ + color difference algorithm. + + + +Example +^^^^^^^ + +The following example sets the black/background color for text to a dark grey +using a variable and sets the foreground to an off-white. This theme is +incomplete, but it works enough to give you an idea of how a theme is defined. +You can copy the code block, save it to a file in +:file:`~/.lnav/configs/installed/` and then activate it by executing +:code:`:config /ui/theme example` in lnav. For a more complete theme +definition, see one of the definitions built into **lnav**, like +`monocai <https://github.com/tstack/lnav/blob/master/src/themes/monocai.json>`_. + + .. code-block:: json + + { + "$schema": "https://lnav.org/schemas/config-v1.schema.json", + "ui": { + "theme-defs": { + "example1": { + "vars": { + "black": "#2d2a2e" + }, + "styles": { + "text": { + "color": "#f6f6f6", + "background-color": "$black" + } + } + } + } + } + } + +Reference +^^^^^^^^^ + +.. jsonschema:: ../schemas/config-v1.schema.json#/properties/ui/properties/theme-defs/patternProperties/([\w\-]+)/properties/vars + +.. jsonschema:: ../schemas/config-v1.schema.json#/properties/ui/properties/theme-defs/patternProperties/([\w\-]+)/properties/styles + +.. jsonschema:: ../schemas/config-v1.schema.json#/properties/ui/properties/theme-defs/patternProperties/([\w\-]+)/properties/syntax-styles + +.. jsonschema:: ../schemas/config-v1.schema.json#/properties/ui/properties/theme-defs/patternProperties/([\w\-]+)/properties/status-styles + +.. jsonschema:: ../schemas/config-v1.schema.json#/properties/ui/properties/theme-defs/patternProperties/([\w\-]+)/properties/log-level-styles + +.. _theme_style: + +.. jsonschema:: ../schemas/config-v1.schema.json#/definitions/style + + +.. _keymaps: + +Keymap Definitions +------------------ + +Keymaps in **lnav** map a key sequence to a command to execute. When a key is +pressed, it is converted into a hex-encoded string that is looked up in the +keymap. The :code:`command` value associated with the entry in the keymap is +then executed. Note that the "command" can be an **lnav** +:ref:`command<commands>`, a :ref:`SQL statement/query<sql-ext>`, or an +**lnav** script. If an :code:`alt-msg` value is included in the entry, the +bottom-right section of the UI will be updated with the help text. + +.. note:: + + Not all functionality is available via commands or SQL at the moment. Also, + some hotkeys are not implemented via keymaps. + +Key Sequence Encoding +^^^^^^^^^^^^^^^^^^^^^ + +Key presses are converted into a hex-encoded string that is used to lookup an +entry in the keymap. Each byte of the keypress value is formatted as an +:code:`x` followed by the hex-encoding in lowercase. For example, the encoding +for the £ key would be :code:`xc2xa3`. To make it easier to discover the +encoding for unassigned keys, **lnav** will print in the command prompt the +:code:`:config` command and +`JSON-Pointer <https://tools.ietf.org/html/rfc6901>`_ for assigning a command +to the key. + +.. figure:: key-encoding-prompt.png + :align: center + + Screenshot of the command prompt when an unassigned key is pressed. + +.. note:: + + Since **lnav** is a terminal application, it can only receive keypresses that + can be represented as characters or escape sequences. For example, it cannot + handle the press of a modifier key. + +Reference +^^^^^^^^^ + +.. jsonschema:: ../schemas/config-v1.schema.json#/properties/ui/properties/keymap-defs/patternProperties/([\w\-]+) + + +Log Handling +------------ + +The handling of logs is largely determined by the +:ref:`log file formats<log_formats>`, this section covers options that are not +specific to a particular format. + +Watch Expressions (v0.11.0+) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Watch expressions can be used to fire an event when a log message matches a +condition. You can then install a listener for these events and trigger an +action to be performed. For example, to automate filtering based on +identifiers, a watch expression can match messages that mention the ID and then +a trigger can install a filter for that ID. Creating a watch expression is +done by adding an entry into the :code:`/log/watch-expressions` configuration +tree. For example, to create a watch named "dhcpdiscover" that matches +DHCPDISCOVER messages from the :code:`dhclient` daemon, you would run the +following: + +.. code-block:: lnav + + :config /log/watch-expressions/dhcpdiscover/expr :log_procname = 'dhclient' AND startswith(:log_body, 'DHCPDISCOVER') + +The watch expression can refer to column names in the log message by prefixing +them with a colon. The expression is evaluated by passing the log message +fields as bound parameters and not against a table. The easiest way to test +out an expression is with the :ref:`mark_expr` command, since it will behave +similarly. After changing the configuration, you'll need to restart lnav +for the effect to take place. You can then query the :code:`lnav_events` +table to see any generated +:code:`https://lnav.org/event-log-msg-detected-v1.schema.json` events from the +logs that were loaded: + +.. code-block:: custsqlite + + ;SELECT * FROM lnav_events + +From there, you can create a SQLite trigger on the :code:`lnav_events` table +that will examine the event contents and perform an action. See the +:ref:`Events` section for more information on handling events. + +Reference +^^^^^^^^^ + +.. jsonschema:: ../schemas/config-v1.schema.json#/properties/log/properties/watch-expressions/patternProperties/([\w\-]+) + +.. _tuning: + +Tuning +------ + +The following configuration options can be used to tune the internals of +**lnav** to your liking. The options can be changed using the :code:`:config` +command. + +.. jsonschema:: ../schemas/config-v1.schema.json#/properties/tuning/properties/archive-manager + +.. jsonschema:: ../schemas/config-v1.schema.json#/properties/tuning/properties/clipboard + +.. jsonschema:: ../schemas/config-v1.schema.json#/definitions/clip-commands + +.. jsonschema:: ../schemas/config-v1.schema.json#/properties/tuning/properties/file-vtab + +.. jsonschema:: ../schemas/config-v1.schema.json#/properties/tuning/properties/logfile + +.. jsonschema:: ../schemas/config-v1.schema.json#/properties/tuning/properties/remote/properties/ssh diff --git a/docs/source/cookbook.rst b/docs/source/cookbook.rst new file mode 100644 index 0000000..3eb0ffd --- /dev/null +++ b/docs/source/cookbook.rst @@ -0,0 +1,104 @@ + +.. _Cookbook: + +Cookbook +======== + +This chapter contains recipes for common tasks that can be done in **lnav**. +These recipes can be used as a starting point for your own needs after some +adaptation. + + +Log Formats +----------- + +TBD + +Defining a New Format +^^^^^^^^^^^^^^^^^^^^^ + +TBD + + +Annotating Logs +--------------- + +Log messages can be annotated in a couple of different ways in **lnav** to help +you get organized. + +Create partitions for Linux boots +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +When digging through logs that can be broken up into multiple sections, +**lnav**'s :ref:`partitioning feature<taking_notes>` can be used to keep track +of which section you are in. For example, if a collection of Linux logs +covered multiple boots, the following script could be used to create partitions +for each boot. After the partition name is set for the log messages, the +current name will show up in the top status bar next to the current time. + +.. literalinclude:: ../../src/scripts/partition-by-boot.lnav + :language: custsqlite + :caption: partition-by-boot.lnav + :linenos: + +Tagging SSH log messages +^^^^^^^^^^^^^^^^^^^^^^^^ + +Log messages can be tagged interactively with the :ref:`:tag<tag>` command or +programmatically using the :ref:`sql-ext`. This example uses a script to +search for interesting SSH messages and automatically adds an appropriate tag. + +.. literalinclude:: ../../example-scripts/tag-ssh-msgs.lnav + :language: custsqlite + :caption: tag-ssh-msgs.lnav + :linenos: + +Log Analysis +------------ + +Most log analysis within **lnav** is done through the :ref:`sql-ext`. The +following examples should give you some ideas to start leveraging this +functionality. One thing to keep in mind is that if a query gets to be too +large or multiple statements need to be executed, you can create a +:code:`.lnav` script that contains the statements and execute it using the +:kbd:`\|` command prompt. + +Count client IPs in web access logs +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +To count the occurrences of an IP in web access logs and order the results +from highest to lowest: + + .. code-block:: custsqlite + + ;SELECT c_ip, count(*) as hits FROM access_log GROUP BY c_ip ORDER BY hits DESC + + +Show only lines where a numeric field is in a range +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The :ref:`:filter-expr<filter_expr>` command can be used to filter web access +logs to only show lines where the number of bytes transferred to the client is +between 10,000 and 40,000 bytes like so: + + .. code-block:: custsqlite + + :filter-expr :sc_bytes BETWEEN 10000 AND 40000 + + +Generating a Report +^^^^^^^^^^^^^^^^^^^ + +Reports can be generated by writing an **lnav** :ref:`script<scripts>` that +uses SQL queries and commands to format a document. A basic script can simply +execute a SQL query that is shown in the DB view. More sophisticated scripts +can use the following commands to generate customized output for a report: + +* The :ref:`:echo<echo>` command to write plain text +* :ref:`SQL queries<sql-ext>` followed by a "write" command, like + :ref:`:write-table-to<write_table_to>`. + +.. literalinclude:: ../../example-scripts/report-demo.lnav + :language: custsqlite + :caption: report-demo.lnav + :linenos: diff --git a/docs/source/data.rst b/docs/source/data.rst new file mode 100644 index 0000000..a7352e1 --- /dev/null +++ b/docs/source/data.rst @@ -0,0 +1,193 @@ + +.. _data-ext: + +Extracting Data +=============== + +**Note**: This feature is still in **BETA**, you should expect bugs and +incompatible changes in the future. + +Log messages contain a good deal of useful data, but it's not always easy to get +at. The log parser built into **lnav** is able to extract data as described by +:ref:`log_formats` as well as discovering data in plain text messages. This data +can then be queried and processed using the SQLite front-end that is also +incorporated into **lnav**. As an example, the following Syslog message from +:code:`sudo` can be processed to extract several key/value pairs:: + + Jul 31 11:42:26 Example-MacBook-Pro.local sudo[87024]: testuser : TTY=ttys004 ; PWD=/Users/testuser/github/lbuild ; USER=root ; COMMAND=/usr/bin/make install + +The data that can be extracted by the parser is viewable directly in **lnav** +by pressing the 'p' key. The results will be shown in an overlay like the +following:: + + Current Time: 2013-07-31T11:42:26.000 Original Time: 2013-07-31T11:42:26.000 Offset: +0.000 + Known message fields: + ├ log_hostname = Example-MacBook-Pro.local + ├ log_procname = sudo + ├ log_pid = 87024 + Discovered message fields: + ├ col_0 = testuser + ├ TTY = ttys004 + ├ PWD = /Users/testuser/github/lbuild + ├ USER = root + └ COMMAND = /usr/bin/make install + +Notice that the parser has detected pairs of the form '<key>=<value>'. The data +parser will also look for pairs separated by a colon. If there are no clearly +demarcated pairs, then the parser will extract anything that looks like data +values and assign them keys of the form 'col_N'. For example, two data values, +an IPv4 address and a symbol, will be extracted from the following log +message:: + + Apr 29 08:13:43 sample-centos5 avahi-daemon[2467]: Registering new address record for 10.1.10.62 on eth0. + +Since there are no keys for the values in the message, the parser will assign +'col_0' for the IP address and 'col_1' for the symbol, as seen here:: + + Current Time: 2013-04-29T08:13:43.000 Original Time: 2013-04-29T08:13:43.000 Offset: +0.000 + Known message fields: + ├ log_hostname = sample-centos5 + ├ log_procname = avahi-daemon + ├ log_pid = 2467 + Discovered message fields: + ├ col_0 = 10.1.10.62 + └ col_1 = eth0 + +Now that you have an idea of how the parser works, you can begin to perform +queries on the data that is being extracted. The SQLite database engine is +embedded into **lnav** and its `Virtual Table +<http://www.sqlite.org/vtab.html>`_ mechanism is used to provide a means to +process this log data. Each log format has its own table that can be used to +access all of the loaded messages that are in that format. For accessing log +message content that is more free-form, like the examples given here, the +**logline** table can be used. The **logline** table is recreated for each +query and is based on the format and pairs discovered in the log message at +the top of the display. + +Queries can be performed by pressing the semi-colon (;) key in **lnav**. After +pressing the key, the overlay showing any known or discovered fields will be +displayed to give you an idea of what data is available. The query can be any +`SQL query <http://sqlite.org/lang.html>`_ supported by SQLite. To make +analysis easier, **lnav** includes many extra functions for processing strings, +paths, and IP addresses. See :ref:`sql-ext` for more information. + +As an example, the simplest query to perform initially would be a "select all", +like so: + +.. code-block:: sql + + SELECT * FROM logline + +When this query is run against the second example log message given above, the +following results are received:: + + log_line log_part log_time log_idle_msecs log_level log_hostname log_procname log_pid col_0 col_1 + + 292 p.0 2013-04-11T16:42:51.000 0 info localhost avahi-daemon 2480 fe80::a00:27ff:fe98:7f6e eth0 + 293 p.0 2013-04-11T16:42:51.000 0 info localhost avahi-daemon 2480 10.0.2.15 eth0 + 330 p.0 2013-04-11T16:47:02.000 0 info localhost avahi-daemon 2480 fe80::a00:27ff:fe98:7f6e eth0 + 336 p.0 2013-04-11T16:47:02.000 0 info localhost avahi-daemon 2480 10.1.10.75 eth0 + 343 p.0 2013-04-11T16:47:02.000 0 info localhost avahi-daemon 2480 10.1.10.75 eth0 + 370 p.0 2013-04-11T16:59:39.000 0 info localhost avahi-daemon 2480 10.1.10.75 eth0 + 377 p.0 2013-04-11T16:59:39.000 0 info localhost avahi-daemon 2480 10.1.10.75 eth0 + 382 p.0 2013-04-11T16:59:41.000 0 info localhost avahi-daemon 2480 fe80::a00:27ff:fe98:7f6e eth0 + 401 p.0 2013-04-11T17:20:45.000 0 info localhost avahi-daemon 4247 fe80::a00:27ff:fe98:7f6e eth0 + 402 p.0 2013-04-11T17:20:45.000 0 info localhost avahi-daemon 4247 10.1.10.75 eth0 + + 735 p.0 2013-04-11T17:41:46.000 0 info sample-centos5 avahi-daemon 2465 fe80::a00:27ff:fe98:7f6e eth0 + 736 p.0 2013-04-11T17:41:46.000 0 info sample-centos5 avahi-daemon 2465 10.1.10.75 eth0 + 781 p.0 2013-04-12T03:32:30.000 0 info sample-centos5 avahi-daemon 2465 10.1.10.64 eth0 + 788 p.0 2013-04-12T03:32:30.000 0 info sample-centos5 avahi-daemon 2465 10.1.10.64 eth0 + 1166 p.0 2013-04-25T10:56:00.000 0 info sample-centos5 avahi-daemon 2467 fe80::a00:27ff:fe98:7f6e eth0 + 1167 p.0 2013-04-25T10:56:00.000 0 info sample-centos5 avahi-daemon 2467 10.1.10.111 eth0 + 1246 p.0 2013-04-26T06:06:25.000 0 info sample-centos5 avahi-daemon 2467 10.1.10.49 eth0 + 1253 p.0 2013-04-26T06:06:25.000 0 info sample-centos5 avahi-daemon 2467 10.1.10.49 eth0 + 1454 p.0 2013-04-28T06:53:55.000 0 info sample-centos5 avahi-daemon 2467 10.1.10.103 eth0 + 1461 p.0 2013-04-28T06:53:55.000 0 info sample-centos5 avahi-daemon 2467 10.1.10.103 eth0 + + 1497 p.0 2013-04-29T08:13:43.000 0 info sample-centos5 avahi-daemon 2467 10.1.10.62 eth0 + 1504 p.0 2013-04-29T08:13:43.000 0 info sample-centos5 avahi-daemon 2467 10.1.10.62 eth0 + +Note that **lnav** is not returning results for all messages that are in this +syslog file. Rather, it searches for messages that match the format for the +given line and returns only those messages in results. In this case, that +format is "Registering new address record for <IP> on <symbol>", which +corresponds to the parts of the message that were not recognized as data. + +More sophisticated queries can be done, of course. For example, to find out the +frequency of IP addresses mentioned in these messages, you can run: + +.. code-block:: sql + + SELECT col_0,count(*) FROM logline GROUP BY col_0 + +The results for this query are:: + + col_0 count(*) + + 10.0.2.15 1 + 10.1.10.49 2 + 10.1.10.62 2 + 10.1.10.64 2 + 10.1.10.75 6 + 10.1.10.103 2 + 10.1.10.111 1 + fe80::a00:27ff:fe98:7f6e 6 + +Since this type of query is fairly common, **lnav** includes a "summarize" +command that will compute the frequencies of identifiers as well as min, max, +average, median, and standard deviation for number columns. In this case, you +can run the following to compute the frequencies and return an ordered set of +results:: + + :summarize col_0 + + +Recognized Data Types +--------------------- + +When searching for data to extract from log messages, **lnav** looks for the +following set of patterns: + + +Strings + Single and double-quoted strings. Example: "The quick brown fox." + +URLs + URLs that contain the '://' separator. Example: http://example.com + +Paths + File system paths. Examples: /path/to/file, ./relative/path + +MAC Address + Ethernet MAC addresses. Example: c4:2c:03:0e:e4:4a + +Hex Dumps + A colon-separated string of hex numbers. Example: e8:06:88:ff + +Date/Time + Date and time stamps of the form "YYYY-mm-DD" and "HH:MM:SS". + +IP Addresses + IPv4 and IPv6 addresses. Examples: 127.0.0.1, fe80::c62c:3ff:fe0e:e44a%en0 + +UUID + The common formatting for 128-bit UUIDs. Example: + 0E305E39-F1E9-4DE4-B10B-5829E5DF54D0 + +Version Numbers + Dot-separated version numbers. Example: 3.7.17 + +Numbers + Numbers in base ten, hex, and octal formats. Examples: 1234, 0xbeef, 0777 + +E-Mail Address + Strings that look close to an e-mail address. Example: gary@example.com + +Constants + Common constants in languages, like: true, false, null, None. + +Symbols + Words that follow the common conventions for symbols in programming + languages. For example, containing all capital letters, or separated + by colons. Example: SOME_CONSTANT_VALUE, namespace::value diff --git a/docs/source/docutils.conf b/docs/source/docutils.conf new file mode 100644 index 0000000..1bf4d83 --- /dev/null +++ b/docs/source/docutils.conf @@ -0,0 +1,2 @@ +[restructuredtext parser] +syntax_highlight = short diff --git a/docs/source/events.rst b/docs/source/events.rst new file mode 100644 index 0000000..8318abf --- /dev/null +++ b/docs/source/events.rst @@ -0,0 +1,56 @@ +.. _Events: + +Events (v0.11.0+) +================= + +The events mechanism allows **lnav** to be automated based on events that +occur during processing. For example, filters could be added only when a +particular log file format is detected instead of always installing them. +Events are published through the :ref:`lnav_events<table_lnav_events>` SQLite +table. Reacting to events can be done by creating a SQLite trigger on the +table and inspecting the content of the event. + +Trigger Example +--------------- + +The following is an example of a trigger that adds an out filter when a +syslog file is loaded. You can copy the code into an :file:`.sql` file and +install it by running :code:`lnav -i my_trigger.sql`. + +.. code-block:: sql + :caption: my_trigger.sql + :linenos: + + CREATE TRIGGER IF NOT EXISTS add_format_specific_filters + AFTER INSERT ON lnav_events WHEN + -- Check the event type + jget(NEW.content, '/$schema') = + 'https://lnav.org/event-file-format-detected-v1.schema.json' AND + -- Only create the filter when a given format is seen + jget(NEW.content, '/format') = 'syslog_log' AND + -- Don't create the filter if it's already there + NOT EXISTS ( + SELECT 1 FROM lnav_view_filters WHERE pattern = 'noisy message') + BEGIN + INSERT INTO lnav_view_filters (view_name, enabled, type, pattern) VALUES + ('log', 1, 'OUT', 'noisy message'); + END; + +.. _event_reference: + +Reference +--------- + +The following tables describe the schema of the event JSON objects. + +.. jsonschema:: ../schemas/event-file-open-v1.schema.json# + :lift_description: + +.. jsonschema:: ../schemas/event-file-format-detected-v1.schema.json# + :lift_description: + +.. jsonschema:: ../schemas/event-log-msg-detected-v1.schema.json# + :lift_description: + +.. jsonschema:: ../schemas/event-session-loaded-v1.schema.json# + :lift_description: diff --git a/docs/source/faq.rst b/docs/source/faq.rst new file mode 100644 index 0000000..6ca1d92 --- /dev/null +++ b/docs/source/faq.rst @@ -0,0 +1,57 @@ + +.. _faq: + +Frequently Asked Questions +========================== + +Q: How can I copy & paste without decorations? +---------------------------------------------- + +:Answer: There are a few ways to do this: + + * Use the :ref:`bookmark<hotkeys_bookmarks>` hotkeys to mark lines and then + press :kbd:`c` to copy to the local system keyboard. + + * Press :kbd:`CTRL` + :kbd:`l` to temporarily switch to "lo-fi" + mode where the contents of the current view are printed to the terminal. + This option is useful when you are logged into a remote host. + + +Q: How can I force a format for a file? +--------------------------------------- + +:Answer: The log format for a file is automatically detected and cannot be + forced. + +:Solution: Add some of the log file lines to the :ref:`sample<format_sample>` + array and then startup lnav to get a detailed explanation of where the format + patterns are not matching the sample lines. + +:Details: The first lines of the file are matched against the + :ref:`regular expressions defined in the format definitions<format_regex>`. + The order of the formats is automatically determined so that more specific + formats are tried before more generic ones. Therefore, if the expected + format is not being chosen for a file, then it means the regular expressions + defined by that format are not matching the first few lines of the file. + + See :ref:`format_order` for more information. + +Q: Why isn't my log file highlighted correctly? +----------------------------------------------- + +TBD + +Q: Why isn't a file being displayed? +------------------------------------ + +:Answer: Plaintext files are displayed separately from log files in the TEXT + view. + +:Solution: Press the :kbd:`t` key to switch to the text view. Or, open the + files configuration panel by pressing :kbd:`TAB` to cycle through the + panels, and then press :kbd:`/` to search for the file you're interested in. + If the file is a log, a new :ref:`log format<log_formats>` will need to be + created or an existing one modified. + +:Details: If a file being monitored by lnav does not match a known log file + format, it is treated as plaintext and will be displayed in the TEXT view. diff --git a/docs/source/filter-out-preview.png b/docs/source/filter-out-preview.png Binary files differnew file mode 100644 index 0000000..8f9816f --- /dev/null +++ b/docs/source/filter-out-preview.png diff --git a/docs/source/formats.rst b/docs/source/formats.rst new file mode 100644 index 0000000..8960761 --- /dev/null +++ b/docs/source/formats.rst @@ -0,0 +1,527 @@ +.. _log_formats: + +Log Formats +=========== + +Log files loaded into **lnav** are parsed based on formats defined in +configuration files. Many +formats are already built in to the **lnav** binary and you can define your own +using a JSON file. When loading files, each format is checked to see if it can +parse the first few lines in the file. Once a match is found, that format will +be considered that files format and used to parse the remaining lines in the +file. If no match is found, the file is considered to be plain text and can +be viewed in the "text" view that is accessed with the **t** key. + +The following log formats are built into **lnav**: + +.. csv-table:: + :header: "Name", "Table Name", "Description" + :widths: 8 5 20 + :file: format-table.csv + +In addition to the above formats, the following self-describing formats are +supported: + +* The + `Bro Network Security Monitor <https://www.bro.org/sphinx/script-reference/log-files.html>`_ + TSV log format is supported in lnav versions v0.8.3+. The Bro log format is + self-describing, so **lnav** will read the header to determine the shape of + the file. +* The + `W3C Extend Log File Format <https://www.w3.org/TR/WD-logfile.html>`_ + is supported in lnav versions v0.10.0+. The W3C log format is + self-describing, so **lnav** will read the header to determine the shape of + the file. + +There is also basic support for the `logfmt <https://brandur.org/logfmt>`_ +convention for formatting log messages. Files that use this format must +have the entire line be key/value pairs and the timestamp contained in a +field named :code:`time` or :code:`ts`. If the file you're using does not +quite follow this formatting, but wraps logfmt data with another recognized +format, you can use the :ref:`logfmt2json` SQL function to convert the data +into JSON for further analysis. + + +Defining a New Format +--------------------- + +New log formats can be defined by placing JSON configuration files in +subdirectories of the :file:`~/.lnav/formats/` directory. The directories and +files can be named anything you like, but the files must have the '.json' suffix. A +sample file containing the builtin configuration will be written to this +directory when **lnav** starts up. You can consult that file when writing your +own formats or if you need to modify existing ones. Format directories can +also contain '.sql' and '.lnav' script files that can be used automate log file +analysis. + +Creating a Format Using Regex101.com (v0.11.0+) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +For plain-text log files, the easiest way to create a log format definition is +to create the regular expression that recognizes log messages using +https://regex101.com . Simply copy a log line into the test string input box +on the site and then start editing the regular expression. When building the +regular expression, you'll want to use named captures for the structured parts +of the log message. Any raw message text should be matched by a captured named +"body". Once you have a regex that matches the whole log message, you can use +**lnav**'s "management CLI" to create a skeleton format file. The skeleton +will be populated with the regular expression from the site and the test +string, along with any unit tests, will be added to the "samples" list. The +"regex101 import" management command is used to create the skeleton and has +the following form: + +.. prompt:: bash + + lnav -m regex101 import <regex101-url> <format-name> [<regex-name>] + +If the import was successful, the path to the new format file should be +printed out. The skeleton will most likely need some changes to make it +fully functional. For example, the :code:`kind` properties for captured values +default to :code:`string`, but you'll want to change them to the appropriate +type. + +Format File Reference +^^^^^^^^^^^^^^^^^^^^^ + +An **lnav** format file must contain a single JSON object, preferably with a +:code:`$schema` property that refers to the +`format-v1.schema <https://lnav.org/schemas/format-v1.schema.json>`_, +like so: + +.. code-block:: json + + { + "$schema": "https://lnav.org/schemas/format-v1.schema.json" + } + +Each format to be defined in the file should be a separate field in the top-level +object. The field name should be the symbolic name of the format. This value +will also be used as the SQL table name for the log. The value for each field +should be another object with the following fields: + + :title: The short and human-readable name for the format. + :description: A longer description of the format. + :url: A URL to the definition of the format. + + :file-pattern: A regular expression used to match log file paths. Typically, + every file format will be tried during the detection process. This field + can be used to limit which files a format is applied to in case there is + a potential for conflicts. + + .. _format_regex: + + :regex: This object contains sub-objects that describe the message formats + to match in a plain-text log file. Each :code:`regex` MUST only match one + type of log message. It must not match log messages that are matched by + other regexes in this format. This uniqueness requirement is necessary + because **lnav** will "lock-on" to a regex and use it to match against + the next line in a file. So, if the regexes do not uniquely match each + type of log message, messages can be matched by the wrong regex. The + "lock-on" behavior is needed to avoid the performance hit of having to + try too many different regexes. + + .. note:: Log files that contain JSON messages should not specify this field. + + :pattern: The regular expression that should be used to match log messages. + The `PCRE2 <http://www.pcre.org>`_ library is used by **lnav** to do all + regular expression matching. + + :module-format: If true, this regex will only be used to parse message + bodies for formats that can act as containers, such as syslog. Default: + false. + + :json: True if each log line is JSON-encoded. + + :line-format: An array that specifies the text format for JSON-encoded + log messages. Log files that are JSON-encoded will have each message + converted from the raw JSON encoding into this format. Each element + is either an object that defines which fields should be inserted into + the final message string and or a string constant that should be + inserted. For example, the following configuration will tranform each + log message object into a string that contains the timestamp, followed + by a space, and then the message body: + + .. code-block:: json + + [ { "field": "ts" }, " ", { "field": "msg" } ] + + :field: The name or `JSON-Pointer <https://tools.ietf.org/html/rfc6901>`_ + of the message field that should be inserted at this point in the + message. The special :code:`__timestamp__` field name can be used to + insert a human-readable timestamp. The :code:`__level__` field can be + used to insert the level name as defined by lnav. + + .. tip:: + + Use a JSON-Pointer to reference nested fields. For example, to include + a "procname" property that is nested in a "details" object, you would + write the field reference as :code:`/details/procname`. + + :min-width: The minimum width for the field. If the value for the field + in a given log message is shorter, padding will be added as needed to + meet the minimum-width requirement. (v0.8.2+) + :max-width: The maximum width for the field. If the value for the field + in a given log message is longer, the overflow algorithm will be applied + to try and shorten the field. (v0.8.2+) + :align: Specifies the alignment for the field, either "left" or "right". + If "left", padding to meet the minimum-width will be added on the right. + If "right", padding will be added on the left. (v0.8.2+) + :overflow: The algorithm used to shorten a field that is longer than + "max-width". The following algorithms are supported: + + :abbrev: Removes all but the first letter in dotted text. For example, + "com.example.foo" would be shortened to "c.e.foo". + :truncate: Truncates any text past the maximum width. + :dot-dot: Cuts out the middle of the text and replaces it with two + dots (i.e. '..'). + + (v0.8.2+) + :timestamp-format: The timestamp format to use when displaying the time + for this log message. (v0.8.2+) + :default-value: The default value to use if the field could not be found + in the current log message. The built-in default is "-". + :text-transform: Transform the text in the field. Supported options are: + none, uppercase, lowercase, capitalize + + :timestamp-field: The name of the field that contains the log message + timestamp. Defaults to "timestamp". + + :timestamp-format: An array of timestamp formats using a subset of the + strftime conversion specification. The following conversions are + supported: %a, %b, %L, %M, %H, %I, %d, %e, %k, %l, %m, %p, %y, %Y, %S, %s, + %Z, %z. In addition, you can also use the following: + + :%L: Milliseconds as a decimal number (range 000 to 999). + :%f: Microseconds as a decimal number (range 000000 to 999999). + :%N: Nanoseconds as a decimal number (range 000000000 to 999999999). + :%q: Seconds from the epoch as a hexidecimal number. + :%i: Milliseconds from the epoch. + :%6: Microseconds from the epoch. + + :timestamp-divisor: For JSON logs with numeric timestamps, this value is used + to divide the timestamp by to get the number of seconds and fractional + seconds. + + :subsecond-field: (v0.11.1+) The path to the property in a JSON-lines log + message that contains the sub-second time value + + :subsecond-units: (v0.11.1+) The units of the subsecond-field property value. + The following values are supported: + + :milli: for milliseconds + :micro: for microseconds + :nano: for nanoseconds + + :ordered-by-time: (v0.8.3+) Indicates that the order of messages in the file + is time-based. Files that are not naturally ordered by time will be sorted + in order to display them in the correct order. Note that this sorting can + incur a performance penalty when tailing logs. + + :level-field: The name of the regex capture group that contains the log + message level. Defaults to "level". + + :body-field: The name of the field that contains the main body of the + message. Defaults to "body". + + :opid-field: The name of the field that contains the "operation ID" of the + message. An "operation ID" establishes a thread of messages that might + correspond to a particular operation/request/transaction. The user can + press the 'o' or 'Shift+O' hotkeys to move forward/backward through the + list of messages that have the same operation ID. Note: For JSON-encoded + logs, the opid field can be a path (e.g. "foo/bar/opid") if the field is + nested in an object and it MUST be included in the "line-format" for the + 'o' hotkeys to work. + + :module-field: The name of the field that contains the module identifier + that distinguishes messages from one log source from another. This field + should be used if this message format can act as a container for other + types of log messages. For example, an Apache access log can be sent to + syslog instead of written to a file. In this case, **lnav** will parse + the syslog message and then separately parse the body of the message to + determine the "sub" format. This module identifier is used to help + **lnav** quickly identify the format to use when parsing message bodies. + + :hide-extra: A boolean for JSON logs that indicates whether fields not + present in the line-format should be displayed on their own lines. + + :level: A mapping of error levels to regular expressions. During scanning + the contents of the capture group specified by *level-field* will be + checked against each of these regexes. Once a match is found, the log + message level will set to the corresponding level. The available levels, + in order of severity, are: **fatal**, **critical**, **error**, + **warning**, **stats**, **info**, **debug**, **debug2-5**, **trace**. + For JSON logs with exact numeric levels, the number for the corresponding + level can be supplied. If the JSON log format uses numeric ranges instead + of exact numbers, you can supply a pattern and the number found in the log + will be converted to a string for pattern-matching. + + :multiline: If false, **lnav** will consider any log lines that do not + match one of the message patterns to be in error when checking files with + the '-C' option. This flag will not affect normal viewing operation. + Default: true. + + :value: This object contains the definitions for the values captured by the + regexes. + + :kind: The type of data that was captured **string**, **integer**, + **float**, **json**, **quoted**. + :collate: The name of the SQLite collation function for this value. + The standard SQLite collation functions can be used as well as the + ones defined by lnav, as described in :ref:`collators`. + :identifier: A boolean that indicates whether or not this field represents + an identifier and should be syntax colored. + :foreign-key: A boolean that indicates that this field is a key and should + not be graphed. This should only need to be set for integer fields. + :hidden: A boolean for log fields that indicates whether they should + be displayed. The behavior is slightly different for JSON logs and text + logs. For a JSON log, this property determines whether an extra line + will be added with the key/value pair. For text logs, this property + controls whether the value should be displayed by default or replaced + with an ellipsis. + :rewriter: A command to rewrite this field when pretty-printing log + messages containing this value. The command must start with ':', ';', + or '|' to signify whether it is a regular command, SQL query, or a script + to be executed. The other fields in the line are accessible in SQL by + using the ':' prefix. The text value of this field will then be replaced + with the result of the command when pretty-printing. For example, the + HTTP access log format will rewrite the status code field to include the + textual version (e.g. 200 (OK)) using the following SQL query: + + .. code-block:: sql + + ;SELECT :sc_status || ' (' || ( + SELECT message FROM http_status_codes + WHERE status = :sc_status) || ') ' + + :tags: This object contains the tags that should automatically be added to + log messages. + + :pattern: The regular expression evaluated over a line in the log file as + it is read in. If there is a match, the log message the line is a part + of will have this tag added to it. + :paths: This array contains objects that define restrictions on the file + paths that the tags will be applied to. The objects in this array can + contain: + + :glob: A glob pattern to check against the log files read by lnav. + + .. _format_sample: + + :sample: A list of objects that contain sample log messages. All formats + must include at least one sample and it must be matched by one of the + included regexes. Each object must contain the following field: + + :line: The sample message. + :level: The expected error level. An error will be raised if this level + does not match the level parsed by lnav for this sample message. + + :highlights: This object contains the definitions for patterns to be + highlighted in a log message. Each entry should have a name and a + definition with the following fields: + + :pattern: The regular expression to match in the log message body. + :color: The foreground color to use when highlighting the part of the + message that matched the pattern. If no color is specified, one will be + picked automatically. Colors can be specified using hexadecimal notation + by starting with a hash (e.g. #aabbcc) or using a color name as found + at http://jonasjacek.github.io/colors/. + :background-color: The background color to use when highlighting the part + of the message that matched the pattern. If no background color is + specified, black will be used. The background color is only considered + if a foreground color is specified. + :underline: If true, underline the part of the message that matched the + pattern. + :blink: If true, blink the part of the message that matched the pattern. + +Example format: + +.. code-block:: json + + { + "$schema": "https://lnav.org/schemas/format-v1.schema.json", + "example_log" : { + "title" : "Example Log Format", + "description" : "Log format used in the documentation example.", + "url" : "http://example.com/log-format.html", + "regex" : { + "basic" : { + "pattern" : "^(?<timestamp>\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}Z)>>(?<level>\\w+)>>(?<component>\\w+)>>(?<body>.*)$" + } + }, + "level-field" : "level", + "level" : { + "error" : "ERROR", + "warning" : "WARNING" + }, + "value" : { + "component" : { + "kind" : "string", + "identifier" : true + } + }, + "sample" : [ + { + "line" : "2011-04-01T15:14:34.203Z>>ERROR>>core>>Shit's on fire yo!" + } + ] + } + } + +Patching an Existing Format +--------------------------- + +When loading log formats from files, **lnav** will overlay any new data over +previously loaded data. This feature allows you to override existing value or +append new ones to the format configurations. For example, you can separately +add a new regex to the example log format given above by creating another file +with the following contents: + +.. code-block:: json + + { + "$schema": "https://lnav.org/schemas/format-v1.schema.json", + "example_log" : { + "regex" : { + "custom1" : { + "pattern" : "^(?<timestamp>\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}Z)<<(?<level>\\w+)--(?<component>\\w+)>>(?<body>.*)$" + } + }, + "sample" : [ + { + "line" : "2011-04-01T15:14:34.203Z<<ERROR--core>>Shit's on fire yo!" + } + ] + } + } + +.. _scripts: + +Scripts +------- + +Format directories may also contain :file:`.sql` and :file:`.lnav` files to help automate +log file analysis. The SQL files are executed on startup to create any helper +tables or views and the '.lnav' script files can be executed using the pipe +hotkey :kbd:`|`. For example, **lnav** includes a "partition-by-boot" script that +partitions the log view based on boot messages from the Linux kernel. A script +can have a mix of SQL and **lnav** commands, as well as include other scripts. +The type of statement to execute is determined by the leading character on a +line: a semi-colon begins a SQL statement; a colon starts an **lnav** command; +and a pipe :code:`|` denotes another script to be executed. Lines beginning with a +hash are treated as comments. The following variables are defined in a script: + +.. envvar:: # + + The number of arguments passed to the script. + +.. envvar:: __all__ + + A string containing all the arguments joined by a single space. + +.. envvar:: 0 + + The path to the script being executed. + +.. envvar:: 1-N + + The arguments passed to the script. + +Remember that you need to use the :ref:`:eval<eval>` command when referencing +variables in most **lnav** commands. Scripts can provide help text to be +displayed during interactive usage by adding the following tags in a comment +header: + + :@synopsis: The synopsis should contain the name of the script and any + parameters to be passed. For example:: + + # @synopsis: hello-world <name1> [<name2> ... <nameN>] + + :@description: A one-line description of what the script does. For example:: + + # @description: Say hello to the given names. + + + +.. tip:: + + The :ref:`:eval<eval>` command can be used to do variable substitution for + commands that do not natively support it. For example, to substitute the + variable, :code:`pattern`, in a :ref:`:filter-out<filter_out>` command: + + .. code-block:: lnav + + :eval :filter-out ${pattern} + +VSCode Extension +^^^^^^^^^^^^^^^^ + +The `lnav VSCode Extension <https://marketplace.visualstudio.com/items?itemName=lnav.lnav>`_ +can be installed to add syntax highlighting to lnav scripts. + +Installing Formats +------------------ + +File formats are loaded from subdirectories in :file:`/etc/lnav/formats` and +:file:`~/.lnav/formats/`. You can manually create these subdirectories and +copy the format files into there. Or, you can pass the '-i' option to **lnav** +to automatically install formats from the command-line. For example: + +.. code-block:: bash + + $ lnav -i myformat.json + info: installed: /home/example/.lnav/formats/installed/myformat_log.json + +Format files installed using this method will be placed in the :file:`installed` +subdirectory and named based on the first format name found in the file. + +You can also install formats from git repositories by passing the repository's +clone URL. A standard set of repositories is maintained at +(https://github.com/tstack/lnav-config) and can be installed by passing 'extra' +on the command line, like so: + +.. prompt:: bash + + lnav -i extra + +These repositories can be updated by running **lnav** with the '-u' flag. + +Format files can also be made executable by adding a shebang (#!) line to the +top of the file, like so:: + + #! /usr/bin/env lnav -i + { + "myformat_log" : ... + } + +Executing the format file should then install it automatically: + +.. code-block:: bash + + $ chmod ugo+rx myformat.json + $ ./myformat.json + info: installed: /home/example/.lnav/formats/installed/myformat_log.json + +.. _format_order: + +Format Order When Scanning a File +--------------------------------- + +When **lnav** loads a file, it tries each log format against the first 15,000 +lines [#]_ of the file trying to find a match. When a match is found, that log +format will be locked in and used for the rest of the lines in that file. +Since there may be overlap between formats, **lnav** performs a test on +startup to determine which formats match each others sample lines. Using +this information it will create an ordering of the formats so that the more +specific formats are tried before the more generic ones. For example, a +format that matches certain syslog messages will match its own sample lines, +but not the ones in the syslog samples. On the other hand, the syslog format +will match its own samples and those in the more specific format. You can +see the order of the format by enabling debugging and checking the **lnav** +log file for the "Format order" message: + +.. prompt:: bash + + lnav -d /tmp/lnav.log + +.. [#] The maximum number of lines to check can be configured. See the + :ref:`tuning` section for more details. diff --git a/docs/source/group_concat-help.png b/docs/source/group_concat-help.png Binary files differnew file mode 100644 index 0000000..f7b6f08 --- /dev/null +++ b/docs/source/group_concat-help.png diff --git a/docs/source/hotkey-tips.png b/docs/source/hotkey-tips.png Binary files differnew file mode 100644 index 0000000..5b78686 --- /dev/null +++ b/docs/source/hotkey-tips.png diff --git a/docs/source/hotkeys.rst b/docs/source/hotkeys.rst new file mode 100644 index 0000000..f91c1e8 --- /dev/null +++ b/docs/source/hotkeys.rst @@ -0,0 +1,289 @@ +.. _hotkeys: + +Hotkey Reference +================ + +This reference covers the keys used to control **lnav**. Consult the `built-in +help <https://github.com/tstack/lnav/blob/master/src/help.txt>`_ in **lnav** for +a more detailed explanation of each key. + +Spatial Navigation +------------------ + +The majority of these hotkeys should be available in all views. + +.. list-table:: + :header-rows: 1 + :widths: 6 6 6 20 + + * - Keypress + - + - + - Command + * - :kbd:`Space` + - :kbd:`PgDn` + - + - Down a page + * - :kbd:`b` + - :kbd:`Backspace` + - :kbd:`PgUp` + - Up a page + * - :kbd:`j` + - :kbd:`↓` + - + - Down a line + * - :kbd:`k` + - :kbd:`↑` + - + - Up a line + * - :kbd:`h` + - :kbd:`←` + - + - Left half a page. In the log view, pressing left while at the start of + the message text will reveal the source file name for each line. + Pressing again will reveal the full path. + * - :kbd:`Shift` + :kbd:`h` + - :kbd:`Shift` + :kbd:`←` + - + - Left ten columns + * - :kbd:`l` + - :kbd:`→` + - + - Right half a page + * - :kbd:`Shift` + :kbd:`l` + - :kbd:`Shift` + :kbd:`→` + - + - Right ten columns + * - :kbd:`Home` + - :kbd:`g` + - + - Top of the view + * - :kbd:`End` + - :kbd:`G` + - + - Bottom of the view + * - :kbd:`e` + - :kbd:`Shift` + :kbd:`e` + - + - Next/previous error + * - :kbd:`w` + - :kbd:`Shift` + :kbd:`w` + - + - Next/previous warning + * - :kbd:`n` + - :kbd:`Shift` + :kbd:`n` + - + - Next/previous search hit + * - :kbd:`>` + - :kbd:`<` + - + - Next/previous search hit (horizontal) + * - :kbd:`f` + - :kbd:`Shift` + :kbd:`f` + - + - Next/previous file + * - :kbd:`u` + - :kbd:`Shift` + :kbd:`u` + - + - Next/previous bookmark + * - :kbd:`o` + - :kbd:`Shift` + :kbd:`o` + - + - Forward/backward through log messages with a matching "opid" field + * - :kbd:`s` + - :kbd:`Shift` + :kbd:`s` + - + - Next/previous slow down in the log message rate + * - :kbd:`{` + - :kbd:`}` + - + - Previous/next location in history + +Chronological Navigation +------------------------ + +These hotkeys are only functional on views that are time-based, like the log +view or the histogram view. + +.. list-table:: + :header-rows: 1 + :widths: 5 5 20 + + * - Keypress + - + - Command + * - :kbd:`d` + - :kbd:`Shift` + :kbd:`d` + - Forward/backward 24 hours + * - :kbd:`1` - :kbd:`6` + - :kbd:`Shift` + :kbd:`1` - :kbd:`6` + - Next/previous n'th ten minute of the hour + * - :kbd:`7` + - :kbd:`8` + - Previous/next minute + * - :kbd:`0` + - :kbd:`Shift` + :kbd:`0` + - Next/previous day + * - :kbd:`r` + - :kbd:`Shift` + :kbd:`r` + - Forward/backward by the relative time that was last used with the goto command. + +Breadcrumb Navigation +--------------------- + +The following hotkeys are related to the breadcrumb bar that is below the top +status bar. + +.. list-table:: + :header-rows: 1 + :widths: 5 20 + + * - Keypress + - Description + * - :kbd:`ENTER` + - Focus on the breadcrumb bar. Or, if the bar is currently focused, + accept the selected value and drop focus. + * - :kbd:`Escape` + - Drop focus on the breadcrumb bar. + * - :kbd:`←` + - Select the crumb to the left. If the first crumb is selected, the + selection will wrap around to the last crumb. + * - :kbd:`→` + - Accept the current value, which might mean navigating to the value in + the view, then selecting the crumb to the right. + * - :kbd:`Ctrl` + :kbd:`a` + - Select the first crumb. + * - :kbd:`Ctrl` + :kbd:`e` + - Select the last crumb. + * - :kbd:`↓` + - Select the next value in the crumb dropdown. + * - :kbd:`↑` + - Select the previous value in the crumb dropdown. + * - :kbd:`Home` + - Select the first value in the crumb dropdown. + * - :kbd:`End` + - Select the last value in the crumb dropdown. + +While a crumb is selected, you can perform a fuzzy search on the possible +values by typing in the value you are interested in. + +.. _hotkeys_bookmarks: + +Bookmarks +--------- + +.. list-table:: + :header-rows: 1 + :widths: 5 20 + + * - Keypress + - Command + * - :kbd:`m` + - Mark/unmark the top line + * - :kbd:`Shift` + :kbd:`m` + - Mark/unmark the range of lines from the last marked to the top + * - :kbd:`Shift` + :kbd:`j` + - Mark/unmark the next line after the previously marked + * - :kbd:`Shift` + :kbd:`k` + - Mark/unmark the previous line + * - :kbd:`c` + - Copy marked lines to the clipboard + * - :kbd:`Shift` + :kbd:`c` + - Clear marked lines + +.. _hotkeys_display: + +Display +------- + +.. list-table:: + :header-rows: 1 + :widths: 5 20 + + * - Keypress + - Command + * - :kbd:`?` + - View/leave builtin help + * - :kbd:`q` + - Return to the previous view/quit + * - :kbd:`Shift` + :kbd:`q` + - Return to the previous view/quit while matching the top times of the two views + * - :kbd:`a` + - Restore the view that was previously popped with 'q/Q' + * - :kbd:`Shift` + :kbd:`a` + - Restore the view that was previously popped with 'q/Q' and match the top times of the views + * - :kbd:`Shift` + :kbd:`p` + - Switch to/from the pretty-printed view of the displayed log or text files + * - :kbd:`Shift` + :kbd:`t` + - Display elapsed time between lines + * - :kbd:`t` + - Switch to/from the text file view + * - :kbd:`i` + - Switch to/from the histogram view + * - :kbd:`Shift` + :kbd:`i` + - Switch to/from the histogram view + * - :kbd:`v` + - Switch to/from the SQL result view + * - :kbd:`Shift` + :kbd:`v` + - Switch to/from the SQL result view and move to the corresponding in the + log_line column + * - :kbd:`p` + - Toggle the display of the log parser results + * - :kbd:`Tab` + - In the log/text views, focus on the configuration panel for editing + filters and examining the list of loaded files. In the SQL result view, + cycle through columns to display as bar graphs + * - :kbd:`Ctrl` + :kbd:`l` + - Switch to lo-fi mode. The displayed log lines will be dumped to the + terminal without any decorations so they can be copied easily. + * - :kbd:`Ctrl` + :kbd:`w` + - Toggle word-wrap. + * - :kbd:`Ctrl` + :kbd:`p` + - Show/hide the data preview panel that may be opened when entering + commands or SQL queries. + * - :kbd:`Ctrl` + :kbd:`f` + - Toggle the enabled/disabled state of all filters in the current view. + * - :kbd:`x` + - Toggle the hiding of log message fields. The hidden fields will be + replaced with three bullets and highlighted in yellow. + * - :kbd:`=` + - Pause/unpause loading of new file data. + +Session +------- + +.. list-table:: + :header-rows: 1 + :widths: 5 20 + + * - Keypress + - Command + * - :kbd:`Ctrl` + :kbd:`R` + - Reset the current :ref:`session<sessions>` state. The session state + includes things like filters, bookmarks, and hidden fields. + +Query Prompts +------------- + +.. list-table:: + :header-rows: 1 + :widths: 5 20 + + * - Keypress + - Command + * - :kbd:`/` + - Search for lines matching a regular expression + * - :kbd:`;` + - Open the :ref:`sql-ext` to execute SQL statements/queries + * - :kbd:`:` + - Execute an internal command, see :ref:`commands` for more information + * - :kbd:`\|` + - Execute an lnav script located in a format directory + * - :kbd:`Ctrl` + :kbd:`]` + - Abort the prompt + +Customizing +----------- + +You can customize the behavior of hotkeys by defining your own keymaps. +Consult the :ref:`Keymaps<keymaps>` configuration section for more information. diff --git a/docs/source/howitworks.rst b/docs/source/howitworks.rst new file mode 100644 index 0000000..d111fd0 --- /dev/null +++ b/docs/source/howitworks.rst @@ -0,0 +1,13 @@ + +.. _howitworks: + +How It Works +============ + +"Magic" + +Internal Architecture +--------------------- + +The `ARCHITECTURE.md <https://github.com/tstack/lnav/blob/master/ARCHITECTURE.md>`_ +file in the source tree contains some information about lnav's internals. diff --git a/docs/source/index.rst b/docs/source/index.rst new file mode 100644 index 0000000..d94c0eb --- /dev/null +++ b/docs/source/index.rst @@ -0,0 +1,35 @@ +Welcome to lnav's documentation! +================================ + +The `Log File Navigator <http://lnav.org>`_ (**lnav**) is an advanced log file +viewer for the console. + +Contents: + +.. toctree:: + :maxdepth: 2 + + intro + ui + hotkeys + cli + usage + cookbook + config + formats + sessions + commands + sqlext + sqltab + events + data + howitworks + faq + + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` diff --git a/docs/source/intro.rst b/docs/source/intro.rst new file mode 100644 index 0000000..ae3164c --- /dev/null +++ b/docs/source/intro.rst @@ -0,0 +1,131 @@ +Introduction +============ + +The Log File Navigator, **lnav**, is an advanced log file viewer for the +terminal. It provides an :ref:`easy-to-use interface<ui>` for monitoring and +analyzing your log files with little to no setup. Simply point **lnav** at +your log files and it will automatically detect the :ref:`log_formats`, index +their contents, and display a combined view of all log messages. You can +navigate through your logs using a variety of :ref:`hotkeys<hotkeys>`. +:ref:`Commands<commands>` give you additional control over **lnav**'s behavior +for doing things like applying filters, tagging messages, and more. You can +then analyze your log messages using the :ref:`sql-ext`. + +Dependencies +------------ + +When compiling from source, the following dependencies are required: + +* `NCurses <http://www.gnu.org/s/ncurses/>`_ +* `PCRE2 <http://www.pcre.org>`_ +* `SQLite <http://www.sqlite.org>`_ +* `ZLib <http://wwww.zlib.net>`_ +* `Bzip2 <http://www.bzip.org>`_ +* `Readline <http://www.gnu.org/s/readline>`_ +* `libcurl <https://curl.haxx.se>`_ +* `libarchive <https://libarchive.org>`_ + +Installation +------------ + +Check the `downloads page <http://lnav.org/downloads>`_ to see if there are +packages for your operating system. To compile from source, use the following +commands: + +.. prompt:: bash + + ./configure + make + sudo make install + + +Viewing Logs +------------ + +The arguments to **lnav** are the log files, directories, or URLs to be viewed. +For example, to view all of the CUPS logs on your system: + +.. prompt:: bash + + lnav /var/log/cups + +The formats of the logs are determined automatically and indexed on-the-fly. +See :ref:`log_formats` for a listing of the predefined formats and how to +define your own. + +If no arguments are given, **lnav** will try to open the syslog file on your +system: + +.. prompt:: bash + + lnav + + +Setup +----- + +After starting **lnav**, you might want to set the +:ref:`configuration options<Configuration>` mentioned below. Configuration in +**lnav** is done using the :code:`:config` command. To change a configuration +option, start by pressing :kbd:`:` to enter the command prompt. Then, +type "config" followed by the option name and value. + +.. note:: + + Tab-completion is available for these configuration options and, in some + cases, their values as well. + + +Keymap +^^^^^^ + +The keymap defines the mapping from :ref:`hotkeys<hotkeys>` to commands to +execute. The default mapping is for "U.S." keyboards. The following command +can be used to change the keymap: + +.. code-block:: lnav + + :config /ui/keymap <keymap-name> + +The builtin keymaps are: + + :de: `German <https://github.com/tstack/lnav/blob/master/src/keymaps/de-keymap.json>`_ + :fr: `French <https://github.com/tstack/lnav/blob/master/src/keymaps/fr-keymap.json>`_ + :sv: `Swedish <https://github.com/tstack/lnav/blob/master/src/keymaps/sv-keymap.json>`_ + :uk: `United Kingdom <https://github.com/tstack/lnav/blob/master/src/keymaps/uk-keymap.json>`_ + :us: `United States <https://github.com/tstack/lnav/blob/master/src/keymaps/us-keymap.json>`_ + +To create or customize a keymap, consult the :ref:`keymaps` section. + + +Theme +^^^^^ + +The visual styling of **lnav** can be customized using a theme. The following +command can be used to the change the theme: + +.. code-block:: lnav + + :config /ui/theme <theme-name> + +The builtin themes are: +`default <https://github.com/tstack/lnav/blob/master/src/themes/default-theme.json>`_, +`eldar <https://github.com/tstack/lnav/blob/master/src/themes/eldar.json>`_, +`grayscale <https://github.com/tstack/lnav/blob/master/src/themes/grayscale.json>`_, +`monocai <https://github.com/tstack/lnav/blob/master/src/themes/monocai.json>`_, +`night-owl <https://github.com/tstack/lnav/blob/master/src/themes/night-owl.json>`_, +`solarized-dark <https://github.com/tstack/lnav/blob/master/src/themes/solarized-dark.json>`_, +and +`solarized-light <https://github.com/tstack/lnav/blob/master/src/themes/default-theme.json>`_. + +To create or customize a theme, consult the :ref:`themes` section. + + +Log Formats +^^^^^^^^^^^ + +In order for **lnav** to understand your log files, it needs to told how to +parse the log messages using a log format definition. There are many log +formats builtin and **lnav** will automatically determine the best format to +use. In case your log file is not recognized, consult the :ref:`log_formats` +section for information on how to create a format. diff --git a/docs/source/key-encoding-prompt.png b/docs/source/key-encoding-prompt.png Binary files differnew file mode 100644 index 0000000..fe63412 --- /dev/null +++ b/docs/source/key-encoding-prompt.png diff --git a/docs/source/lnav-breadcrumbs-help.png b/docs/source/lnav-breadcrumbs-help.png Binary files differnew file mode 100644 index 0000000..b3b80d0 --- /dev/null +++ b/docs/source/lnav-breadcrumbs-help.png diff --git a/docs/source/lnav-config-header.png b/docs/source/lnav-config-header.png Binary files differnew file mode 100644 index 0000000..079b9c2 --- /dev/null +++ b/docs/source/lnav-config-header.png diff --git a/docs/source/lnav-files-panel.png b/docs/source/lnav-files-panel.png Binary files differnew file mode 100644 index 0000000..7aed793 --- /dev/null +++ b/docs/source/lnav-files-panel.png diff --git a/docs/source/lnav-filters-panel.png b/docs/source/lnav-filters-panel.png Binary files differnew file mode 100644 index 0000000..788e63f --- /dev/null +++ b/docs/source/lnav-filters-panel.png diff --git a/docs/source/lnav-spectro-cpu-pct.png b/docs/source/lnav-spectro-cpu-pct.png Binary files differnew file mode 100644 index 0000000..a9df2b6 --- /dev/null +++ b/docs/source/lnav-spectro-cpu-pct.png diff --git a/docs/source/lnav-ui.png b/docs/source/lnav-ui.png Binary files differnew file mode 100644 index 0000000..2dcad66 --- /dev/null +++ b/docs/source/lnav-ui.png diff --git a/docs/source/open-error.png b/docs/source/open-error.png Binary files differnew file mode 100644 index 0000000..6454b2d --- /dev/null +++ b/docs/source/open-error.png diff --git a/docs/source/open-help.png b/docs/source/open-help.png Binary files differnew file mode 100644 index 0000000..a11656a --- /dev/null +++ b/docs/source/open-help.png diff --git a/docs/source/open-preview.png b/docs/source/open-preview.png Binary files differnew file mode 100644 index 0000000..b888376 --- /dev/null +++ b/docs/source/open-preview.png diff --git a/docs/source/query-results.png b/docs/source/query-results.png Binary files differnew file mode 100644 index 0000000..14ae5b8 --- /dev/null +++ b/docs/source/query-results.png diff --git a/docs/source/sessions.rst b/docs/source/sessions.rst new file mode 100644 index 0000000..e97e598 --- /dev/null +++ b/docs/source/sessions.rst @@ -0,0 +1,23 @@ + +.. _sessions: + +Sessions +======== + +Session information is stored automatically for the set of files that were +passed in on the command-line and reloaded the next time **lnav** is executed. +The information currently stored is: + +* Position within the files being viewed. +* Active searches for each view. +* :ref:`Log filters<filtering>`. +* :ref:`Highlights<highlight>`. +* :ref:`Hidden files<hide_file>`. +* :ref:`Hidden fields<hide_fields>`. + +Bookmarks and log-time adjustments are stored separately on a per-file basis. +Note that the bookmarks are associated with files based on the content of the +first line of the file so that they are preserved even if the file has been +moved from its current location. + +Session data is stored in the :file:`~/.lnav` directory. diff --git a/docs/source/sql-help.png b/docs/source/sql-help.png Binary files differnew file mode 100644 index 0000000..44901a4 --- /dev/null +++ b/docs/source/sql-help.png diff --git a/docs/source/sqlext.rst b/docs/source/sqlext.rst new file mode 100644 index 0000000..e0d7543 --- /dev/null +++ b/docs/source/sqlext.rst @@ -0,0 +1,184 @@ + +.. _sql-ext: + +SQLite Interface +================ + +Log analysis in **lnav** can be done using the SQLite interface. Log messages +can be accessed via `virtual tables <https://www.sqlite.org/vtab.html>`_ that +are created for each file format. The tables have the same name as the log +format and each message is its own row in the table. For example, given the +following log message from an Apache access log:: + + 127.0.0.1 - frank [10/Oct/2000:13:55:36 -0700] "GET /apache_pb.gif HTTP/1.0" 200 2326 + +These columns would be available for its row in the :code:`access_log` table: + +.. csv-table:: + :class: query-results + :header-rows: 1 + + log_line,log_part,log_time,log_idle_msecs,log_level,log_mark,log_comment,log_tags,log_filters,c_ip,cs_method,cs_referer,cs_uri_query,cs_uri_stem,cs_user_agent,cs_username,cs_version,sc_bytes,sc_status + 0,<NULL>,2000-10-10 13:55:36.000,0,info,1,<NULL>,<NULL>,<NULL>,127.0.0.1,GET,<NULL>,<NULL>,/apache_pb.gif,<NULL>,frank,HTTP/1.0,2326,200 + +.. note:: Some columns are hidden by default to reduce the amount of noise in + results, but they can still be accessed when explicitly used. The hidden + columns are: :code:`log_path`, :code:`log_text`, :code:`log_body`, and + :code:`log_raw_text`. + +You can activate the SQL prompt by pressing the :kbd:`;` key. At the +prompt, you can start typing in the desired SQL statement and/or double-tap +:kbd:`TAB` to activate auto-completion. A help window will appear above +the prompt to guide you in the usage of SQL keywords and functions. + +.. figure:: sql-help.png + :align: center + + Screenshot of the online help for the SQL prompt. + +.. figure:: group_concat-help.png + :align: center + + Screenshot of the online help for the :code:`group_concat()` function. + +A simple query to perform on an Apache access log might be to get the average +and maximum number of bytes returned by the server, grouped by IP address: + +.. code-block:: custsqlite + + ;SELECT c_ip, avg(sc_bytes), max(sc_bytes) FROM access_log GROUP BY c_ip + +After pressing :kbd:`Enter`, SQLite will execute the query using **lnav**'s +virtual table implementation to extract the data directly from the log files. +Once the query has finished, the main window will switch to the DB view to +show the results. Press :kbd:`q` to return to the log view and press :kbd:`v` +to return to the log view. If the SQL results contain a +:code:`log_line` column, you can press to :kbd:`Shift` + :kbd:`V` to +switch between the DB view and the log + +.. figure:: query-results.png + :align: center + + Screenshot of the SQL results view. + +The DB view has the following display features: + +* Column headers stick to the top of the view when scrolling. +* A stacked bar chart of the numeric column values is displayed underneath the + rows. Pressing :kbd:`TAB` will cycle through displaying no columns, each + individual column, or all columns. +* JSON columns in the top row can be pretty-printed by pressing :kbd:`p`. + The display will show the value and JSON-Pointer path that can be passed to + the `jget`_ function. + + +Log Tables +---------- + +Each log format has its own database table that can be used to access log +messages that match that format. The table name is the same as the format +name, for example, the :code:`syslog_log` format will have a table that is +also named :code:`syslog_log`. There is also an :code:`all_logs` table +that provides access to all messages from all formats. + +.. note:: Only the displayed log messages are reflected in the SQLite + interface. Any log messages that have been filtered out are not + accessible. + +The columns in the log tables are made up of several builtins along with +the values captured by the log format specification. Use the :code:`.schema` +command in the SQL prompt to examine a dump of the current database schema. + +The following columns are builtin and included in a :code:`SELECT *`: + + :log_line: The line number for the message in the log view. + :log_part: The partition the message is in. This column can be changed by + an :code:`UPDATE` or the :ref:`:parition-name<partition_name>` command. + :log_time: The adjusted timestamp for the log message. This time can differ + from the log message's time stamp if it arrived out-of-order and the log + format expects log files to be time-ordered. + :log_actual_time: The log messages original timestamp in the file. + :log_idle_msecs: The difference in time between this messages and the + previous. The unit of time is milliseconds. + :log_level: The log message level. + :log_mark: True if the log message was marked by the user. + :log_comment: The comment for the message. This column can be changed by + an :code:`UPDATE` or the :ref:`:comment<comment>` command. + :log_tags: A JSON list of tags for the message. This column can be changed + by an :code:`UPDATE` or the :ref:`:tag<tag>` command. + :log_filters: A JSON list of filter IDs that matched this message + +The following columns are builtin and are hidden, so they will *not* be +included in a :code:`SELECT *`: + + :log_time_msecs: The adjusted timestamp for the log message as the number of + milliseconds from the epoch. This column can be more efficient to use for + time-related operations, like :ref:`timeslice()<timeslice>`. + :log_path: The path to the log file this message is from. + :log_text: The full text of the log message. + :log_body: The body of the log message. + :log_raw_text: The raw text of this message from the log file. In this case + of JSON and CSV logs, this will be the exact line of JSON-Line and CSV + text from the file. + +Extensions +---------- + +To make it easier to analyze log data from within **lnav**, there are several +built-in extensions that provide extra functions and collators beyond those +`provided by SQLite <http://www.sqlite.org/lang_corefunc.html>`_. The majority +of the functions are from the +`extensions-functions.c <http://www.sqlite.org/contrib>`_ file available from +the `sqlite.org <http://sqlite.org>`_ web site. + +.. tip:: You can include a SQLite database file on the command-line and use + **lnav**'s interface to perform queries. The database will be attached with + a name based on the database file name. + +Commands +-------- + +A SQL command is an internal macro implemented by lnav. + +* .schema - Open the schema view. This view contains a dump of the schema + for the internal tables and any tables in attached databases. +* .msgformats - Executes a canned query that groups and counts log messages by + the format of their message bodies. This command can be useful for quickly + finding out the types of messages that are most common in a log file. + +Variables +--------- + +The following variables are available in SQL statements: + +* $LINES - The number of lines in the terminal window. +* $COLS - The number of columns in the terminal window. + +Environment +----------- + +Environment variables can be accessed in queries using the usual syntax of +:code:`$VAR_NAME`. For example, to read the value of the "USER" variable, you +can write: + +.. code-block:: custsqlite + + ;SELECT $USER + +.. _collators: + +Collators +--------- + +* **naturalcase** - Compare strings "naturally" so that number values in the + string are compared based on their numeric value and not their character + values. For example, "foo10" would be considered greater than "foo2". +* **naturalnocase** - The same as naturalcase, but case-insensitive. +* **ipaddress** - Compare IPv4/IPv6 addresses. + +Reference +--------- + +The following is a reference of the SQL syntax and functions that are available: + +.. include:: ../../src/internals/sql-ref.rst diff --git a/docs/source/sqltab.rst b/docs/source/sqltab.rst new file mode 100644 index 0000000..ca2fefb --- /dev/null +++ b/docs/source/sqltab.rst @@ -0,0 +1,217 @@ +.. _sql-tab: + +SQLite Tables Reference +======================= + +In addition to the tables generated for each log format, **lnav** includes +the following tables/views: + +* `environ`_ +* `lnav_events`_ +* `lnav_file`_ +* `lnav_user_notifications`_ +* `lnav_views`_ +* `lnav_views_echo`_ +* `lnav_view_stack`_ +* `lnav_view_filters`_ +* `lnav_view_filter_stats`_ +* `lnav_view_filters_and_stats`_ +* `all_logs`_ +* `http_status_codes`_ +* `regexp_capture(<string>, <regex>)`_ + +These extra tables provide useful information and can let you manipulate +**lnav**'s internal state. You can get a dump of the entire database schema +by executing the '.schema' SQL command, like so:: + + ;.schema + +environ +------- + +The **environ** table gives you access to the **lnav** process' environment +variables. You can SELECT, INSERT, and UPDATE environment variables, like +so: + +.. code-block:: custsqlite + + ;SELECT * FROM environ WHERE name = 'SHELL' + name value + SHELL /bin/tcsh + + ;UPDATE environ SET value = '/bin/sh' WHERE name = 'SHELL' + +Environment variables can be used to store simple values or pass values +from **lnav**'s SQL environment to **lnav**'s commands. For example, the +"open" command will do variable substitution, so you can insert a variable +named "FILENAME" and then open it in **lnav** by referencing it with +"$FILENAME": + +.. code-block:: custsqlite + + ;INSERT INTO environ VALUES ('FILENAME', '/path/to/file') + :open $FILENAME + + +.. _table_lnav_events: + +lnav_events +----------- + +The **lnav_events** table allows you to react to events that occur while +**lnav** is running using SQLite triggers. For example, when a file is +opened, a row is inserted into the :code:`lnav_events` table that contains +a timestamp and a JSON object with the event ID and the path of the file. +The following columns are available in this table: + + :ts: The timestamp of the event. + :content: A JSON object that contains the event information. See the + :ref:`event_reference` for more information about the types + of events that are available. + +lnav_file +--------- + +The **lnav_file** table allows you to examine and perform limited updates to +the metadata for the files that are currently loaded into **lnav**. The +following columns are available in this table: + + :device: The device the file is stored on. + :inode: The inode for the file on the device. + :filepath: If this is a real file, it will be the absolute path. Otherwise, + it is a symbolic name. If it is a symbolic name, it can be UPDATEd so that + this file will be considered when saving and loading session information. + :format: The log file format for the file. + :lines: The number of lines in the file. + :time_offset: The millisecond offset for timestamps. This column can be + UPDATEd to change the offset of timestamps in the file. + +.. _table_lnav_user_notifications: + +lnav_user_notifications +----------------------- + +The :code:`lnav_user_notifications` table allows you to display a custom message +in the top-right corner of the UI. For example, to display "Hello, World!", +you can enter: + +.. code-block:: custsqlite + + ;REPLACE INTO lnav_user_notifications (message) VALUES ('Hello, World!') + +There are additional columns to have finer control of what is displayed and +when: + + :id: The unique ID for the message, defaults to "org.lnav.user". This is + the primary key for the table, so more than one type of message is not + allowed. + :priority: The priority of the message. Higher priority messages will be + displayed until they are cleared or are expired. + :created: The time the message was created. + :expiration: The time when the message should expire or NULL if it should + not automatically expire. + :views: A JSON array of view names where the message is applicable or NULL + if the message should be shown in all views. + :message: The message itself. + +This table will most likely be used in combination with :ref:`Events` and the +`lnav_views_echo`_ table. + +lnav_views +---------- + +The **lnav_views** table allows you to SELECT and UPDATE information related +to **lnav**'s "views" (e.g. log, text, ...). The following columns are +available in this table: + + :name: The name of the view. + :top: The line number at the top of the view. This value can be UPDATEd to + move the view to the given line. + :left: The left-most column number to display. This value can be UPDATEd to + move the view left or right. + :height: The number of lines that are displayed on the screen. + :inner_height: The number of lines of content being displayed. + :top_time: The timestamp of the top line in the view or NULL if the view is + not time-based. This value can be UPDATEd to move the view to the given + time. + :paused: Indicates if the view is paused and will not load new data. + :search: The search string for this view. This value can be UPDATEd to + initiate a text search in this view. + +lnav_views_echo +--------------- + +The :code:`lnav_views_echo` table is a real SQLite table that you can create +TRIGGERs on in order to react to users moving around in a view. + +.. note:: + + The table is periodically updated to reflect the current state of the views. + The changes are *not* performed immediately after the user action. + +lnav_view_stack +--------------- + +The **lnav_view_stack** table allows you to SELECT and DELETE from the stack of +**lnav** "views" (e.g. log, text, ...). The following columns are available in +this table: + + :name: The name of the view. + +.. _table_lnav_view_filters: + +lnav_view_filters +----------------- + +The **lnav_view_filters** table allows you to manipulate the filters in the +**lnav** views. The following columns are available in this table: + + :view_name: The name of the view the filter is applied to. + :filter_id: The filter identifier. This will be assigned on insertion. + :enabled: Indicates whether this filter is enabled or disabled. + :type: The type of filter, either 'in' or 'out'. + :pattern: The regular expression to filter on. + +This table supports SELECT, INSERT, UPDATE, and DELETE on the table rows to +read, create, update, and delete filters for the views. + +lnav_view_filter_stats +---------------------- + +The **lnav_view_filter_stats** table allows you to get information about how +many lines matched a given filter. The following columns are available in +this table: + + :view_name: The name of the view. + :filter_id: The filter identifier. + :hits: The number of lines that matched this filter. + +This table is read-only. + +lnav_view_filters_and_stats +--------------------------- + +The **lnav_view_filters_and_stats** view joins the **lnav_view_filters** table +with the **lnav_view_filter_stats** table into a single view for ease of use. + +all_logs +-------- + +.. f0:sql.tables.all_logs + +The **all_logs** table lets you query the format derived from the **lnav** +log message parser that is used to automatically extract data, see +:ref:`data-ext` for more details. + +http_status_codes +----------------- + +The **http_status_codes** table is a handy reference that can be used to turn +HTTP status codes into human-readable messages. + +regexp_capture(<string>, <regex>) +--------------------------------- + +The **regexp_capture()** table-valued function applies the regular expression +to the given string and returns detailed results for the captured portions of +the string. diff --git a/docs/source/ui.rst b/docs/source/ui.rst new file mode 100644 index 0000000..c7d7ffe --- /dev/null +++ b/docs/source/ui.rst @@ -0,0 +1,280 @@ +.. _ui: + +User Interface +============== + +The **lnav** TUI displays the content of the current "view" in the middle, +with status bars above and below, and the interactive prompt as the last line. + +.. figure:: lnav-ui.png + :align: center + :alt: Screenshot of lnav showing a mix of syslog and web access_log messages. + + Screenshot of **lnav** viewing syslog and web access_log messages. + +The default view shows the log messages from the log files that have been +loaded. There are other views for displaying content like plaintext files +and SQL results. The :ref:`ui_views` section describes the characteristics of +each view in more detail. You can switch to the different views using the +hotkeys described in the :ref:`hotkeys_display` section or by pressing +:kbd:`ENTER` to activate the breadcrumb bar, moving to the first crumb, and +then selecting the desired view. You can switch back to the previous view by +pressing :kbd:`q`. You can switch forward to the new view by pressing +:kbd:`a`. If the views are time-based (e.g. log and histogram), pressing +:kbd:`Shift` + :kbd:`q` and :kbd:`Shift` + :kbd:`a` will synchronize the top +times in the views. + +The right side of the display has a proportionally sized 'scrollbar' that +shows: + +* the current position in the file; +* the locations of errors/warnings in the log files by using red or yellow + coloring; +* the locations of search hits by using a tick-mark pointing to the left; +* the locations of bookmarks by using a tick-mark pointing to the right. + +Top Status Bar +-------------- + +The top status bar shows the current time and messages stored in the +:ref:`table_lnav_user_notifications` table. + +Below the top status bar is the breadcrumb bar that displays the semantic +location of the top line in the main view. For example, within a +pretty-printed JSON document, it will show the path to property at the top +of the view. The actual content of the bar depends on the current view and +will be updated as you navigate around the main view. The bar can also be +used to navigate around the document by focusing on it. + +Breadcrumb Bar +-------------- + +.. figure:: lnav-breadcrumbs-help.png + :align: center + :figwidth: 90% + + Screenshot of the breadcrumb bar focused and navigating the help text + +To focus on the breadcrumb bar, press :kbd:`ENTER`. The :kbd:`←`/:kbd:`→` +cursor keys can be used to select a crumb and the :kbd:`↑`/:kbd:`↓` keys can +be used select a value of that crumb. To accept a value and drop focus on the +bar, press :kbd:`ENTER`. To accept a value and move to the next crumb, press +:kbd:`→`. Using :kbd:`→` makes it quicker to drill down into a document +without having to constantly switch focus. To drop focus on the bar without +accepting anything, press :kbd:`Escape`. + +There are three types of crumbs: + +* a dropdown where one of a limited set of values can be selected; +* a combobox where a value can be entered directly or selected; +* a numeric input for entering array indexes. + +When a dropdown or combobox is selected, you can type part of the desired value +to filter the list of values. For example, the first crumb is always the +current view, typing in "hi" will filter the list down to the "HIST" value. + +Configuration Panels +-------------------- + +.. figure:: lnav-config-header.png + :align: center + :figwidth: 90% + + Screenshot of the header for the configuration panels when they are hidden. + +After the main view content, there is a header bar for two configuration +panels: Files and Filters. These panels provide visual access to parts of +lnav's configuration. To access the panels, press the :kbd:`TAB` key. +To hide the panels again, press :kbd:`q`. + +.. figure:: lnav-files-panel.png + :align: center + :figwidth: 90% + + Screenshot of the files panel showing the loaded files. + +The Files panel is open initially to display progress in loading files. +The following information can be displayed for each file: + +* the "unique" portion of the path relative to the other files; +* the amount of data that has been indexed; +* the date range of log messages contained in the file; +* the errors that were encountered while trying to index the file; +* the notes recorded for files where some automatic action was taken, + like hiding the file if it was seen as a duplicate of another file. + +.. figure:: lnav-filters-panel.png + :align: center + :figwidth: 90% + + Screenshot of the filters panel showing an OUT and a disabled IN filter. + +If the view supports filtering, there will be a status line showing the +following: + +* the number of enabled filters and the total number of filters; +* the number of lines that are **not** displayed because of filtering. + +To edit the filters, you can press TAB to change the focus from the main +view to the filter editor. The editor allows you to create, enable/disable, +and delete filters easily. + +Bottom Status Bar +----------------- + +The second to last line is the bottom status bar, which shows the following: + +* the line number of the top line, starting from zero; +* the location within the view, as a percentage; +* the current search hit, the total number of hits, and the search term; +* the loading indicator. + +When the interactive prompt is active, this bar can show the prompt +description, help text, or error message. + +Prompt +------ + +Finally, the last line on the display is where you can enter search +patterns and execute internal commands, such as converting a +unix-timestamp into a human-readable date. The following key-presses +will activate a corresponding prompt: + +* :kbd:`/` - The search prompt. You can enter a PCRE2-flavored regular + expression to search for in the current view. +* :kbd:`:` - The command prompt. Commands are used to perform common + operations. +* :kbd:`;` - The SQL prompt. SQL queries can be used for log analysis + and manipulating **lnav**'s state. +* :kbd:`|` - The script prompt. Enter a path to the lnav script to + execute, along with the arguments to pass in. + +The command-line is by the readline library, so the usual set of keyboard +shortcuts can be used for editing and moving within the command-line. + +.. _ui_views: + +Views +----- + +The accessible content within lnav is separated into the following views. + +LOG +^^^ + +The log view displays the log messages from any loaded log files in time +order. This view will be shown by default if any log messages are available. + +On color displays, the log messages will be highlighted as follows: + +* Errors will be colored in red; +* warnings will be yellow; +* search hits are reverse video; +* various color highlights will be applied to: IP addresses, SQL keywords, + XML tags, file and line numbers in Java backtraces, and quoted strings; +* "identifiers" in the messages will be randomly assigned colors based on their + content (works best on "xterm-256color" terminals). + +.. note:: + + If the coloring is too much for your tastes, you can change to the + "grayscale" theme by entering the following command: + + .. code-block:: lnav + + :config /ui/theme grayscale + +.. note:: + + If a log message has a timestamp that is out-of-order with its neighboring + messages, the timestamp will be highlighted in yellow. When one of these + messages is at the top of the log view, an overlay will display the + difference between the "actual time" and the "received time". The "actual + time" is the original textual timestamp. The "received time" is the time + of an earlier message that is larger than this log message's time. + +The breadcrumb bar will show the following crumbs: + +* the timestamp for the top line; +* the log format for the top line; +* the name of the file the top line was pulled from; +* the "operation ID" of the top log message, if it is supported by the log + format. + +These crumbs are interactive and can be used to navigate to different parts +of the log view. For example, selecting a different value in the log format +crumb will jump to the first message with that format. + +TEXT +^^^^ + +The text view displays files for which lnav could not detect any log messages. + +Markdown +"""""""" + +Files with an :code:`.md` (or :code:`.markdown`) extension will be treated as +Markdown files and rendered separately. + +DB +^^ + +The DB view shows the results of queries done through the SQLite interface. +You can execute a query by pressing :kbd:`;` and then entering a SQL statement. +You can switch to the SQL view by pressing :kbd:`v`. + +HELP +^^^^ + +The help view displays the builtin help text. Press :kbd:`?` to switch to the +help view at any time. While in the help view, the breadcrumb bar can be used +to navigate to different sections of the document. + +HIST +^^^^ + +The histogram view displays a stacked bar chart of messages over time +classified by their log level and whether they've been bookmarked. Press +:kbd:`i` to switch back and forth to the histogram view. You can also press +:kbd:`Shift`+:kbd:`i` to toggle the histogram view while synchronizing the top +time. While in the histogram view, pressing :kbd:`z`/:kbd:`Shift`+:kbd:`z` +will zoom in/out. + +PRETTY +^^^^^^ + +The pretty-print view takes the text displayed in the current view and shows +the result of a pretty-printer run on that text. For example, if a log +message contained an XML message on a single line, the pretty-printer would +break the XML across multiple lines with appropriate indentation. + +SCHEMA +^^^^^^ + +The schema view displays the current schema of the builtin SQLite database. + +SPECTRO +^^^^^^^ + +The spectrogram view is a "three"-dimensional display of data points of a log +field or a SQL query column. The dimensions are time on the Y axis, the range +of data point values on the X axis, and the number of data points as a color. +For example, if you were to visualize process CPU usage over time, the range +of values on the X axis would be CPU percentages and there would be colored +blocks at each point on the line where a process had that CPU percentage, like +so + +.. figure:: lnav-spectro-cpu-pct.png + :align: center + + Screenshot of the **lnav** spectrogram view showing CPU usage of processes. + +The colors correspond to the relative number of data points in a bucket. +The legend overlaid at the top line in the view shows the counts of data +points that are in a particular color, with green having the fewest number of +data points, yellow the middle, and red the most. You can select a particular +bucket using the cursor keys to see the exact number of data points and the +range of values. The panel at the bottom of the view shows the data points +themselves from the original source, the log file or the SQL query results. +You can press :kbd:`TAB` to focus on the details panel so you can scroll +around and get a closer look at the values. diff --git a/docs/source/usage.rst b/docs/source/usage.rst new file mode 100644 index 0000000..d43ed52 --- /dev/null +++ b/docs/source/usage.rst @@ -0,0 +1,291 @@ +.. _usage: + +Usage +===== + +This chapter contains an overview of how to use **lnav**. + + +Basic Controls +-------------- + +Like most file viewers, scrolling through files can be done with the usual +:ref:`hotkeys<hotkeys>`. For non-trivial operations, you can enter the +:ref:`command<commands>` prompt by pressing :kbd:`:`. To analyze data in a +log file, you can enter the :ref:`SQL prompt<sql-ext>` by pressing :kbd:`;`. + +.. tip:: + + Check the bottom right corner of the screen for tips on hotkeys that might + be useful in the current context. + + .. figure:: hotkey-tips.png + :align: center + + When **lnav** is first open, it suggests using :kbd:`e` and + :kbd:`Shift` + :kbd:`e` to jump to error messages. + + +Viewing Files +------------- + +The files to view in **lnav** can be given on the command-line or passed to the +:ref:`:open<open>` command. A +`glob pattern <https://en.wikipedia.org/wiki/Glob_(programming)>`_ can be given +to watch for files with a common name. If the path is a directory, all of the +files in the directory will be opened and the directory will be monitored for +files to be added or removed from the view. If the path is an archive or +compressed file (and lnav was built with libarchive), the archive will be +extracted to a temporary location and the files within will be loaded. The +files that are found will be scanned to identify their file format. Files +that match a log format will be collated by time and displayed in the LOG +view. Plain text files can be viewed in the TEXT view, which can be accessed +by pressing :kbd:`t`. + + +Archive Support +^^^^^^^^^^^^^^^ + +.. f0:archive + +If **lnav** is compiled with `libarchive <https://www.libarchive.org>`_, +any files to be opened will be examined to see if they are a supported archive +type. If so, the contents of the archive will be extracted to the +:code:`$TMPDIR/lnav-user-${UID}-work/archives/` directory. Once extracted, the +files within will be loaded into lnav. To speed up opening large amounts of +files, any file that meets the following conditions will be automatically +hidden and not indexed: + +* Binary files +* Plain text files that are larger than 128KB +* Duplicate log files + +The unpacked files will be left in the temporary directory after exiting +**lnav** so that opening the same archive again will be faster. Unpacked +archives that have not been accessed in the past two days will be automatically +deleted the next time **lnav** is started. + + +.. _remote: + +Remote Files +^^^^^^^^^^^^ + +Files on remote machines can be viewed and tailed if you have access to the +machines via SSH. First, make sure you can SSH into the remote machine without +any interaction by: 1) accepting the host key as known and 2) copying your +identity's public key to the :file:`.ssh/authorized_keys` file on the remote +machine. Once the setup is complete, you can open a file on a remote host +using the same syntax as :command:`scp(1)` where the username and host are +given, followed by a colon, and then the path to the files, like so:: + + [user@]host:/path/to/logs + +For example, to open :file:`/var/log/syslog.log` on "host1.example.com" as the +user "dean", you would write: + +.. prompt:: bash + + lnav dean@host1.example.com:/var/log/syslog.log + +Remote files can also be opened using the :ref:`:open<open>` command. Opening +a remote file in the TUI has the advantage that the file path can be +:kbd:`TAB`-completed and a preview is shown of the first few lines of the +file. + +.. note:: + + If lnav is installed from the `snap <https://snapcraft.io/lnav>`_, you will + need to connect it to the + `ssh-keys plug <https://snapcraft.io/docs/ssh-keys-interface>`_ using the + following command: + + .. prompt:: bash + + sudo snap connect lnav:ssh-keys + +.. note:: + + Remote file access is implemented by transferring an + `αcτµαlly pδrταblε εxεcµταblε <https://justine.lol/ape.html>`_ to the + destination and invoking it. An APE binary can run on most any x86_64 + machine and OS (i.e. MacOS, Linux, FreeBSD, Windows). The binary is + baked into the lnav executable itself, so there is no extra setup that + needs to be done on the remote machine. + + The binary file is named ``tailer.bin.XXXXXX`` where *XXXXXX* is 6 random digits. + The file is, under normal circumstancies, deleted immediately. + +Searching +--------- + +Any log messages that are loaded into **lnav** are indexed by time and log +level (e.g. error, warning) to make searching quick and easy with +:ref:`hotkeys<hotkeys>`. For example, pressing :kbd:`e` will jump to the +next error in the file and pressing :kbd:`Shift` + :kbd:`e` will jump to +the previous error. Plain text searches can be done by pressing :kbd:`/` +to enter the search prompt. A regular expression can be entered into the +prompt to start a search through the current view. + + +.. _filtering: + +Filtering +--------- + +To reduce the amount of noise in a log file, **lnav** can hide log messages +that match certain criteria. The following sub-sections explain ways to go +about that. + + +Regular Expression Match +^^^^^^^^^^^^^^^^^^^^^^^^ + +If there are log messages that you are not interested in, you can do a +"filter out" to hide messages that match a pattern. A filter can be created +using the interactive editor, the :ref:`:filter-out<filter_out>` command, or +by doing an :code:`INSERT` into the +:ref:`lnav_view_filters<table_lnav_view_filters>` table. + +If there are log messages that you are only interested in, you can do a +"filter in" to only show messages that match a pattern. The filter can be +created using the interactive editor, the :ref:`:filter-in<filter_in>` command, +or by doing an :code:`INSERT` into the +:ref:`lnav_view_filters<table_lnav_view_filters>` table. + + +SQLite Expression +^^^^^^^^^^^^^^^^^ + +Complex filtering can be done by passing a SQLite expression to the +:ref:`:filter-expr<filter_expr>` command. The expression will be executed for +every log message and if it returns true, the line will be shown in the log +view. + + +Time +^^^^ + +To limit log messages to a given time frame, the +:ref:`:hide-lines-before<hide_lines_before>` and +:ref:`:hide-lines-after<hide_lines_after>` commands can be used to specify +the beginning and end of the time frame. + + +Log level +^^^^^^^^^ + +To hide messages below a certain log level, you can use the +:ref:`:set-min-log-level<set_min_log_level>` command. + +.. _search_tables: + +Search Tables +------------- + +Search tables allow you to access arbitrary data in log messages through +SQLite virtual tables. If there is some data in a log message that you can +match with a regular expression, you can create a search-table that matches +that data and any capture groups will be plumbed through as columns in the +search table. + +Creating a search table can be done interactively using the +:ref:`:create-search-table<create_search_table>` command or by adding it to +a :ref:`log format definition<log_formats>`. The main difference between +the two is that tables defined as part of a format will only search messages +from log files with that format and the tables will include log message +columns defined in that format. Whereas a table created with the command +will search messages from all different formats and no format-specific +columns will be included in the table. + +.. _taking_notes: + +Taking Notes +------------ + +A few of the columns in the log tables can be updated on a row-by-row basis to +allow you to take notes. The majority of the columns in a log table are +read-only since they are backed by the log files themselves. However, the +following columns can be changed by an :code:`UPDATE` statement: + +* **log_part** - The "partition" the log message belongs to. This column can + also be changed by the :ref:`:partition-name<partition_name>` command. +* **log_mark** - Indicates whether the line has been bookmarked. +* **log_comment** - A free-form text field for storing commentary. This + column can also be changed by the :ref:`:comment<comment>` command. +* **log_tags** - A JSON list of tags associated with the log message. This + column can also be changed by the :ref:`:tag<tag>` command. + +While these columns can be updated by through other means, using the SQL +interface allows you to make changes automatically and en masse. For example, +to bookmark all lines that have the text "something interesting" in the log +message body, you can execute: + +.. code-block:: custsqlite + + ;UPDATE all_logs SET log_mark = 1 WHERE log_body LIKE '%something interesting%' + +As a more advanced example of the power afforded by SQL and **lnav**'s virtual +tables, we will tag log messages where the IP address bound by dhclient has +changed. For example, if dhclient reports "bound to 10.0.0.1" initially and +then reports "bound to 10.0.0.2", we want to tag only the messages where the +IP address was different from the previous message. While this can be done +with a single SQL statement [#]_, we will break things down into a few steps for +this example. First, we will use the :ref:`:create-search-table<create_search_table>` +command to match the dhclient message and extract the IP address: + +.. code-block:: lnav + + :create-search-table dhclient_ip bound to (?<ip>[^ ]+) + +The above command will create a new table named :code:`dhclient_ip` with the +standard log columns and an :code:`ip` column that contains the IP address. +Next, we will create a view over the :code:`dhclient_ip` table that returns +the log message line number, the IP address from the current row and the IP +address from the previous row: + +.. code-block:: custsqlite + + ;CREATE VIEW IF NOT EXISTS dhclient_ip_changes AS SELECT log_line, ip, lag(ip) OVER (ORDER BY log_line) AS prev_ip FROM dhclient_ip + +Finally, the following :code:`UPDATE` statement will concatenate the tag +"#ipchanged" onto the :code:`log_tags` column for any rows in the view where +the current IP is different from the previous IP: + +.. code-block:: custsqlite + + ;UPDATE syslog_log SET log_tags = json_concat(log_tags, '#ipchanged') WHERE log_line IN (SELECT log_line FROM dhclient_ip_changes WHERE ip != prev_ip) + +Since the above can be a lot to type out interactively, you can put these +commands into a :ref:`script<scripts>` and execute that script with the +:kbd:`\|` hotkey. + +.. [#] The expression :code:`regexp_match('bound to ([^ ]+)', log_body) as ip` + can be used to extract the IP address from the log message body. + +Sharing Sessions With Others +---------------------------- + +After setting up filters, bookmarks, and making notes, you might want to share +your work with others. If they have access to the same log files, you can +use the :ref:`:export-session-to<export_session_to>` command to write an +executable **lnav** script that will recreate the current session state. The +script contains various SQL statements and **lnav** commands that capture the +current state. So, you should feel free to modify the script or use it as a +reference to learn about more advanced uses of lnav. + +The script will capture the file paths that were explicitly specified and +not the files that were actually opened. For example, if you specified +"/var/log" on the command line, the script will include +:code:`:open /var/log/*` and not an individual open for each file in that +directory. + +Also, in order to support archives of log files, lnav will try to find the +directory where the archive was unpacked and use that as the base for the +:code:`:open` command. Currently, this is done by searching for the top +"README" file in the directory hierarchy containing the files [1]_. The +consumer of the session script can then set the :code:`LOG_DIR_0` (or 1, 2, +...) environment variable to change where the log files will be loaded from. + +.. [1] It is assumed a log archive would have a descriptive README file. + Other heuristics may be added in the future. |
