diff options
Diffstat (limited to 'src/formats')
48 files changed, 3793 insertions, 0 deletions
diff --git a/src/formats/README.md b/src/formats/README.md new file mode 100644 index 0000000..518dff8 --- /dev/null +++ b/src/formats/README.md @@ -0,0 +1,5 @@ +# Formats + +This directory contains the built-in log file format definitions. These files +are converted to C by `bin2c` and compiled into the executable. New formats +need to be added to the [formats.am](formats.am) file. diff --git a/src/formats/access_log.json b/src/formats/access_log.json new file mode 100644 index 0000000..6a5b020 --- /dev/null +++ b/src/formats/access_log.json @@ -0,0 +1,117 @@ +{ + "$schema": "https://lnav.org/schemas/format-v1.schema.json", + "access_log": { + "title": "Common Access Log", + "description": "The default web access log format for servers like Apache.", + "url": "http://en.wikipedia.org/wiki/Common_Log_Format", + "multiline": false, + "regex": { + "ts-first-noquotes": { + "pattern": "^(?<timestamp>\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}(?:\\.\\d{3})?) (?<c_ip>[^ ]+) (?<cs_username>[^ ]+) (?<cs_method>[A-Z]+) (?!\")(?<cs_uri_stem>[^ \\?]+)(?:\\?(?<cs_uri_query>[^ ]*))? (?:-1|\\d+) (?<sc_status>\\d+) \\d+\\s*(?<body>.*)" + }, + "ts-first": { + "pattern": "^(?<timestamp>\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}(?:\\.\\d{3})?) (?<c_ip>[^ ]+) (?<cs_username>[^ ]+) (?<cs_method>[A-Z]+) \"(?<cs_uri_stem>[^ \\?]+)(?:\\?(?<cs_uri_query>[^ ]*))?\" (?:-1|\\d+) (?<sc_status>\\d+) \\d+\\s*(?<body>.*)" + }, + "std": { + "pattern": "^(?<c_ip>[\\w\\.:\\-]+)\\s+[\\w\\.\\-]+\\s+(?<cs_username>\\S+)\\s+\\[(?<timestamp>[^\\]]+)\\] \"(?:\\-|(?<cs_method>\\w+) (?<cs_uri_stem>[^ \\?]+)(?:\\?(?<cs_uri_query>[^ ]*))? (?<cs_version>[\\w/\\.]+))\" (?<sc_status>\\d+) (?<sc_bytes>\\d+|-)(?: \"(?<cs_referer>[^\"]*)\" \"(?<cs_user_agent>[^\"]+)\")?\\s*(?<body>.*)" + }, + "std-vhost": { + "pattern": "^(?<cs_host>[\\w\\-\\.]*)(?::\\d+)?\\s+(?<c_ip>[\\w\\.:\\-]+)\\s+[\\w\\.\\-]+\\s+(?<cs_username>\\S+)\\s+\\[(?<timestamp>[^\\]]+)\\] \"(?:\\-|(?<cs_method>\\w+) (?<cs_uri_stem>[^ \\?]+)(?:\\?(?<cs_uri_query>[^ ]*))? (?<cs_version>[\\w/\\.]+))\" (?<sc_status>\\d+) (?<sc_bytes>\\d+|-)(?: \"(?<cs_referer>[^\"]+)\" \"(?<cs_user_agent>[^\"]+)\")?\\s*(?<body>.*)" + }, + "mod-std": { + "module-format": true, + "pattern": "^(?<c_ip>[\\w\\.:\\-]+)\\s+[\\w\\.\\-]+\\s+(?<cs_username>\\S+)\\s+\"(?:\\-|(?<cs_method>\\w+) (?<cs_uri_stem>[^ \\?]+)(?:\\?(?<cs_uri_query>[^ ]*))? (?<cs_version>[\\w/\\.]+))\" (?<sc_status>\\d+) (?<sc_bytes>\\d+|-)(?: \"(?<cs_referer>[^\"]+)\" \"(?<cs_user_agent>[^\"]+)\")?\\s*(?<body>.*)" + } + }, + "level-field": "sc_status", + "level": { + "error": "^[^123].*" + }, + "opid-field": "c_ip", + "value": { + "cs_host": { + "kind": "string", + "identifier": true, + "description": "The value of the Host header" + }, + "c_ip": { + "kind": "string", + "collate": "ipaddress", + "identifier": true, + "description": "The client IP address" + }, + "cs_username": { + "kind": "string", + "identifier": true, + "description": "The username passed from the client to the server" + }, + "cs_method": { + "kind": "string", + "identifier": true, + "description": "The request method" + }, + "cs_uri_stem": { + "kind": "string", + "identifier": true, + "description": "The path part of the request URI" + }, + "cs_uri_query": { + "kind": "string", + "description": "The query parameters in the request URI" + }, + "cs_version": { + "kind": "string", + "identifier": true, + "description": "The client's HTTP version" + }, + "sc_status": { + "kind": "integer", + "foreign-key": true, + "rewriter": ";SELECT :sc_status || ' (' || (SELECT message FROM http_status_codes WHERE status = :sc_status) || ') '", + "description": "The status code returned by the server" + }, + "sc_bytes": { + "kind": "integer", + "description": "The number of bytes returned by the server" + }, + "cs_referer": { + "kind": "string", + "identifier": true, + "description": "The client's referrer" + }, + "cs_user_agent": { + "kind": "string", + "identifier": true, + "description": "The client's HTTP agent" + } + }, + "sample": [ + { + "line": "10.112.72.172 - - [11/Feb/2013:06:43:36 +0000] \"GET /client/ HTTP/1.1\" 200 5778 \"-\" \"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.17 (KHTML, like Gecko) Chrome/24.0.1312.57 Safari/537.17\"", + "level": "info" + }, + { + "line": "10.112.72.172 - - [11/Feb/2013:06:43:36 +0000] \"GET /client/ HTTP/1.1\" 404 5778 \"-\" \"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.17 (KHTML, like Gecko) Chrome/24.0.1312.57 Safari/537.17\"", + "level": "error" + }, + { + "line": "2013-02-11T06:43:36 10.112.72.172 - GET \"/client/\" -1 200 5778 \"-\" \"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.17 (KHTML, like Gecko) Chrome/24.0.1312.57 Safari/537.17\"", + "level": "info" + }, + { + "line": "2013-02-11T06:43:36 10.112.72.172 - GET /client/ -1 200 5778 \"-\" \"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.17 (KHTML, like Gecko) Chrome/24.0.1312.57 Safari/537.17\"", + "level": "info" + }, + { + "line": "10.1.10.51 - - [23/Dec/2014:21:20:35 +0000] \"POST /api/1/rest/foo/bar HTTP/1.1\" 200 - \"-\" \"-\" 293" + }, + { + "line": "www.example.com 1.2.3.4 - theuser [10/Feb/2012:16:41:07 -0500] \"GET / HTTP/1.0\" 200 368 \"-\" \"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11\"" + }, + { + "line": "10.112.2.3 - - [16/Sep/2022:00:53:14 +0200] \"POST /api/v4/jobs/request HTTP/1.1\" 204 0 \"\" \"gitlab-runner 15.3.0 (15-3-stable; go1.19; linux/amd64)\" -", + "level": "info" + } + ] + } +} diff --git a/src/formats/alb_log.json b/src/formats/alb_log.json new file mode 100644 index 0000000..06dca84 --- /dev/null +++ b/src/formats/alb_log.json @@ -0,0 +1,133 @@ +{ + "$schema": "https://lnav.org/schemas/format-v1.schema.json", + "alb_log": { + "title": "Amazon ALB log", + "description": "Log format for Amazon Application Load Balancers", + "url": "https://docs.aws.amazon.com/elasticloadbalancing/latest/application/load-balancer-access-logs.html", + "regex": { + "std": { + "pattern": "^(?<type>(http)|(https)|(h2)|(ws)|(wss)) (?<timestamp>\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{6}Z) (?<elb>[^ ]+) (?<client_ip>[\\w\\.:]+):(?<client_port>\\d+) (?<target_ip>[\\w\\.:]+):(?<target_port>\\d+) (?<request_processing_time>(-1)|(\\d+(\\.\\d+))?) (?<target_processing_time>(-1)|(\\d+(\\.\\d+))?) (?<response_processing_time>(-1)|(\\d+(\\.\\d+))?) (?<elb_status_code>\\d+|-) (?<target_status_code>\\d+|-) (?<received_bytes>\\d+) (?<sent_bytes>\\d+) \"(?:\\-|(?<cs_method>\\w+|-) (?<cs_uri_whole>(?<cs_uri_stem>(?:(?<cs_uri_scheme>https|http)?://)?(?:(?<cs_uri_hostname>[^:]+):(?<cs_uri_port>\\d+)?)?(?<cs_uri_path>[^ \\?]+)?)(?:\\?(?<cs_uri_query>[^ ]*))?) (?<cs_version>[\\w/\\.]+|-)\\s*)\" \"(?<user_agent>[^\"]+)\" (?<ssl_cipher>[\\w-]+) (?<ssl_protocol>[\\w\\.-]+) (?<target_group_arn>[^ ]+) \"(?<trace_id>[^ ]+)\" (?<domain_name>[^ ]+) (?<chosen_cert_arn>[^ ]+) ?(?<matched_rule_priority>(-1)|\\b([0-9]|[1-8][0-9]|9[0-9]|[1-8][0-9]{2}|9[0-8][0-9]|99[0-9]|[1-8][0-9]{3}|9[0-8][0-9]{2}|99[0-8][0-9]|999[0-9]|[1-4][0-9]{4}|50000)\\b)?" + } + }, + "level-field": "elb_status_code", + "level": { + "error": "^[^123].*" + }, + "opid-field": "client_ip", + "value": { + "type": { + "kind": "string", + "identifier": true + }, + "elb": { + "kind": "string", + "identifier": true + }, + "client_ip": { + "kind": "string", + "collate": "ipaddress", + "identifier": true + }, + "client_port": { + "kind": "integer", + "foreign-key": true + }, + "target_ip": { + "kind": "string", + "collate": "ipaddress", + "identifier": true + }, + "target_port": { + "kind": "integer", + "foreign-key": true + }, + "request_processing_time": { + "kind": "float" + }, + "target_processing_time": { + "kind": "float" + }, + "response_processing_time": { + "kind": "float" + }, + "elb_status_code": { + "kind": "integer", + "foreign-key": true + }, + "target_status_code": { + "kind": "integer", + "foreign-key": true + }, + "received_bytes": { + "kind": "integer" + }, + "sent_bytes": { + "kind": "integer" + }, + "cs_method": { + "kind": "string", + "identifier": true + }, + "cs_uri_stem": { + "kind": "string", + "identifier": true + }, + "cs_uri_query": { + "kind": "string" + }, + "cs_version": { + "kind": "string", + "identifier": true + }, + "user_agent": { + "kind": "string", + "identifier": true + }, + "ssl_cipher": { + "kind": "string", + "identifier": true + }, + "ssl_protocol": { + "kind": "string", + "identifier": true + }, + "target_group_arn": { + "kind": "string", + "identifier": true + }, + "trace_id": { + "kind": "string", + "identifier": true + }, + "domain_name": { + "kind": "string", + "identifier": true + }, + "chosen_cert_arn": { + "kind": "string", + "identifier": true + }, + "matched_rule_priority": { + "kind": "integer", + "identifier": true + } + }, + "sample": [ + { + "line": "http 2016-08-10T22:08:42.945958Z app/my-loadbalancer/50dc6c495c0c9188 192.168.131.39:2817 10.0.0.1:80 0.000 0.001 0.000 200 200 34 366 \"GET http://www.example.com:80/ HTTP/1.1\" \"curl/7.46.0\" - - arn:aws:elasticloadbalancing:us-east-2:123456789012:targetgroup/my-targets/73e2d6bc24d8a067 \"Root=1-58337262-36d228ad5d99923122bbe354\" - -" + }, + { + "line": "https 2016-08-10T23:39:43.065466Z app/my-loadbalancer/50dc6c495c0c9188 192.168.131.39:2817 10.0.0.1:80 0.086 0.048 0.037 200 200 0 57 \"GET https://www.example.com:443/ HTTP/1.1\" \"curl/7.46.0\" ECDHE-RSA-AES128-GCM-SHA256 TLSv1.2 arn:aws:elasticloadbalancing:us-east-2:123456789012:targetgroup/my-targets/73e2d6bc24d8a067 \"Root=1-58337281-1d84f3d73c47ec4e58577259\" www.example.com arn:aws:acm:us-east-2:123456789012:certificate/12345678-1234-1234-1234-123456789012" + }, + { + "line": "h2 2016-08-10T00:10:33.145057Z app/my-loadbalancer/50dc6c495c0c9188 10.0.1.252:48160 10.0.0.66:9000 0.000 0.002 0.000 200 200 5 257 \"GET https://10.0.2.105:773/ HTTP/2.0\" \"curl/7.46.0\" ECDHE-RSA-AES128-GCM-SHA256 TLSv1.2 arn:aws:elasticloadbalancing:us-east-2:123456789012:targetgroup/my-targets/73e2d6bc24d8a067 \"Root=1-58337327-72bd00b0343d75b906739c42\" - -" + }, + { + "line": "ws 2016-08-10T00:32:08.923954Z app/my-loadbalancer/50dc6c495c0c9188 10.0.0.140:40914 10.0.1.192:8010 0.001 0.003 0.000 101 101 218 587 \"GET http://10.0.0.30:80/ HTTP/1.1\" \"-\" - - arn:aws:elasticloadbalancing:us-east-2:123456789012:targetgroup/my-targets/73e2d6bc24d8a067 \"Root=1-58337364-23a8c76965a2ef7629b185e3\" - -" + }, + { + "line": "wss 2016-08-10T00:42:46.423695Z app/my-loadbalancer/50dc6c495c0c9188 10.0.0.140:44244 10.0.0.171:8010 0.000 0.001 0.000 101 101 218 786 \"GET https://10.0.0.30:443/ HTTP/1.1\" \"-\" ECDHE-RSA-AES128-GCM-SHA256 TLSv1.2 arn:aws:elasticloadbalancing:us-west-2:123456789012:targetgroup/my-targets/73e2d6bc24d8a067 \"Root=1-58337364-23a8c76965a2ef7629b185e3\" - -" + } + ] + } +}
\ No newline at end of file diff --git a/src/formats/block_log.json b/src/formats/block_log.json new file mode 100644 index 0000000..aaba6d8 --- /dev/null +++ b/src/formats/block_log.json @@ -0,0 +1,23 @@ +{ + "$schema": "https://lnav.org/schemas/format-v1.schema.json", + "block_log": { + "title": "Generic Block", + "description": "A generic format for logs, like cron, that have a date at the start of a block.", + "regex": { + "std": { + "pattern": "^(?<timestamp>\\S{3,8} \\w{3}\\s+\\d{1,2} \\d{2}:\\d{2}:\\d{2} \\w+ \\d{4})\\s*(?<body>.*)$" + }, + "sq-brackets": { + "pattern": "^\\[(?<timestamp>\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}(?:\\.\\d{3,6})?)Z?\\]\\s*(?<body>.*)$" + } + }, + "sample": [ + { + "line": "Sat Apr 27 03:33:07 PDT 2013\nHello, World" + }, + { + "line": "[2021-05-21T21:58:57.022497Z]" + } + ] + } +}
\ No newline at end of file diff --git a/src/formats/candlepin_log.json b/src/formats/candlepin_log.json new file mode 100644 index 0000000..cab0395 --- /dev/null +++ b/src/formats/candlepin_log.json @@ -0,0 +1,49 @@ +{ + "$schema": "https://lnav.org/schemas/format-v1.schema.json", + "candlepin_log": { + "title": "Candlepin log format", + "description": "Log format used by Candlepin registration system", + "regex": { + "reqorg": { + "pattern": "^(?<timestamp>\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2},\\d{3}) \\[(req=(?<req>[0-9a-f-]+)|=), org=(?<org>\\w*)\\] (?<alert_level>\\w+) (?<module>[\\w.]+) - (?<body>.*)$" + }, + "other": { + "pattern": "^(?<timestamp>\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}[+-]\\d{4}) (?<body>.*)$" + } + }, + "value": { + "req": { + "kind": "string", + "identifier": true + }, + "org": { + "kind": "string", + "identifier": true + }, + "alert_level": { + "kind": "string" + }, + "module": { + "kind": "string", + "identifier": true + }, + "body": { + "kind": "string" + } + }, + "sample": [ + { + "line": "2015-04-17 09:41:50,544 [=, org=] INFO org.candlepin.guice.CustomizableModules - Found custom module module.config.katello" + }, + { + "line": "2015-04-17 09:41:56,320 [req=f91d4a84-020d-4874-9741-3979d0baf58d, org=] INFO org.candlepin.common.filter.LoggingFilter - Request: verb=GET, uri=/candlepin/status" + }, + { + "line": "2015-04-17 09:42:39+0200 principalType=trusteduser principal=admin target=OWNER entityId=8ab219c64cc653a7014cc6545a6c0001 type=CREATED owner=8ab219c64cc653a7014cc6545a6c0001" + }, + { + "line": "2015-04-17 10:49:21,912 [req=ec7867ea-2501-4036-bb08-e2d830720cb5, org=npr_goep_hm_com] INFO org.candlepin.common.filter.LoggingFilter - Response: status=200, content-type=\"application/json\", time=235ms" + } + ] + } +}
\ No newline at end of file diff --git a/src/formats/choose_repo_log.json b/src/formats/choose_repo_log.json new file mode 100644 index 0000000..6397049 --- /dev/null +++ b/src/formats/choose_repo_log.json @@ -0,0 +1,24 @@ +{ + "$schema": "https://lnav.org/schemas/format-v1.schema.json", + "choose_repo_log": { + "title": "Yum choose_repo Log", + "description": "The log format for the yum choose_repo tool.", + "regex": { + "std": { + "pattern": "^\\[(?<level>\\w+):[^\\]]+] [^:]+:\\d+ (?<timestamp>\\d{4}-\\d{2}-\\d{2}[T ]\\d{2}:\\d{2}:\\d{2}(?:[\\.,]\\d{3})?):(?<body>.*)" + } + }, + "level-field": "level", + "level": { + "error": "ERROR", + "debug": "DEBUG", + "info": "INFO", + "warning": "WARNING" + }, + "sample": [ + { + "line": "[INFO:choose_repo] choose_repo:47 2013-06-20 17:26:10,691: Setting region in redhat-rhui.repo" + } + ] + } +}
\ No newline at end of file diff --git a/src/formats/cups_log.json b/src/formats/cups_log.json new file mode 100644 index 0000000..b79f636 --- /dev/null +++ b/src/formats/cups_log.json @@ -0,0 +1,43 @@ +{ + "$schema": "https://lnav.org/schemas/format-v1.schema.json", + "cups_log": { + "title": "CUPS log format", + "description": "Log format used by the Common Unix Printing System", + "regex": { + "system": { + "pattern": "^(?<level>[IEW]) \\[(?<timestamp>\\d{2}/\\S{3,8}/\\d{4}:\\d{2}:\\d{2}:\\d{2} [+-]\\d{2,4})\\] (?<section>\\w+): (?<body>.*)$" + }, + "default": { + "pattern": "^(?<level>[IEW]) \\[(?<timestamp>\\d{2}/\\S{3,8}/\\d{4}:\\d{2}:\\d{2}:\\d{2} [+-]\\d{2,4})\\] (?!\\w+:)(?<body>.*)$" + } + }, + "level": { + "error": "E", + "warning": "W" + }, + "value": { + "level": { + "kind": "string", + "identifier": true + }, + "section": { + "kind": "string", + "identifier": true + }, + "body": { + "kind": "string" + } + }, + "sample": [ + { + "line": "I [04/Nov/2010:17:37:40 -0400] Allowing up to 100 client connections per host." + }, + { + "line": "I [04/Nov/2010:17:37:40 -0400] LoadPPDs: Wrote \"/etc/cups/ppds.dat\", 14 PPDs..." + }, + { + "line": "E [04/Nov/2010:17:37:40 -0400] StartListening: Unable to find IP address for server name \"localhost.localdomain\" - Host name lookup failure" + } + ] + } +}
\ No newline at end of file diff --git a/src/formats/dpkg_log.json b/src/formats/dpkg_log.json new file mode 100644 index 0000000..d485831 --- /dev/null +++ b/src/formats/dpkg_log.json @@ -0,0 +1,43 @@ +{ + "$schema": "https://lnav.org/schemas/format-v1.schema.json", + "dpkg_log": { + "title": "Dpkg Log", + "description": "The debian dpkg log.", + "regex": { + "std": { + "pattern": "^(?<timestamp>\\d{4}-\\d{2}-\\d{2}[T ]\\d{2}:\\d{2}:\\d{2}(?:\\.\\d{3})?) (?:(?:(?<action>startup|status|configure|install|upgrade|trigproc|remove|purge)(?: (?<status>config-files|failed-config|half-configured|half-installed|installed|not-installed|post-inst-failed|removal-failed|triggers-awaited|triggers-pending|unpacked))? (?<package>[^ ]+) (?<installed_version>[^ ]+)(?: (?<available_version>[^ ]+))?)|update-alternatives: (?<body>.*))$" + } + }, + "value": { + "action": { + "kind": "string", + "identifier": true + }, + "status": { + "kind": "string", + "identifier": true + }, + "package": { + "kind": "string", + "identifier": true + }, + "installed_version": { + "kind": "string" + }, + "available_version": { + "kind": "string" + } + }, + "sample": [ + { + "line": "2012-02-14 10:44:10 configure base-files 5.0.0ubuntu20 5.0.0ubuntu20" + }, + { + "line": "2012-02-14 10:44:30 status unpacked rsyslog 4.2.0-2ubuntu8" + }, + { + "line": "2012-02-14 10:44:32 update-alternatives: run with --install /usr/bin/rview rview /usr/bin/vim.tiny 10" + } + ] + } +}
\ No newline at end of file diff --git a/src/formats/elb_log.json b/src/formats/elb_log.json new file mode 100644 index 0000000..e13dc53 --- /dev/null +++ b/src/formats/elb_log.json @@ -0,0 +1,109 @@ +{ + "$schema": "https://lnav.org/schemas/format-v1.schema.json", + "elb_log": { + "title": "Amazon ELB log", + "description": "Log format for Amazon Elastic Load Balancers", + "url": "http://docs.aws.amazon.com/ElasticLoadBalancing/latest/DeveloperGuide/access-log-collection.html", + "regex": { + "std": { + "pattern": "^(?<timestamp>\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{6}Z) (?<elb>[^ ]+) (?<client_ip>[\\w\\.:]+):(?<client_port>\\d+) (?<backend_ip>[\\w\\.:]+):(?<backend_port>\\d+) (?<request_processing_time>\\d+(\\.\\d+)?) (?<backend_processing_time>\\d+(\\.\\d+)?) (?<response_processing_time>\\d+(\\.\\d+)?) (?<elb_status_code>\\d+|-) (?<backend_status_code>\\d+|-) (?<received_bytes>\\d+) (?<sent_bytes>\\d+) \"(?:\\-|(?<cs_method>\\w+|-) (?<cs_uri_stem>[^ \\?]+)(?:\\?(?<cs_uri_query>[^ ]*))? (?<cs_version>[\\w/\\.]+|-)\\s*)\" \"(?<user_agent>[^\"]+)\" (?<ssl_cipher>[\\w-]+) (?<ssl_protocol>[\\w\\.-]+)(?<body>.*)" + } + }, + "level-field": "elb_status_code", + "level": { + "error": "^[^123].*" + }, + "opid-field": "client_ip", + "value": { + "elb": { + "kind": "string", + "identifier": true + }, + "client_ip": { + "kind": "string", + "collate": "ipaddress", + "identifier": true + }, + "client_port": { + "kind": "integer", + "foreign-key": true + }, + "backend_ip": { + "kind": "string", + "collate": "ipaddress", + "identifier": true + }, + "backend_port": { + "kind": "integer", + "foreign-key": true + }, + "request_processing_time": { + "kind": "float" + }, + "backend_processing_time": { + "kind": "float" + }, + "response_processing_time": { + "kind": "float" + }, + "elb_status_code": { + "kind": "integer", + "foreign-key": true + }, + "backend_status_code": { + "kind": "integer", + "foreign-key": true + }, + "received_bytes": { + "kind": "integer" + }, + "sent_bytes": { + "kind": "integer" + }, + "cs_method": { + "kind": "string", + "identifier": true + }, + "cs_uri_stem": { + "kind": "string", + "identifier": true + }, + "cs_uri_query": { + "kind": "string" + }, + "cs_version": { + "kind": "string", + "identifier": true + }, + "user_agent": { + "kind": "string", + "identifier": true + }, + "ssl_cipher": { + "kind": "string", + "identifier": true + }, + "ssl_protocol": { + "kind": "string", + "identifier": true + } + }, + "sample": [ + { + "line": "2015-11-17T05:45:24.077255Z elastic-prod 54.161.222.121:40909 10.231.68.180:443 0.000031 0.009511 0.000029 200 200 0 415 \"GET https://example.com/foo/bar?baz=1234 HTTP/1.1\" \"test agent\" ECDHE-RSA-AES128-GCM-SHA256 TLSv1.2" + }, + { + "line": "2015-05-13T23:39:43.945958Z my-loadbalancer 192.168.131.39:2817 10.0.0.1:80 0.000073 0.001048 0.000057 200 200 0 29 \"GET http://www.example.com:80/ HTTP/1.1\" \"curl/7.38.0\" - -" + }, + { + "line": "2015-05-13T23:39:43.945958Z my-loadbalancer 192.168.131.39:2817 10.0.0.1:80 0.000086 0.001048 0.001337 200 200 0 57 \"GET https://www.example.com:443/ HTTP/1.1\" \"curl/7.38.0\" DHE-RSA-AES128-SHA TLSv1.2" + }, + { + "line": "2015-05-13T23:39:43.945958Z my-loadbalancer 192.168.131.39:2817 10.0.0.1:80 0.001069 0.000028 0.000041 - - 82 305 \"- - - \" \"-\" - -" + }, + { + "line": "2015-05-13T23:39:43.945958Z my-loadbalancer 192.168.131.39:2817 10.0.0.1:80 0.001065 0.000015 0.000023 - - 57 502 \"- - - \" \"-\" ECDHE-ECDSA-AES128-GCM-SHA256 TLSv1.2" + } + ] + } +}
\ No newline at end of file diff --git a/src/formats/engine_log.json b/src/formats/engine_log.json new file mode 100644 index 0000000..eeea2e8 --- /dev/null +++ b/src/formats/engine_log.json @@ -0,0 +1,34 @@ +{ + "$schema": "https://lnav.org/schemas/format-v1.schema.json", + "engine_log": { + "title": "engine log", + "description": "The log format for the engine.log files from RHEV/oVirt", + "regex": { + "std": { + "pattern": "^(?<timestamp>\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2},\\d{3}+)\\s+(?<level>\\w+)\\s+\\[(?<logger>[^\\]]+)\\]\\s+\\((?<tid>[^\\)]+)\\)\\s+(?<body>.*)" + } + }, + "opid-field": "tid", + "value": { + "tid": { + "kind": "string", + "identifier": true + }, + "logger": { + "kind": "string", + "identifier": true + } + }, + "level-field": "level", + "level": { + "error": "ERROR", + "info": "INFO", + "warning": "WARN" + }, + "sample": [ + { + "line": "2014-09-21 04:01:29,522 INFO [org.ovirt.engine.core.bll.OvfDataUpdater] (DefaultQuartzScheduler_Worker-90) Successfully updated VM OVFs in Data Center Test" + } + ] + } +}
\ No newline at end of file diff --git a/src/formats/error_log.json b/src/formats/error_log.json new file mode 100644 index 0000000..d3a94ad --- /dev/null +++ b/src/formats/error_log.json @@ -0,0 +1,67 @@ +{ + "$schema": "https://lnav.org/schemas/format-v1.schema.json", + "error_log": { + "title": "Common Error Log", + "description": "The default web error log format for servers like Apache.", + "regex": { + "cups": { + "pattern": "^(?<level>\\w) \\[(?<timestamp>[^\\]]+)\\] (?<body>.*)" + }, + "apache": { + "pattern": "^\\[(?<timestamp>[^\\]]+)\\] \\[(?:(?<module>[^:]+):)?(?<level>\\w+)\\](?: \\[pid (?<pid>\\d+)(:tid (?<tid>\\d+))?\\])?(?: \\[client (?<c_ip>[\\w\\.:\\-]+):(?<c_port>\\d+)\\])? (?<body>.*)" + } + }, + "level-field": "level", + "value": { + "module": { + "kind": "string", + "identifier": true + }, + "pid": { + "kind": "integer", + "identifier": true + }, + "tid": { + "kind": "integer", + "identifier": true, + "description": "The thread id" + }, + "c_ip": { + "kind": "string", + "collate": "ipaddress", + "identifier": true, + "description": "The client IP address" + }, + "c_port": { + "kind": "integer", + "identifier": true + } + }, + "sample": [ + { + "line": "E [08/Jun/2013:11:28:58 -0700] Unknown directive BrowseOrder on line 22 of /private/etc/cups/cupsd.conf.", + "level": "error" + }, + { + "line": "[Tue Apr 04 06:18:29.712806 2017] [mpm_prefork:notice] [pid 17725] AH00163: Apache/2.4.23 (Unix) configured -- resuming normal operations", + "level": "notice" + }, + { + "line": "[Tue Apr 04 06:28:08.605341 2017] [core:error] [pid 17962] [client 127.0.0.1:60444] AH00135: Invalid method in request FOO /", + "level": "error" + }, + { + "line": "[Thu Jan 17 02:42:49 2013] [notice] Digest: generating secret for digest authentication ...", + "level": "notice" + }, + { + "line": "[Thu May 12 08:28:57.652118 2011] [core:error] [pid 8777:tid 4326490112] [client ::1:58619] File does not exist: /usr/local/apache2/htdocs/favicon.ico", + "level": "error" + }, + { + "line": "[Thu Jan 02 22:23:07.368853 2020] [http:info] [pid 4784:tid 139701043291904] [client 66.220.149.10:45948] AH01593: chunked Transfer-Encoding forbidden: /", + "level": "info" + } + ] + } +}
\ No newline at end of file diff --git a/src/formats/esx_syslog_log.json b/src/formats/esx_syslog_log.json new file mode 100644 index 0000000..85fa881 --- /dev/null +++ b/src/formats/esx_syslog_log.json @@ -0,0 +1,66 @@ +{ + "$schema": "https://lnav.org/schemas/format-v1.schema.json", + "esx_syslog_log": { + "title": "ESXi Syslog", + "description": "Format specific to the ESXi syslog", + "regex": { + "std": { + "pattern": "^(?<timestamp>(?:\\S{3,8}\\s+\\d{1,2} \\d{2}:\\d{2}:\\d{2}|\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}(?:\\.\\d{3})?Z))\\s+(?<level>\\w+)\\((?<syslog_pri>\\d+)\\)(?:\\[\\+\\]|\\+)?(?:(?: (?<log_syslog_tag>(?<log_procname>(?:[^\\[:]+|[^:]+))(?:\\[(?<log_pid>\\d+)\\])?):\\s*(?<body>.*))$|:?(?:(?: ---)? last message repeated \\d+ times?(?: ---)?))" + }, + "host": { + "pattern": "^(?<timestamp>(?:\\S{3,8}\\s+\\d{1,2} \\d{2}:\\d{2}:\\d{2}|\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}(?:\\.\\d{3})?Z))\\s+(?<level>\\w+)\\((?<syslog_pri>\\d+)\\)(?:\\[\\+\\]|\\+)?(?:(?: (?<log_syslog_tag>(?:host-(?<log_pid>\\d+))?)\\s+(?<body>.*))$|:?(?:(?: ---)? last message repeated \\d+ times?(?: ---)?))" + }, + "notime": { + "pattern": "^(?<timestamp>\\d{4}-\\d{2}-\\d{2})\\s+(?<level>\\w+)\\((?<syslog_pri>\\d+)\\)\\s+(?<log_procname>[^\\[]+)\\[(?<log_pid>\\d+)\\]:\\s(?<new_time>\\d{2}:\\d{2}:\\d{2}\\.\\d+)\\s+(?<body>.*)" + } + }, + "level": { + "debug": "^Db$", + "info": "^In$", + "notice": "^No$", + "warning": "^Wa$", + "error": "^Er$", + "critical": "^Cr$", + "fatal": "^(?:Al|Em)$" + }, + "opid-field": "log_syslog_tag", + "time-field": "new_time", + "multiline": false, + "value": { + "body": { + "kind": "string" + }, + "log_pid": { + "kind": "string", + "identifier": true + }, + "log_procname": { + "kind": "string", + "identifier": true + }, + "log_syslog_tag": { + "kind": "string" + }, + "syslog_pri": { + "kind": "string" + }, + "timestamp": { + "kind": "string" + } + }, + "sample": [ + { + "line": "2022-06-02T05:34:56.746Z In(14) ConfigStore[1001430703]: Log for ConfigStore version=1.0 build=build-19833347 option=BETA" + }, + { + "line": "2022-06-02T05:34:23Z In(14)[+] hostprofile[1001430319]: {'mode': 'Disabled', 'exceptionUsers': []}" + }, + { + "line": "2022-06-02 In(14) hostprofile[1001430319]: 05:34:23.666 {'mode': 'Disabled', 'exceptionUsers': []}" + }, + { + "line": "2022-06-01T13:42:40.681Z In(05) host-16250 <analytics> Skip service health check. State STOPPED, Curr request 0" + } + ] + } +} diff --git a/src/formats/formats.am b/src/formats/formats.am new file mode 100644 index 0000000..bca37ec --- /dev/null +++ b/src/formats/formats.am @@ -0,0 +1,43 @@ + +FORMAT_FILES = \ + $(srcdir)/%reldir%/access_log.json \ + $(srcdir)/%reldir%/alb_log.json \ + $(srcdir)/%reldir%/block_log.json \ + $(srcdir)/%reldir%/candlepin_log.json \ + $(srcdir)/%reldir%/choose_repo_log.json \ + $(srcdir)/%reldir%/cups_log.json \ + $(srcdir)/%reldir%/dpkg_log.json \ + $(srcdir)/%reldir%/elb_log.json \ + $(srcdir)/%reldir%/engine_log.json \ + $(srcdir)/%reldir%/error_log.json \ + $(srcdir)/%reldir%/esx_syslog_log.json \ + $(srcdir)/%reldir%/fsck_hfs_log.json \ + $(srcdir)/%reldir%/glog_log.json \ + $(srcdir)/%reldir%/haproxy_log.json \ + $(srcdir)/%reldir%/java_log.json \ + $(srcdir)/%reldir%/journald_json_log.json \ + $(srcdir)/%reldir%/katello_log.json \ + $(srcdir)/%reldir%/openam_log.json \ + $(srcdir)/%reldir%/openamdb_log.json \ + $(srcdir)/%reldir%/openstack_log.json \ + $(srcdir)/%reldir%/page_log.json \ + $(srcdir)/%reldir%/papertrail_log.json \ + $(srcdir)/%reldir%/pcap_log.json \ + $(srcdir)/%reldir%/procstate_log.json \ + $(srcdir)/%reldir%/snaplogic_log.json \ + $(srcdir)/%reldir%/sssd_log.json \ + $(srcdir)/%reldir%/strace_log.json \ + $(srcdir)/%reldir%/sudo_log.json \ + $(srcdir)/%reldir%/syslog_log.json \ + $(srcdir)/%reldir%/s3_log.json \ + $(srcdir)/%reldir%/tcf_log.json \ + $(srcdir)/%reldir%/tcsh_history.json \ + $(srcdir)/%reldir%/unifi_log.json \ + $(srcdir)/%reldir%/uwsgi_log.json \ + $(srcdir)/%reldir%/vdsm_log.json \ + $(srcdir)/%reldir%/vmk_log.json \ + $(srcdir)/%reldir%/vmw_log.json \ + $(srcdir)/%reldir%/vmw_vc_svc_log.json \ + $(srcdir)/%reldir%/vmw_py_log.json \ + $(srcdir)/%reldir%/xmlrpc_log.json \ + $() diff --git a/src/formats/fsck_hfs_log.json b/src/formats/fsck_hfs_log.json new file mode 100644 index 0000000..21757ab --- /dev/null +++ b/src/formats/fsck_hfs_log.json @@ -0,0 +1,23 @@ +{ + "$schema": "https://lnav.org/schemas/format-v1.schema.json", + "fsck_hfs_log": { + "title": "Fsck_hfs Log", + "description": "Log for the fsck_hfs tool on Mac OS X.", + "regex": { + "std": { + "pattern": "^(?<device>[^:]+): fsck_hfs (?:run|started) at (?<timestamp>\\S{3,8} \\S{3,8}\\s+\\d{1,2} \\d{2}:\\d{2}:\\d{2} \\d{4})(?<body>.*)" + } + }, + "value": { + "device": { + "kind": "string", + "identifier": true + } + }, + "sample": [ + { + "line": "/dev/rdisk0s2: fsck_hfs run at Wed Jul 25 23:01:18 2012" + } + ] + } +}
\ No newline at end of file diff --git a/src/formats/glog_log.json b/src/formats/glog_log.json new file mode 100644 index 0000000..09fcff4 --- /dev/null +++ b/src/formats/glog_log.json @@ -0,0 +1,52 @@ +{ + "$schema": "https://lnav.org/schemas/format-v1.schema.json", + "glog_log": { + "title": "Glog", + "description": "The google glog format.", + "url": "https://code.google.com/p/google-glog/", + "regex": { + "std": { + "pattern": "^(?<level>[IWECF])(?<timestamp>\\d{4} \\d{2}:\\d{2}:\\d{2}\\.\\d{6}) +(?<thread>\\d+) (?<src_file>[^:]+):(?<src_line>\\d+)\\] (?<body>.*)" + }, + "std-with-year": { + "pattern": "^(?<level>[IWECF])(?<timestamp>\\d{8} \\d{2}:\\d{2}:\\d{2}\\.\\d{6}) +(?<thread>\\d+) (?<src_file>[^:]+):(?<src_line>\\d+)\\] (?<body>.*)" + } + }, + "level-field": "level", + "level": { + "error": "E", + "warning": "W", + "info": "I", + "critical": "C", + "fatal": "F" + }, + "opid-field": "thread", + "value": { + "thread": { + "kind": "integer", + "identifier": true, + "foreign-key": true + }, + "src_file": { + "kind": "string", + "identifier": true + }, + "src_line": { + "kind": "integer", + "foreign-key": true + } + }, + "sample": [ + { + "line": "E0517 15:04:22.619632 1952452992 logging_unittest.cc:253] Log every 3, iteration 19" + }, + { + "line": "E0517 15:04:22.619632 52992 logging_unittest.cc:253] Log every 3, iteration 19" + }, + { + "line": "I20200308 23:47:32.089828 400441 config.cc:27] Loading user configuration: /home/aesophor/.config/wmderland/config", + "level": "info" + } + ] + } +}
\ No newline at end of file diff --git a/src/formats/haproxy_log.json b/src/formats/haproxy_log.json new file mode 100644 index 0000000..9795a19 --- /dev/null +++ b/src/formats/haproxy_log.json @@ -0,0 +1,173 @@ +{ + "$schema": "https://lnav.org/schemas/format-v1.schema.json", + "haproxy_log": { + "title": "HAProxy HTTP Log Format", + "description": "The HAProxy log format", + "url": "http://www.haproxy.org/download/1.4/doc/configuration.txt", + "regex": { + "event_started": { + "pattern": "(?<timestamp>\\w{3} \\d{2} \\d{2}:\\d{2}:\\d{2}) (?<logging_host>[^ ]+) (?<process_name>\\w+)\\[(?<pid>\\d+)\\]: Proxy (?<frontend_name>[^ ]+) started." + }, + "event_stopping": { + "pattern": "(?<timestamp>\\w{3} \\d{2} \\d{2}:\\d{2}:\\d{2}) (?<logging_host>[^ ]+) (?<process_name>\\w+)\\[(?<pid>\\d+)\\]: Stopping frontend (?<frontend_name>[^ ]+) in (?<stopping_timeout>\\d+) ms." + }, + "event_stopped": { + "pattern": "(?<timestamp>\\w{3} \\d{2} \\d{2}:\\d{2}:\\d{2}) (?<logging_host>[^ ]+) (?<process_name>\\w+)\\[(?<pid>\\d+)\\]: Proxy (?<frontend_name>[^ ]+) stopped \\(FE: (?<frontend_connections>\\d+) conns, BE: (?<backend_connections>\\d+) conns\\)." + }, + "tcp": { + "pattern": "(?<timestamp>\\w{3} \\d{2} \\d{2}:\\d{2}:\\d{2}) (?<logging_host>[^ ]+) (?<process_name>\\w+)\\[(?<pid>\\d+)\\]: (?<client_ip>[^:]+):(?<client_port>\\d+) \\[(?<accept_date>\\d{2}\\/\\w{3}\\/\\d{4}:\\d{2}:\\d{2}:\\d{2}.\\d{3})\\] (?<frontend_name>[^ ]+) (?<backend_name>[^ ]+)\\/(?<server_name>[^ ]+) (?<tw>\\d+)\\/(?<tc>\\d+)\\/(?<tt>\\d+) (?<bytes_read>\\d+) (?<termination_state>..) (?<actconn>\\d+)\\/(?<feconn>\\d+)\\/(?<beconn>\\d+)\\/(?<srv_conn>\\d+)\\/(?<retries>\\d+) (?<srv_queue>\\d+)\\/(?<backend_queue>\\d+)" + }, + "http": { + "pattern": "(?<timestamp>\\w{3} \\d{2} \\d{2}:\\d{2}:\\d{2}) (?<logging_host>[^ ]+) (?<process_name>\\w+)\\[(?<pid>\\d+)\\]: (?<client_ip>[^:]+):(?<client_port>\\d+) \\[(?<accept_date>\\d{2}\\/\\w{3}\\/\\d{4}:\\d{2}:\\d{2}:\\d{2}.\\d{3})\\] (?<frontend_name>[^ ]+)(?<ssl>~)? (?<backend_name>[^ ]+)\\/(?<server_name>[^ ]+) (?<tq>-?\\d+)\\/(?<tw>-?\\d+)\\/(?<tc>-?\\d+)\\/(?<tr>-?\\d+)\\/(?<tt>\\d+) (?<status_code>\\d{3}|-1) (?<bytes_read>\\d+) (?<captured_request_cookie>.*) (?<captured_response_cookie>.*) (?<termination_state>....) (?<actconn>\\d+)\\/(?<feconn>\\d+)\\/(?<beconn>\\d+)\\/(?<srv_conn>\\d+)\\/(?<retries>\\d+) (?<srv_queue>\\d+)\\/(?<backend_queue>\\d+) (?:\\{(?<captured_request_headers>.*)\\} \\{(?<captured_response_headers>.*)\\} )?\"(?<http_method>[A-Z<>]+)(?: (?<http_url>.*?))?(?: (?<http_version>HTTP\\/\\d+.\\d+))?\"?$" + }, + "ssl": { + "pattern": "(?<timestamp>\\w{3} \\d{2} \\d{2}:\\d{2}:\\d{2}) (?<logging_host>[^ ]+) (?<process_name>\\w+)\\[(?<pid>\\d+)\\]: (?<client_ip>[^:]+):(?<client_port>\\d+) \\[(?<accept_date>\\d{2}\\/\\w{3}\\/\\d{4}:\\d{2}:\\d{2}:\\d{2}.\\d{3})\\] (?<backend_name>[^ ]+)\\/(?<server_name>[^ ]+): (?<ssl_error>.+)$" + } + }, + "json": false, + "value": { + "stopping_timeout": { + "kind": "integer" + }, + "frontend_connections": { + "kind": "integer" + }, + "backend_connections": { + "kind": "integer" + }, + "logging_host": { + "kind": "string" + }, + "process_name": { + "kind": "string" + }, + "pid": { + "kind": "integer", + "foreign-key": true + }, + "client_ip": { + "kind": "string", + "collate": "ipaddress" + }, + "client_port": { + "kind": "integer", + "foreign-key": true + }, + "accept_date": { + "kind": "string" + }, + "frontend_name": { + "kind": "string", + "identifier": true + }, + "ssl": { + "kind": "string" + }, + "ssl_error": { + "kind": "string" + }, + "backend_name": { + "kind": "string", + "identifier": true + }, + "server_name": { + "kind": "string", + "identifier": true + }, + "tq": { + "kind": "integer" + }, + "tw": { + "kind": "integer" + }, + "tc": { + "kind": "integer" + }, + "tr": { + "kind": "integer" + }, + "tt": { + "kind": "integer" + }, + "status_code": { + "kind": "integer", + "identifier": true + }, + "bytes_read": { + "kind": "integer" + }, + "captured_request_cookie": { + "kind": "string" + }, + "captured_response_cookie": { + "kind": "string" + }, + "termination_state": { + "kind": "string" + }, + "actconn": { + "kind": "integer", + "foreign-key": true + }, + "feconn": { + "kind": "integer", + "foreign-key": true + }, + "beconn": { + "kind": "integer", + "foreign-key": true + }, + "srv_conn": { + "kind": "integer", + "foreign-key": true + }, + "retries": { + "kind": "integer" + }, + "srv_queue": { + "kind": "integer", + "foreign-key": true + }, + "backend_queue": { + "kind": "integer", + "foreign-key": true + }, + "captured_request_headers": { + "kind": "string" + }, + "captured_response_headers": { + "kind": "string" + }, + "http_method": { + "kind": "string", + "identifier": true + }, + "http_url": { + "kind": "string" + }, + "http_version": { + "kind": "string" + } + }, + "sample": [ + { + "line": "Feb 26 10:07:24 192.168.8.2 haproxy[1]: Proxy prod_http_in started." + }, + { + "line": "Feb 26 10:00:47 192.168.8.2 haproxy[7]: Stopping frontend prod_http_in in 0 ms." + }, + { + "line": "Feb 26 10:00:47 192.168.8.2 haproxy[7]: Proxy prod_http_in stopped (FE: 847876 conns, BE: 0 conns)." + }, + { + "line": "Feb 26 23:08:47 192.168.8.2 haproxy[7]: 178.203.144.192:50210 [26/Feb/2019:23:08:47.266] prod_http_in/slsp: Connection closed during SSL handshake" + }, + { + "line": "Feb 26 23:16:16 192.168.8.2 haproxy[7]: 178.203.144.192:50210 [26/Feb/2019:23:16:15.321] prod_ssh_in prod_ssh_out/ssh1 1/1/861 1485 -- 2/1/0/0/0 0/0" + }, + { + "line": "Feb 26 00:29:44 192.168.8.2 haproxy[7]: 178.203.144.192:50210 [26/Feb/2019:00:29:44.326] prod_http_in~ prod_http_out/nginx1 0/0/1/48/49 200 3313 - - ---- 3/2/0/0/0 0/0 {Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:65.0) Gecko/20100101 Firefox/65.0} {} \"POST /schulportal/?Script=934&lehrer=126537&anm=3235&onlinetest=admin HTTP/1.1\"" + } + ] + } +}
\ No newline at end of file diff --git a/src/formats/java_log.json b/src/formats/java_log.json new file mode 100644 index 0000000..0d6297a --- /dev/null +++ b/src/formats/java_log.json @@ -0,0 +1,147 @@ +{ + "$schema": "https://lnav.org/schemas/format-v1.schema.json", + "java_log": { + "title": "Java log format", + "description": "Log format used by log4j and output by most java programs", + "url": "", + "regex": { + "jvm": { + "pattern": "^(?<level>\\w+)\\s+\\|\\s+jvm (?<jvm_no>\\d+)\\s+\\|\\s(?<timestamp>\\d{4}/\\d{2}/\\d{2} \\d{2}:\\d{2}:\\d{2})\\s+\\| (?<timestamp_f>\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2},\\d{3}) \\[(?<function>\\w+-\\d+)\\]\\s+(?<debug_level>\\w+)\\s+(?<class>[\\w.]+)\\s+-\\s+(?<body>.*)" + }, + "dump": { + "pattern": "^(?<level>\\w+)\\s+\\|\\s+jvm (?<jvm_no>\\d+)\\s+\\|\\s(?<timestamp>\\d{4}/\\d{2}/\\d{2} \\d{2}:\\d{2}:\\d{2})\\s+\\| JVMDUMP\\w+\\s(?<body>.*)$" + }, + "tasko": { + "pattern": "^(?<timestamp>\\d{4}-\\d{2}-\\d{2}( |T)\\d{2}:\\d{2}:\\d{2}(,|\\.)\\d{3}Z?)\\s+\\[(?<thread>[\\w\\-\\.]+)\\]\\s+(?<level>ERROR|WARN|INFO|DEBUG)\\s+(?<class>[\\w.]+)\\s+(-\\s+)?(?<body>.*)$" + }, + "prefix-brackets": { + "pattern": "^\\[(?<timestamp>\\d{4}-\\d{2}-\\d{2}( |T)\\d{2}:\\d{2}:\\d{2}(,|\\.)\\d{3}Z?)\\s+(?<thread>[\\w\\-\\.]+)\\s+(?<level>ERROR|WARN|INFO|DEBUG)\\s+(?<class>[\\w.]+)(?:\\s+opId=(?<opid>[^\\]]*))?\\]\\s*(-\\s+)?(?<body>.*)$" + }, + "in-brackets": { + "pattern": "^(?<timestamp>\\d{4}-\\d{2}-\\d{2}( |T)\\d{2}:\\d{2}:\\d{2}(,|\\.)\\d{3}Z?)\\s+\\[(?<thread>[\\w\\-\\.]+)(?:\\s+(?:\\[\\]|null))?\\s+(?<level>ERROR|WARN|INFO|DEBUG)\\s+(?<class>[\\w.]+)(?:\\s+opId=(?<opid>[^\\]]*))?\\]\\s*(-\\s+)?(?<body>.*)$" + }, + "nobrackets": { + "pattern": "^(?<timestamp>\\d{4}-\\d{2}-\\d{2}( |T)\\d{2}:\\d{2}:\\d{2}(,|\\.)\\d{3}Z?)\\s+(?<thread>[\\w\\-\\.]+)\\s+(?<level>ERROR|WARN|INFO|DEBUG)\\s+(?<class>[\\w.]+)\\s+(-\\s+)?(?<body>.*)$" + }, + "vmw1": { + "pattern": "^(?<timestamp>\\d{4}-\\d{2}-\\d{2}( |T)\\d{2}:\\d{2}:\\d{2}(,|\\.)\\d{3}Z?)\\s*\\|\\s*(?<level>ERROR|WARN|INFO|DEBUG)\\s*\\|\\s*(?<thread>[^\\|]+)\\s*\\|\\s*(?<srcfile>[^\\|]+)\\s*\\|\\s*(?<srcline>\\d+)\\s*\\|\\s*(?<body>.*)$" + }, + "vmw2": { + "pattern": "^\\[(?<timestamp>\\d{4}-\\d{2}-\\d{2}( |T)\\d{2}:\\d{2}:\\d{2}(,|\\.)\\d{3}Z?)\\]\\s*(?<level>ERROR|WARN|INFO|DEBUG)\\s*\\d+\\[(?<thread>[^\\]]+)\\]\\s+-\\s+(?<class>[^\\(]+)\\.(?<method>\\w+)\\((?<srcfile>[^:]+):(?<srcline>\\d+)\\)\\s+-\\s+(?<body>.*)$" + }, + "vmw3": { + "pattern": "^(?<timestamp>\\d{4}-\\d{2}-\\d{2}( |T)\\d{2}:\\d{2}:\\d{2}(,|\\.)\\d{3}Z?)\\s*\\|\\s*(?<level>ERROR|WARN|INFO|DEBUG)\\s*\\|\\s*(?<thread>[^\\|]+)\\s*\\|\\s*(?<class>[^\\|]+)\\s*\\|\\s+(?!\\d+\\s*\\|)(?<body>.*)$" + }, + "vmw-sso": { + "pattern": "^(?<timestamp>\\d{4}-\\d{2}-\\d{2}( |T)\\d{2}:\\d{2}:\\d{2}(,|\\.)\\d{3}Z?)\\s+(?<level>ERROR|WARN|INFO|DEBUG)\\s+[\\w\\-]+\\[\\d+:(?<thread>[^\\]]+)\\]\\s+\\[CorId=(?<corid>[^\\s\\]]*)(?:\\s+OpId=(?<opid>[^\\]]*))?\\]\\s+\\[(?<class>[^\\]]+)\\]\\s+(?<body>.*)$" + }, + "vmw-sps": { + "pattern": "^(?<timestamp>\\d{4}-\\d{2}-\\d{2}( |T)\\d{2}:\\d{2}:\\d{2}(,|\\.)\\d{3}Z?)\\s+\\[(?<thread>[^\\]]+)\\]\\s+(?<level>ERROR|WARN|INFO|DEBUG)\\s+opId=(?<opid>\\S*)\\s+(?<class>\\S+)\\s+-\\s+(?<body>.*)$" + } + }, + "level-field": "level", + "opid-field": "opid", + "level": { + "error": "ERROR", + "warning": "WARN", + "debug": "DEBUG", + "info": "INFO" + }, + "value": { + "function": { + "kind": "string", + "identifier": true + }, + "thread": { + "kind": "string", + "identifier": true + }, + "level": { + "kind": "string" + }, + "jvm_no": { + "kind": "integer" + }, + "debug_level": { + "kind": "string" + }, + "opid": { + "kind": "string", + "identifier": true + }, + "corid": { + "kind": "string", + "identifier": true + }, + "class": { + "kind": "string", + "identifier": true + }, + "method": { + "kind": "string", + "identifier": true + }, + "srcfile": { + "kind": "string", + "identifier": true + }, + "srcline": { + "kind": "string", + "identifier": true + }, + "body": { + "kind": "string" + } + }, + "sample": [ + { + "line": "INFO | jvm 1 | 2015/04/28 18:40:00 | 2015-04-28 18:40:00,077 [DefaultQuartzScheduler_Worker-8] INFO com.redhat.rhn.taskomatic.TaskoJob - errata-queue-default: bunch errata-queue-bunch STARTED" + }, + { + "line": "INFO | jvm 1 | 2015/04/28 18:34:18 | 2015-04-28 18:34:18,872 [Thread-46] DEBUG com.redhat.rhn.common.hibernate.ConnectionManager - Adding resource com/redhat/rhn/domain/action/ActionArchType.hbm.xml" + }, + { + "line": "2015-05-22 16:10:00,123 [DefaultQuartzScheduler_Worker-5] INFO com.redhat.rhn.taskomatic.task.ErrataCacheTask - In the queue: 24" + }, + { + "line": "INFO | jvm 1 | 2015/05/24 07:35:50 | JVMDUMP013I Processed dump event \"user\", detail \"\"." + }, + { + "line": "2022-06-02T12:12:38.414Z phProdLogDrainerTaskExecutor-5 INFO org.bouncycastle.jsse.provider.ProvTrustManagerFactorySpi Initializing with trust store at path: /usr/java/jre-vmware/lib/security/cacerts" + }, + { + "line": "2022-06-02T12:23:11.514Z | INFO | vim-async-1 | VcEventManager.java | 806 | [EventIndex: 2154] Event posted." + }, + { + "line": "2022-06-02T12:23:44.971Z [syncaas-grpc-5 INFO com.vmware.hvc.topology.util.LookupServiceUtil opId=] Local Node id is 9c66ff98-3fee-420c-a2bb-dbe2276c1aab" + }, + { + "line": "[2022-06-02T10:45:15.969Z tomcat-http--188 ERROR com.vmware.vim.vmomi.server.http.impl.AsyncServlet30Template] Internal server error during asynchronous request processing" + }, + { + "line": "[2022-06-01T13:37:36,371] WARN574240[Thread-35] - com.vmware.observability.observer.Observer.execute(Observer.java:384) - No metric configured for observation from source LimitCollectorPlugin.limit" + }, + { + "line": "2022-06-02T12:23:44.070Z INFO tokenservice[83:tomcat-http--36] [CorId=95c59584-4472-4f7c-ad9e-f228b94d9b45 OpId=16205349-254c-4f76-a7f1-aa15aae385c5] [com.vmware.vcenter.tokenservice.ExchangeFacadeImpl] Parsed Caller token; tokenType=SAML2" + }, + { + "line": "2022-06-01T13:43:59.791Z [main [] INFO com.vmware.vcenter.trustmanagement.service.TrustManagement opId=] trustmanagement-vlsi.xml" + }, + { + "line": "2022-06-02T08:34:01.203Z | INFO | state-manager1 | org.eclipse.jetty.server.session | DefaultSessionIdManager workerName=node0" + }, + { + "line": "2022-06-02T11:26:26.803Z [pool-26-thread-1] INFO opId=sps-Main-158837-921 com.vmware.vim.storage.common.util.OperationIdUtil - OperationID present in invoker thread, adding suffix and re-using it - sps-Main-158837-921-169186-507." + }, + { + "line": "2022-06-02T08:34:19.574Z [main null INFO com.vmware.cis.server.util.PerfLog opId=] Requesting LDAP connection" + }, + { + "line": "2022-06-01T13:42:32.739Z INFO sts-perf[23:localhost-startStop-1] [CorId=] [com.vmware.identity.performanceSupport.PerfDataSink] restarting PerfDataSink." + }, + { + "line": "2022-06-01T13:42:32.742Z INFO sts-default[23:localhost-startStop-1] [CorId= OpId=] [com.vmware.identity.idm.server.provider.PooledLdapConnectionFactory] New connection created in pool PooledLdapConnectionIdentity [tenantName=null, username=vc.vlcm.com@vsphere.local, authType=SRP, useGCPort=false, connectionString=ldap://vc.vlcm.com:389]" + } + ] + } +}
\ No newline at end of file diff --git a/src/formats/journald_json_log.json b/src/formats/journald_json_log.json new file mode 100644 index 0000000..b6ec0e9 --- /dev/null +++ b/src/formats/journald_json_log.json @@ -0,0 +1,84 @@ +{ + "$schema": "https://lnav.org/schemas/format-v1.schema.json", + "journald_json_log": { + "title": "journalctl JSON log format", + "description": "Logger format as created by systemd journalctl -o json", + "url": "https://www.freedesktop.org/wiki/Software/systemd/json/", + "json": true, + "hide-extra": true, + "convert-to-local-time": true, + "line-format": [ + { + "field": "__REALTIME_TIMESTAMP" + }, + " ", + { + "field": "__MONOTONIC_TIMESTAMP" + }, + " ", + { + "field": "_SYSTEMD_UNIT" + }, + " ", + { + "field": "SYSLOG_IDENTIFIER" + }, + "[", + { + "field": "_PID" + }, + "] ", + { + "field": "__level__", + "text-transform": "uppercase" + }, + " ", + { + "field": "MESSAGE" + } + ], + "timestamp-field": "__REALTIME_TIMESTAMP", + "timestamp-format": [ + "%6" + ], + "level-field": "PRIORITY", + "level": { + "fatal": "0|1", + "critical": "2", + "error": "3", + "warning": "4", + "stats": "5", + "info": "6", + "debug": "7" + }, + "body-field": "MESSAGE", + "value": { + "__REALTIME_TIMESTAMP": { + "kind": "integer" + }, + "__MONOTONIC_TIMESTAMP": { + "kind": "integer" + }, + "_SYSTEMD_UNIT": { + "kind": "string", + "identifier": true + }, + "SYSLOG_IDENTIFIER": { + "kind": "string", + "identifier": true + }, + "_PID": { + "kind": "integer", + "identifier": true + }, + "PRIORITY": { + "kind": "string", + "identifier": true, + "foreign-key": true + }, + "MESSAGE": { + "kind": "string" + } + } + } +}
\ No newline at end of file diff --git a/src/formats/katello_log.json b/src/formats/katello_log.json new file mode 100644 index 0000000..750b14a --- /dev/null +++ b/src/formats/katello_log.json @@ -0,0 +1,48 @@ +{ + "$schema": "https://lnav.org/schemas/format-v1.schema.json", + "katello_log": { + "title": "Katello log format", + "description": "Log format used by katello and foreman as used in Satellite 6.", + "url": "http://theforeman.org/", + "regex": { + "log": { + "pattern": "^\\[\\s?(?<alert_level>\\w+)\\s(?<timestamp>\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2})\\s(?<module>\\w+)\\]\\s+(?<message>.*)$" + } + }, + "level-field": "alert_level", + "level": { + "error": "ERROR", + "warning": "WARN", + "debug": "DEBUG" + }, + "value": { + "alert_level": { + "kind": "string" + }, + "module": { + "kind": "string" + }, + "message": { + "kind": "string" + } + }, + "sample": [ + { + "line": "[DEBUG 2015-05-20 12:22:19 main] /Stage[main]/Certs::Candlepin/Exec[create candlepin qpid exchange]/unless: Failed: ConnectError: [Errno 1] _ssl.c:504: error:14094418:SSL routines:SSL3_READ_BYTES:tlsv1 alert unknown ca", + "level": "debug" + }, + { + "line": "[DEBUG 2015-05-20 12:22:19 main] Exec[create candlepin qpid exchange](provider=posix): Executing 'qpid-config --ssl-certificate /etc/pki/katello/certs/java-client.crt --ssl-key /etc/pki/katello/private/java-client.key -b 'amqps://avl248.bcc.qld.gov.au:5671' add exchange topic event --durable'", + "level": "debug" + }, + { + "line": "[ERROR 2015-05-20 12:22:19 main] qpid-config --ssl-certificate /etc/pki/katello/certs/java-client.crt --ssl-key /etc/pki/katello/private/java-client.key -b 'amqps://avl248.bcc.qld.gov.au:5671' add exchange topic event --durable returned 1 instead of one of [0]", + "level": "error" + }, + { + "line": "[ INFO 2015-05-20 12:22:19 main] /usr/share/ruby/vendor_ruby/puppet/util/errors.rb:104:in `fail'", + "level": "info" + } + ] + } +}
\ No newline at end of file diff --git a/src/formats/logfmt/CMakeLists.txt b/src/formats/logfmt/CMakeLists.txt new file mode 100644 index 0000000..a24abae --- /dev/null +++ b/src/formats/logfmt/CMakeLists.txt @@ -0,0 +1,40 @@ + +add_library( + logfmt + STATIC + logfmt.parser.hh + logfmt.parser.cc +) + +target_include_directories( + logfmt + PUBLIC + . + ${CMAKE_BINARY_DIR}/src + ${CMAKE_SOURCE_DIR}/src +) +target_link_libraries( + logfmt + PRIVATE + cppfmt + cppscnlib +) + +add_executable( + logfmt.parser.test + logfmt.parser.test.cc +) +target_include_directories( + logfmt.parser.test + PUBLIC + . + ${CMAKE_BINARY_DIR}/src + ${CMAKE_SOURCE_DIR}/src + ../../third-party/doctest-root) +target_link_libraries( + logfmt.parser.test + logfmt + base +) + +add_test(NAME logfmt.parser.test COMMAND logfmt.parser.test) diff --git a/src/formats/logfmt/Makefile.am b/src/formats/logfmt/Makefile.am new file mode 100644 index 0000000..8d1f3ff --- /dev/null +++ b/src/formats/logfmt/Makefile.am @@ -0,0 +1,41 @@ + +include $(top_srcdir)/aminclude_static.am + +AM_CPPFLAGS = \ + $(CODE_COVERAGE_CPPFLAGS) \ + -Wall \ + -I$(top_srcdir)/src/ \ + -I$(top_srcdir)/src/third-party \ + -I$(top_srcdir)/src/fmtlib \ + -I$(top_srcdir)/src/third-party/scnlib/include \ + $(LIBARCHIVE_CFLAGS) \ + $(READLINE_CFLAGS) \ + $(SQLITE3_CFLAGS) \ + $(LIBCURL_CPPFLAGS) + +AM_LIBS = $(CODE_COVERAGE_LIBS) +AM_CFLAGS = $(CODE_COVERAGE_CFLAGS) +AM_CXXFLAGS = $(CODE_COVERAGE_CXXFLAGS) + +noinst_LIBRARIES = liblogfmt.a + +noinst_HEADERS = \ + logfmt.parser.hh + +liblogfmt_a_SOURCES = \ + logfmt.parser.cc + +check_PROGRAMS = \ + logfmt.parser.test + +logfmt_parser_test_SOURCES = \ + logfmt.parser.test.cc + +logfmt_parser_test_LDADD = \ + liblogfmt.a \ + $(top_builddir)/src/base/libbase.a \ + $(top_builddir)/src/pcrepp/libpcrepp.a \ + $(top_builddir)/src/third-party/scnlib/src/libscnlib.a + +TESTS = \ + logfmt.parser.test diff --git a/src/formats/logfmt/logfmt.parser.cc b/src/formats/logfmt/logfmt.parser.cc new file mode 100644 index 0000000..20c7252 --- /dev/null +++ b/src/formats/logfmt/logfmt.parser.cc @@ -0,0 +1,266 @@ +/** + * Copyright (c) 2021, Timothy Stack + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * * Neither the name of Timothy Stack nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ''AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * @file logfmt.parser.cc + */ + +#include "logfmt.parser.hh" + +#include "base/intern_string.hh" +#include "config.h" +#include "scn/scn.h" + +logfmt::parser::parser(string_fragment sf) : p_next_input(sf) {} + +static bool +is_not_eq(char ch) +{ + return ch != '='; +} + +struct bare_value_predicate { + enum class int_state_t { + INIT, + NEED_DIGIT, + DIGITS, + INVALID, + }; + + enum class float_state_t { + INIT, + NEED_DIGIT, + DIGITS, + FRACTION_DIGIT, + EXPONENT_INIT, + EXPONENT_NEED_DIGIT, + EXPONENT_DIGIT, + INVALID, + }; + + int_state_t bvp_int_state{int_state_t::INIT}; + float_state_t bvp_float_state{float_state_t::INIT}; + size_t bvp_index{0}; + + bool is_integer() const + { + return this->bvp_int_state == int_state_t::DIGITS; + } + + bool is_float() const + { + switch (this->bvp_float_state) { + case float_state_t::DIGITS: + case float_state_t::FRACTION_DIGIT: + case float_state_t::EXPONENT_DIGIT: + return true; + default: + return false; + } + } + + bool operator()(char ch) + { + if (ch == ' ') { + return false; + } + + bool got_digit = isdigit(ch); + switch (this->bvp_int_state) { + case int_state_t::INIT: + if (got_digit) { + this->bvp_int_state = int_state_t::DIGITS; + } else if (ch == '-') { + this->bvp_int_state = int_state_t::NEED_DIGIT; + } else { + this->bvp_int_state = int_state_t::INVALID; + } + break; + case int_state_t::DIGITS: + case int_state_t::NEED_DIGIT: + if (got_digit) { + this->bvp_int_state = int_state_t::DIGITS; + } else { + this->bvp_int_state = int_state_t::INVALID; + } + break; + case int_state_t::INVALID: + break; + } + + switch (this->bvp_float_state) { + case float_state_t::INIT: + if (got_digit) { + this->bvp_float_state = float_state_t::DIGITS; + } else if (ch == '-') { + this->bvp_float_state = float_state_t::NEED_DIGIT; + } else { + this->bvp_float_state = float_state_t::INVALID; + } + break; + case float_state_t::DIGITS: + case float_state_t::NEED_DIGIT: + if (got_digit) { + this->bvp_float_state = float_state_t::DIGITS; + } else if (ch == '.') { + this->bvp_float_state = float_state_t::FRACTION_DIGIT; + } else if (ch == 'e' || ch == 'E') { + this->bvp_float_state = float_state_t::EXPONENT_INIT; + } else { + this->bvp_float_state = float_state_t::INVALID; + } + break; + case float_state_t::FRACTION_DIGIT: + if (got_digit) { + this->bvp_float_state = float_state_t::FRACTION_DIGIT; + } else if (ch == 'e' || ch == 'E') { + this->bvp_float_state = float_state_t::EXPONENT_INIT; + } else { + this->bvp_float_state = float_state_t::INVALID; + } + break; + case float_state_t::EXPONENT_INIT: + if (got_digit) { + this->bvp_float_state = float_state_t::EXPONENT_DIGIT; + } else if (ch == '-' || ch == '+') { + this->bvp_float_state = float_state_t::EXPONENT_NEED_DIGIT; + } else { + this->bvp_float_state = float_state_t::INVALID; + } + break; + case float_state_t::EXPONENT_NEED_DIGIT: + case float_state_t::EXPONENT_DIGIT: + if (got_digit) { + this->bvp_float_state = float_state_t::EXPONENT_DIGIT; + } else { + this->bvp_float_state = float_state_t::INVALID; + } + break; + case float_state_t::INVALID: + break; + } + + this->bvp_index += 1; + + return true; + } +}; + +logfmt::parser::step_result +logfmt::parser::step() +{ + const static auto IS_DQ = string_fragment::tag1{'"'}; + + auto remaining = this->p_next_input.skip(isspace); + + if (remaining.empty()) { + return end_of_input{}; + } + + auto pair_opt = remaining.split_while(is_not_eq); + + if (!pair_opt) { + return error{remaining.sf_begin, "expecting key followed by '='"}; + } + + auto key_frag = pair_opt->first; + auto after_eq = pair_opt->second.consume(string_fragment::tag1{'='}); + + if (!after_eq) { + return error{pair_opt->second.sf_begin, "expecting '='"}; + } + + auto value_start = after_eq.value(); + + if (value_start.startswith("\"")) { + string_fragment::quoted_string_body qsb; + auto quoted_pair = value_start.consume_n(1)->split_while(qsb); + + if (!quoted_pair) { + return error{value_start.sf_begin + 1, "string body missing"}; + } + + auto after_quote = quoted_pair->second.consume(IS_DQ); + + if (!after_quote) { + return error{quoted_pair->second.sf_begin, "non-terminated string"}; + } + + this->p_next_input = after_quote.value(); + return std::make_pair( + key_frag, + quoted_value{string_fragment{quoted_pair->first.sf_string, + quoted_pair->first.sf_begin - 1, + quoted_pair->first.sf_end + 1}}); + } + + bare_value_predicate bvp; + auto value_pair = value_start.split_while(bvp); + + if (value_pair) { + static const auto TRUE_FRAG = string_fragment::from_const("true"); + static const auto FALSE_FRAG = string_fragment::from_const("false"); + + this->p_next_input = value_pair->second; + if (bvp.is_integer()) { + int_value retval; + + auto int_scan_res + = scn::scan_value<int64_t>(value_pair->first.to_string_view()); + if (int_scan_res) { + retval.iv_value = int_scan_res.value(); + } + retval.iv_str_value = value_pair->first; + + return std::make_pair(key_frag, retval); + } + if (bvp.is_float()) { + float_value retval; + + auto float_scan_res + = scn::scan_value<double>(value_pair->first.to_string_view()); + if (float_scan_res) { + retval.fv_value = float_scan_res.value(); + } + retval.fv_str_value = value_pair->first; + + return std::make_pair(key_frag, retval); + } + if (value_pair->first.iequal(TRUE_FRAG)) { + return std::make_pair(key_frag, + bool_value{true, value_pair->first}); + } + if (value_pair->first.iequal(FALSE_FRAG)) { + return std::make_pair(key_frag, + bool_value{false, value_pair->first}); + } + return std::make_pair(key_frag, unquoted_value{value_pair->first}); + } + + this->p_next_input = value_start; + return std::make_pair(key_frag, unquoted_value{string_fragment{}}); +} diff --git a/src/formats/logfmt/logfmt.parser.hh b/src/formats/logfmt/logfmt.parser.hh new file mode 100644 index 0000000..7806001 --- /dev/null +++ b/src/formats/logfmt/logfmt.parser.hh @@ -0,0 +1,91 @@ +/** + * Copyright (c) 2021, Timothy Stack + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * * Neither the name of Timothy Stack nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ''AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * @file logfmt.parser.hh + */ + +#ifndef lnav_logfmt_parser_hh +#define lnav_logfmt_parser_hh + +#include "base/intern_string.hh" +#include "base/result.h" +#include "mapbox/variant.hpp" + +namespace logfmt { + +class parser { +public: + explicit parser(string_fragment sf); + + struct end_of_input {}; + struct error { + int e_offset; + const std::string e_msg; + }; + struct unquoted_value { + string_fragment uv_value; + }; + struct quoted_value { + string_fragment qv_value; + }; + struct bool_value { + bool bv_value{false}; + string_fragment bv_str_value; + }; + struct int_value { + int64_t iv_value{0}; + string_fragment iv_str_value; + }; + struct float_value { + double fv_value{0}; + string_fragment fv_str_value; + }; + using value_type = mapbox::util::variant< + bool_value, + int_value, + float_value, + unquoted_value, + quoted_value + >; + + using kvpair = std::pair<string_fragment, value_type>; + + using step_result = mapbox::util::variant< + end_of_input, + kvpair, + error + >; + + step_result step(); +private: + string_fragment p_next_input; +}; + +} + +#endif diff --git a/src/formats/logfmt/logfmt.parser.test.cc b/src/formats/logfmt/logfmt.parser.test.cc new file mode 100644 index 0000000..2193bfe --- /dev/null +++ b/src/formats/logfmt/logfmt.parser.test.cc @@ -0,0 +1,221 @@ +/** + * Copyright (c) 2021, Timothy Stack + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * * Neither the name of Timothy Stack nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ''AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * @file logfmt.parser.test.cc + */ + +#include "config.h" + +#include <iostream> + +#define DOCTEST_CONFIG_IMPLEMENT_WITH_MAIN +#include "doctest/doctest.h" + +#include "logfmt.parser.hh" + +TEST_CASE("basic") +{ + static const char *line = "abc=def ghi=\"1 2 3 4\" time=333 empty1= tf=true empty2="; + + auto p = logfmt::parser{string_fragment{line}}; + + auto pair1 = p.step(); + + CHECK(pair1.is<logfmt::parser::kvpair>()); + CHECK(pair1.get<logfmt::parser::kvpair>().first == "abc"); + CHECK(pair1.get<logfmt::parser::kvpair>().second + .get<logfmt::parser::unquoted_value>().uv_value == "def"); + + auto pair2 = p.step(); + + CHECK(pair2.is<logfmt::parser::kvpair>()); + CHECK(pair2.get<logfmt::parser::kvpair>().first == "ghi"); + CHECK(pair2.get<logfmt::parser::kvpair>().second + .get<logfmt::parser::quoted_value>().qv_value == "\"1 2 3 4\""); + + auto pair3 = p.step(); + + CHECK(pair3.is<logfmt::parser::kvpair>()); + CHECK(pair3.get<logfmt::parser::kvpair>().first == "time"); + CHECK(pair3.get<logfmt::parser::kvpair>().second + .get<logfmt::parser::int_value>().iv_value == 333); + + auto pair4 = p.step(); + + CHECK(pair4.is<logfmt::parser::kvpair>()); + CHECK(pair4.get<logfmt::parser::kvpair>().first == "empty1"); + CHECK(pair4.get<logfmt::parser::kvpair>().second + .get<logfmt::parser::unquoted_value>().uv_value == ""); + + auto pair5 = p.step(); + + CHECK(pair5.is<logfmt::parser::kvpair>()); + CHECK(pair5.get<logfmt::parser::kvpair>().first == "tf"); + CHECK(pair5.get<logfmt::parser::kvpair>().second + .get<logfmt::parser::bool_value>().bv_value); + + auto pair6 = p.step(); + + CHECK(pair6.is<logfmt::parser::kvpair>()); + CHECK(pair6.get<logfmt::parser::kvpair>().first == "empty2"); + CHECK(pair6.get<logfmt::parser::kvpair>().second + .get<logfmt::parser::unquoted_value>().uv_value == ""); + + auto eoi = p.step(); + CHECK(eoi.is<logfmt::parser::end_of_input>()); +} + +TEST_CASE("floats") +{ + static const char *line = "f1=1.0 f2=-2.0 f3=1.2e3 f4=1.2e-2 f5=2e1 f6=2e+1"; + + auto p = logfmt::parser{string_fragment{line}}; + + auto pair1 = p.step(); + + CHECK(pair1.is<logfmt::parser::kvpair>()); + CHECK(pair1.get<logfmt::parser::kvpair>().first == "f1"); + CHECK(pair1.get<logfmt::parser::kvpair>().second + .get<logfmt::parser::float_value>().fv_value == 1.0); + + auto pair2 = p.step(); + + CHECK(pair2.is<logfmt::parser::kvpair>()); + CHECK(pair2.get<logfmt::parser::kvpair>().first == "f2"); + CHECK(pair2.get<logfmt::parser::kvpair>().second + .get<logfmt::parser::float_value>().fv_value == -2.0); + + auto pair3 = p.step(); + + CHECK(pair3.is<logfmt::parser::kvpair>()); + CHECK(pair3.get<logfmt::parser::kvpair>().first == "f3"); + CHECK(pair3.get<logfmt::parser::kvpair>().second + .get<logfmt::parser::float_value>().fv_value == 1200); + + auto pair4 = p.step(); + + CHECK(pair4.is<logfmt::parser::kvpair>()); + CHECK(pair4.get<logfmt::parser::kvpair>().first == "f4"); + CHECK(pair4.get<logfmt::parser::kvpair>().second + .get<logfmt::parser::float_value>().fv_value == 0.012); + + auto pair5 = p.step(); + + CHECK(pair5.is<logfmt::parser::kvpair>()); + CHECK(pair5.get<logfmt::parser::kvpair>().first == "f5"); + CHECK(pair5.get<logfmt::parser::kvpair>().second + .get<logfmt::parser::float_value>().fv_value == 20); + + auto pair6 = p.step(); + + CHECK(pair6.is<logfmt::parser::kvpair>()); + CHECK(pair6.get<logfmt::parser::kvpair>().first == "f6"); + CHECK(pair6.get<logfmt::parser::kvpair>().second + .get<logfmt::parser::float_value>().fv_value == 20); +} + +TEST_CASE("bad floats") +{ + static const char *line = "bf1=- bf2=-1.2e bf3=1.2.3 bf4=1e2e4"; + + auto p = logfmt::parser{string_fragment{line}}; + + auto pair1 = p.step(); + + CHECK(pair1.is<logfmt::parser::kvpair>()); + CHECK(pair1.get<logfmt::parser::kvpair>().first == "bf1"); + CHECK(pair1.get<logfmt::parser::kvpair>().second + .get<logfmt::parser::unquoted_value>().uv_value == "-"); + + auto pair2 = p.step(); + + CHECK(pair2.is<logfmt::parser::kvpair>()); + CHECK(pair2.get<logfmt::parser::kvpair>().first == "bf2"); + CHECK(pair2.get<logfmt::parser::kvpair>().second + .get<logfmt::parser::unquoted_value>().uv_value == "-1.2e"); + + auto pair3 = p.step(); + + CHECK(pair3.is<logfmt::parser::kvpair>()); + CHECK(pair3.get<logfmt::parser::kvpair>().first == "bf3"); + CHECK(pair3.get<logfmt::parser::kvpair>().second + .get<logfmt::parser::unquoted_value>().uv_value == "1.2.3"); + + auto pair4 = p.step(); + + CHECK(pair4.is<logfmt::parser::kvpair>()); + CHECK(pair4.get<logfmt::parser::kvpair>().first == "bf4"); + CHECK(pair4.get<logfmt::parser::kvpair>().second + .get<logfmt::parser::unquoted_value>().uv_value == "1e2e4"); +} + +TEST_CASE("non-terminated string") +{ + static const char *line = "abc=\"12 2"; + + auto p = logfmt::parser{string_fragment{line}}; + auto pair1 = p.step(); + + CHECK(pair1.is<logfmt::parser::error>()); + CHECK(pair1.get<logfmt::parser::error>().e_offset == 9); + CHECK(pair1.get<logfmt::parser::error>().e_msg == "non-terminated string"); +} + +TEST_CASE("missing equals") +{ + static const char *line = "abc"; + + auto p = logfmt::parser{string_fragment{line}}; + auto pair1 = p.step(); + + CHECK(pair1.is<logfmt::parser::error>()); + CHECK(pair1.get<logfmt::parser::error>().e_offset == 3); + CHECK(pair1.get<logfmt::parser::error>().e_msg == "expecting '='"); +} + +TEST_CASE("missing key") +{ + static const char *line = "=def"; + + auto p = logfmt::parser{string_fragment{line}}; + auto pair1 = p.step(); + + CHECK(pair1.is<logfmt::parser::error>()); + CHECK(pair1.get<logfmt::parser::error>().e_offset == 0); + CHECK(pair1.get<logfmt::parser::error>().e_msg == "expecting key followed by '='"); +} + +TEST_CASE("empty") +{ + static const char *line = ""; + + auto p = logfmt::parser{string_fragment{line}}; + auto pair1 = p.step(); + + CHECK(pair1.is<logfmt::parser::end_of_input>()); +} diff --git a/src/formats/openam_log.json b/src/formats/openam_log.json new file mode 100644 index 0000000..c1a8090 --- /dev/null +++ b/src/formats/openam_log.json @@ -0,0 +1,73 @@ +{ + "$schema": "https://lnav.org/schemas/format-v1.schema.json", + "openam_log": { + "title": "OpenAM Log", + "description": "The OpenAM identity provider.", + "url": "http://openam.forgerock.org", + "level-field": "level", + "level": { + "error": "ERROR", + "warning": "WARNING", + "info": "INFO", + "critical": "SEVERE", + "trace": "FINE|FINEST" + }, + "multiline": false, + "regex": { + "std": { + "pattern": "^\"(?<timestamp>\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2})\"\\s+(?<data>[^ \"]+|\"(?:[^\"]*|\"\")*\")\\s+(?<loginid>[^ \"]+|\"(?:[^\"]*|\"\")*\")\\s+(?<contextid>[^ \"]+|\"(?:[^\"]|\"\")*\")\\s+(?<ipaddr>[^ \"]+|\"(?:[^\"]|\"\")*\")\\s+(?<level>[^ \"]+|\"(?:[^\"]|\"\")*\")\\s+(?<domain>[^ \"]+|\"(?:[^\"]|\"\")*\")\\s+(?<loggedby>[^ \"]+|\"(?:[^\"]|\"\")*\")\\s+(?<messageid>[^ \"]+|\"(?:[^\"]|\"\")*\")\\s+(?<modulename>[^ \"]+|\"(?:[^\"]|\"\")*\")\\s+(?<nameid>[^ \"]+|\"(?:[^\"]|\"\")*\")\\s+(?<hostname>[^ \"]+|\"(?:[^\"]|\"\")*\")(?<body>.*)$" + } + }, + "value": { + "data": { + "kind": "quoted" + }, + "loginid": { + "kind": "quoted", + "identifier": true + }, + "contextid": { + "kind": "quoted", + "identifier": true + }, + "ipaddr": { + "kind": "quoted", + "identifier": true, + "collate": "ipaddress" + }, + "domain": { + "kind": "quoted", + "identifier": true + }, + "loggedby": { + "kind": "quoted", + "identifier": true + }, + "messageid": { + "kind": "quoted", + "identifier": true + }, + "modulename": { + "kind": "quoted", + "identifier": true + }, + "nameid": { + "kind": "quoted", + "identifier": true + }, + "hostname": { + "kind": "quoted", + "identifier": true, + "collate": "ipaddress" + } + }, + "sample": [ + { + "line": "\"2014-06-14 17:08:39\" \"http://localhost:8086|/|<samlp:AuthnRequest ID=\"\"139a40bba4d340108d91022750c2a3a8\"\" Version=\"\"2.0\"\" IssueInstant=\"\"2014-06-14T17:09:04Z\"\" ProtocolBinding=\"\"urn:oasis:names:tc:SAML:2.0:bindings:HTTP-POST\"\" AssertionConsumerServiceURL=\"\"http://localhost:8086/api/1/rest/admin/org/530e42ccd6f45fd16d0d0717/saml/consume\"\">\\n<saml:Issuer>http://localhost:8086</saml:Issuer>\\n<samlp:NameIDPolicy Format=\"\"urn:oasis:names:tc:SAML:1.1:nameid-format:emailAddress\"\" AllowCreate=\"\"true\"\"></samlp:NameIDPolicy>\\n<samlp:RequestedAuthnContext Comparison=\"\"exact\"\"><saml:AuthnContextClassRef>urn:oasis:names:tc:SAML:2.0:ac:classes:PasswordProtectedTransport</saml:AuthnContextClassRef></samlp:RequestedAuthnContext>\\n</samlp:AuthnRequest>\" \"cn=dsameuser,ou=DSAME Users,dc=openam\" 8fc43a8f6a8c14101 \"Not Available\" INFO dc=openam \"cn=dsameuser,ou=DSAME Users,dc=openam\" SAML2-36 SAML2.access \"Not Available\" 127.0.1.1" + }, + { + "line": "\"2014-06-09 14:49:56\" /etc/openam/openam/log/ \"cn=dsameuser,ou=DSAME Users,dc=openam\" 3d956febb91fed31 \"Not Available\" INFO dc=openam \"cn=dsameuser,ou=DSAME Users,dc=openam\" LOG-1 amPolicy.access \"Not Available\" 127.0.1.1" + } + ] + } +}
\ No newline at end of file diff --git a/src/formats/openamdb_log.json b/src/formats/openamdb_log.json new file mode 100644 index 0000000..1d5ea37 --- /dev/null +++ b/src/formats/openamdb_log.json @@ -0,0 +1,21 @@ +{ + "$schema": "https://lnav.org/schemas/format-v1.schema.json", + "openamdb_log": { + "title": "OpenAM Debug Log", + "description": "Debug logs for the OpenAM identity provider.", + "url": "http://openam.forgerock.org", + "regex": { + "std": { + "pattern": "^(?<module>[\\w]+):(?<timestamp>\\d{2}/\\d{2}/\\d{4} \\d{2}:\\d{2}:\\d{2}:\\d{3} [AP]M \\w+): Thread\\[(?<thread>[^,]+,\\d+,[^,]+)\\]\\n?(?:\\*+|(?<body>.*))$" + } + }, + "sample": [ + { + "line": "amMonitoring:06/09/2014 02:49:59:447 PM UTC: Thread[http-80-1,5,main]\n**********************************************" + }, + { + "line": "amLog:06/09/2014 04:08:22:515 PM UTC: Thread[http-80-8,5,main]\nERROR: LogMessageProviderBase.createLogRecord: unable to locate message ID object for ATTEMPT_GET_METAALIAS" + } + ] + } +}
\ No newline at end of file diff --git a/src/formats/openstack_log.json b/src/formats/openstack_log.json new file mode 100644 index 0000000..4dc280f --- /dev/null +++ b/src/formats/openstack_log.json @@ -0,0 +1,65 @@ +{ + "$schema": "https://lnav.org/schemas/format-v1.schema.json", + "openstack_log": { + "title": "OpenStack log format", + "description": "The log format for the OpenStack log files", + "url": "http://docs.openstack.org/openstack-ops/content/logging_monitoring.html", + "regex": { + "std": { + "pattern": "^(?<timestamp>\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}.\\d{3}) (?<pid>\\d+) (?<level>\\w+) (?<logger>\\S+) \\[(?<tid>[^\\]]+)\\] (?<body>.*)" + }, + "mod-std": { + "module-format": true, + "pattern": "^(?<level>\\w+) (?<logger>\\S+) \\[(?<tid>[^\\]]+)\\] (?<body>.*)" + }, + "keystone": { + "pattern": "^[(](?<logger>[^)]+)[)]: (?<timestamp>\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2},\\d{3}) (?<level>\\w+) (?!\\()(?<body>.*)" + }, + "keystone-debug": { + "pattern": "^[(](?<logger>[^)]+)[)]: (?<timestamp>\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2},\\d{3}) (?<level>\\w+) [(](?<user>[^)]+)[)] (?<body>.*)" + } + }, + "timestamp-format": [ + "%Y-%m-%d %H:%M:%S.%L", + "%Y-%m-%d %H:%M:%S,%L" + ], + "level-field": "level", + "level": { + "critical": "CRITICAL", + "error": "ERROR", + "info": "INFO", + "warning": "WARNING", + "trace": "TRACE", + "debug": "DEBUG" + }, + "value": { + "tid": { + "kind": "string", + "identifier": true + }, + "pid": { + "kind": "string", + "identifier": true + }, + "logger": { + "kind": "string", + "identifier": true + }, + "user": { + "kind": "string", + "identifier": true + } + }, + "sample": [ + { + "line": "2014-10-28 10:42:22.772 23623 INFO neutron.wsgi [req-40743023-00ed-441c-9d0a-19b8167ea0ad None] 10.1.255.252 - - [28/Oct/2014 10:42:22] GET /v2.0/floatingips.json?fixed_ip_address=80.0.0.9&port_id=b4291e0e-a941-4663-9379-7af6471e983f HTTP/1.1 200 208 0.008971" + }, + { + "line": "(sqlalchemy.orm.mapper.Mapper): 2021-04-27 06:25:32,122 INFO (User|user) Identified primary key columns: ColumnSet([Column('id', String(length=64), table=<user>, primary_key=True, nullable=False)])" + }, + { + "line": "(sqlalchemy.pool.QueuePool): 2021-04-28 16:37:00,355 DEBUG Connection <pymysql.connections.Connection object at 0x7fd88717d0d0> being returned to pool" + } + ] + } +} diff --git a/src/formats/page_log.json b/src/formats/page_log.json new file mode 100644 index 0000000..0758b13 --- /dev/null +++ b/src/formats/page_log.json @@ -0,0 +1,67 @@ +{ + "$schema": "https://lnav.org/schemas/format-v1.schema.json", + "page_log": { + "title": "CUPS Page Log", + "description": "The CUPS server log of printed pages.", + "url": "http://www.cups.org/documentation.php/doc-1.7/ref-page_log.html", + "multiline": false, + "regex": { + "pre-1.7": { + "pattern": "^(?<printer>[\\w_\\-\\.]+) (?<username>[\\w\\.\\-]+) (?<job_id>\\d+) \\[(?<timestamp>[^\\]]+)\\] (?<page_number>total|\\d+) (?<num_copies>\\d+) (?<job_billing>[^ ]+) (?<job_originating_hostname>[\\w\\.:\\-]+)$" + }, + "1.7": { + "pattern": "^(?<printer>[\\w_\\-\\.]+) (?<username>[\\w\\.\\-]+) (?<job_id>\\d+) \\[(?<timestamp>[^\\]]+)\\] (?<page_number>total|\\d+) (?<num_copies>\\d+) (?<job_billing>[^ ]+) (?<job_originating_hostname>[\\w\\.:\\-]+) (?<job_name>.+) (?<media>[^ ]+) (?<sides>.+)(?<body>.*)$" + } + }, + "value": { + "printer": { + "kind": "string", + "identifier": true + }, + "username": { + "kind": "string", + "identifier": true + }, + "job_id": { + "kind": "integer", + "identifier": true, + "foreign-key": true + }, + "page_number": { + "kind": "string" + }, + "num_copies": { + "kind": "integer" + }, + "job_billing": { + "kind": "string", + "identifier": true + }, + "job_originating_hostname": { + "kind": "string", + "collate": "ipaddress", + "identifier": true + }, + "job_name": { + "kind": "string", + "identifier": true + }, + "media": { + "kind": "string", + "identifier": true + }, + "sides": { + "kind": "string", + "identifier": true + } + }, + "sample": [ + { + "line": "Photosmart_7520_series stack 11 [18/May/2013:13:21:15 -0700] total 0 - localhost 5615311548-159003235-tickets.pdf Letter one-sided" + }, + { + "line": "tec_IS2027 kurt 401 [22/Apr/2003:10:28:43 +0100] 1 3 #marketing 10.160.50.13" + } + ] + } +}
\ No newline at end of file diff --git a/src/formats/papertrail_log.json b/src/formats/papertrail_log.json new file mode 100644 index 0000000..b1a9d87 --- /dev/null +++ b/src/formats/papertrail_log.json @@ -0,0 +1,52 @@ +{ + "$schema": "https://lnav.org/schemas/format-v1.schema.json", + "papertrail_log": { + "title": "Papertrail Service", + "url": "https://papertrailapp.com/", + "description": "Log format for the papertrail log management service", + "json": true, + "hide-extra": true, + "file-pattern": "pt:.*", + "line-format": [ + { + "field": "display_received_at" + }, + " ", + { + "field": "hostname" + }, + " ", + { + "field": "program" + }, + ": ", + { + "field": "message" + } + ], + "level-field": "severity", + "level": { + "error": "Error", + "debug": "Debug", + "warning": "Warning", + "info": "Info(?:rmational)?|Notice", + "critical": "Crit(?:ical)?", + "fatal": "Emerg(?:ency)?|Alert" + }, + "timestamp-field": "generated_at", + "body-field": "message", + "value": { + "display_received_at": { + "kind": "string" + }, + "program": { + "kind": "string", + "identifier": true + }, + "hostname": { + "kind": "string", + "identifier": true + } + } + } +}
\ No newline at end of file diff --git a/src/formats/pcap_log.json b/src/formats/pcap_log.json new file mode 100644 index 0000000..a3192e1 --- /dev/null +++ b/src/formats/pcap_log.json @@ -0,0 +1,82 @@ +{ + "$schema": "https://lnav.org/schemas/format-v1.schema.json", + "pcap_log": { + "json": true, + "title": "Packet Capture", + "description": "Internal format for pcap files", + "mime-types": [ + "application/vnd.tcpdump.pcap" + ], + "multiline": false, + "convert-to-local-time": true, + "line-format": [ + { + "field": "time" + }, + " ", + { + "field": "source", + "min-width": 15, + "align": "right" + }, + " → ", + { + "field": "destination", + "min-width": 15, + "align": "left" + }, + " ", + { + "field": "protocol", + "min-width": 7, + "align": "left" + }, + " ", + { + "field": "length", + "min-width": 4, + "align": "right" + }, + " ", + { + "field": "info" + } + ], + "level": { + "warning": "^6291456$", + "error": "^8388608$" + }, + "timestamp-field": "time", + "level-pointer": "/_ws_expert__ws_expert_severity$", + "body-field": "info", + "hide-extra": true, + "value": { + "source": { + "kind": "string", + "foreign-key": true, + "collate": "ipaddress", + "identifier": true + }, + "destination": { + "kind": "string", + "foreign-key": true, + "collate": "ipaddress", + "identifier": true + }, + "protocol": { + "kind": "string", + "identifier": true + }, + "length": { + "kind": "integer" + }, + "info": { + "kind": "string" + }, + "layers": { + "kind": "json", + "hidden": true + } + } + } +}
\ No newline at end of file diff --git a/src/formats/procstate_log.json b/src/formats/procstate_log.json new file mode 100644 index 0000000..74332dd --- /dev/null +++ b/src/formats/procstate_log.json @@ -0,0 +1,22 @@ +{ + "$schema": "https://lnav.org/schemas/format-v1.schema.json", + "procstate_log": { + "title": "Process State", + "description": "Periodic dumps of process state", + "regex": { + "std": { + "pattern": "========== Start of system state dump at (?<timestamp>[^=]+)==========(?<body>.*)" + } + }, + "sample": [ + { + "line": "========== Start of system state dump at Thu Jun 2 00:01:01 UTC 2022 ==========" + } + ], + "search-table": { + "procstate_procs": { + "pattern": "^(?<user>\\S+)\\s+(?<pid>\\d+)\\s+(?<cpu_pct>\\d+(?:\\.\\d+)?)\\s+(?<mem_pct>\\d+(?:\\.\\d+)?)\\s+(?<vsz>\\d+)\\s+(?<rss>\\d+)\\s(?<tty>\\S+)\\s+(?<stat>\\S+)\\s+(?<start_time>\\S+)\\s+(?<cpu_time>\\S+)\\s+(?<cmd>(?<cmd_name>[^ \\n]+)(?: (?<cmd_args>[^\\n]+))?)$" + } + } + } +}
\ No newline at end of file diff --git a/src/formats/s3_log.json b/src/formats/s3_log.json new file mode 100644 index 0000000..1472f87 --- /dev/null +++ b/src/formats/s3_log.json @@ -0,0 +1,158 @@ +{ + "$schema": "https://lnav.org/schemas/format-v1.schema.json", + "s3_log": { + "title": "S3 Access Log", + "description": "S3 server access log format", + "url": "https://docs.aws.amazon.com/AmazonS3/latest/dev/LogFormat.html", + "multiline": false, + "regex": { + "std": { + "pattern": "^(?<owner>\\S+)\\s+(?<bucket>\\S+)\\s+\\[(?<timestamp>[^\\]]+)\\]\\s+(?<c_ip>[\\w*.:-]+)\\s+(?<cs_userid>\\S+)\\s+(?<req_id>\\S+)\\s+(?<op>\\S+)\\s+(?<cs_key>\\S+)\\s+\"(?<cs_method>\\S+)\\s+(?<cs_uri_stem>[^ \\?]+)(?:\\?(?<cs_uri_query>[^ ]*))?\\s+(?<cs_version>\\S+)\"\\s+(?<sc_status>\\d+|-)\\s+(?<sc_error_code>\\S+)\\s+(?<sc_bytes>\\d+|-)\\s+(?<obj_size>\\d+|-)\\s+(?<total_time>\\d+|-)\\s+(?<turn_around_time>\\d+|-)\\s+\"(?<cs_referer>.*?)\"\\s+\"(?<cs_user_agent>.*?)\"$" + }, + "std-v2": { + "pattern": "^(?<owner>\\S+)\\s+(?<bucket>\\S+)\\s+\\[(?<timestamp>[^\\]]+)\\]\\s+(?<c_ip>[\\w*.:-]+)\\s+(?<cs_userid>\\S+)\\s+(?<req_id>\\S+)\\s+(?<op>\\S+)\\s+(?<cs_key>\\S+)\\s+\"(?<cs_method>\\S+)\\s+(?<cs_uri_stem>[^ \\?]+)(?:\\?(?<cs_uri_query>[^ ]*))?\\s+(?<cs_version>\\S+)\"\\s+(?<sc_status>\\d+|-)\\s+(?<sc_error_code>\\S+)\\s+(?<sc_bytes>\\d+|-)\\s+(?<obj_size>\\d+|-)\\s+(?<total_time>\\d+|-)\\s+(?<turn_around_time>\\d+|-)\\s+\"(?<cs_referer>.*?)\"\\s+\"(?<cs_user_agent>.*?)\"\\s+(?<version_id>\\S+)\\s+(?<host_id>\\S+)\\s+(?<sig_version>\\S+)\\s+(?<cipher_suite>\\S+)\\s+(?<auth_type>\\S+)\\s+(?<cs_host>\\S+)\\s+(?<tls_version>\\S+)$" + } + }, + "level-field": "sc_status", + "level": { + "error": "^[^123].*" + }, + "opid-field": "c_ip", + "value": { + "owner": { + "kind": "string", + "identifier": true, + "description": "The bucket owner" + }, + "bucket": { + "kind": "string", + "identifier": true, + "description": "The bucket" + }, + "c_ip": { + "kind": "string", + "collate": "ipaddress", + "identifier": true, + "description": "The client IP address" + }, + "cs_userid": { + "kind": "string", + "identifier": true, + "description": "The user ID passed from the client to the server" + }, + "req_id": { + "kind": "string", + "description": "The request ID" + }, + "op": { + "kind": "string", + "identifier": true, + "description": "The operation" + }, + "cs_key": { + "kind": "string", + "identifier": true, + "description": "The key for the bucket" + }, + "cs_method": { + "kind": "string", + "identifier": true, + "description": "The request method" + }, + "cs_uri_stem": { + "kind": "string", + "identifier": true, + "description": "The path part of the request URI" + }, + "cs_uri_query": { + "kind": "string", + "description": "The query parameters in the request URI" + }, + "cs_version": { + "kind": "string", + "identifier": true, + "description": "The client's HTTP version" + }, + "sc_status": { + "kind": "integer", + "foreign-key": true, + "rewriter": ";SELECT :sc_status || ' (' || (SELECT message FROM http_status_codes WHERE status = :sc_status) || ') '", + "description": "The status code returned by the server" + }, + "sc_error_code": { + "kind": "string", + "identifier": true, + "description": "The Amazon S3 error code" + }, + "sc_bytes": { + "kind": "integer", + "description": "The number of bytes returned by the server" + }, + "obj_size": { + "kind": "integer", + "description": "The size of the object" + }, + "total_time": { + "kind": "integer", + "description": "The total time taken to satisfy the request" + }, + "turn_around_time": { + "kind": "integer", + "description": "The turn around time" + }, + "cs_referer": { + "kind": "string", + "identifier": true, + "description": "The client's referrer" + }, + "cs_user_agent": { + "kind": "string", + "identifier": true, + "description": "The client's HTTP agent" + }, + "version_id": { + "kind": "string", + "identifier": true, + "description": "The version ID" + }, + "host_id": { + "kind": "string", + "identifier": true, + "description": "The host ID" + }, + "sig_version": { + "kind": "string", + "identifier": true, + "description": "The signature version" + }, + "cipher_suite": { + "kind": "string", + "identifier": true, + "description": "The SSL layer negotiated cipher suite" + }, + "auth_type": { + "kind": "string", + "identifier": true, + "description": "The type of request authentication used" + }, + "cs_host": { + "kind": "string", + "identifier": true, + "description": "The endpoint used to connect to S3" + }, + "tls_version": { + "kind": "string", + "identifier": true, + "description": "The TLS version negotiated by the client" + } + }, + "sample": [ + { + "line": "b659b576cff1e15e4c0313ff8930fba9f53e6794567f5c60dab3abf2f8dfb6cc www.example.com [10/Feb/2012:16:42:07 -0500] 1.2.3.4 arn:aws:iam::179580289999:user/phillip.boss EB3502676500C6BE WEBSITE.GET.OBJECT index \"GET /index HTTP/1.1\" 200 - 368 368 10 9 \"-\" \"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11\"" + }, + { + "line": "79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be awsexamplebucket1 [06/Feb/2019:00:00:38 +0000] 192.0.2.3 79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be 3E57427F3EXAMPLE REST.GET.VERSIONING - \"GET /awsexamplebucket1?versioning HTTP/1.1\" 200 - 113 - 7 - \"-\" \"S3Console/0.4\" - s9lzHYrFp76ZVxRcpX9+5cjAnEH2ROuNkd2BHfIa6UkFVdtjf5mKR3/eTPFvsiP/XV/VLi31234= SigV2 ECDHE-RSA-AES128-GCM-SHA256 AuthHeader awsexamplebucket1.s3.us-west-1.amazonaws.com TLSV1.1" + } + ] + } +}
\ No newline at end of file diff --git a/src/formats/snaplogic_log.json b/src/formats/snaplogic_log.json new file mode 100644 index 0000000..db49c69 --- /dev/null +++ b/src/formats/snaplogic_log.json @@ -0,0 +1,55 @@ +{ + "$schema": "https://lnav.org/schemas/format-v1.schema.json", + "snaplogic_log": { + "title": "SnapLogic Server Log", + "description": "The SnapLogic server log format.", + "url": "http://www.snaplogic.com/docs/user-guide/user-guide.htm", + "regex": { + "std": { + "pattern": "^(?<timestamp>\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}(?:\\.\\d{3})?) (?:(?:(?<level>\\w{4,}) (?<logger>[^ ]+) (?<facility>[^ ]+) (?<msgid>[^ ]+) (?<pipe_rid>-|\\d+)(?:\\.(?<comp_rid>[^ ]+))? (?<resource_name>[^ ]+) (?<invoker>[^ ]+))|(?:(?:stdout|stderr): ))(?<body>.*)" + } + }, + "level-field": "level", + "level": { + "error": "ERROR", + "debug": "DEBUG", + "info": "INFO", + "warning": "WARNING" + }, + "value": { + "logger": { + "kind": "string", + "identifier": true + }, + "facility": { + "kind": "string", + "identifier": true + }, + "msgid": { + "kind": "string", + "identifier": true + }, + "pipe_rid": { + "kind": "string", + "identifier": true + }, + "comp_rid": { + "kind": "string", + "identifier": true + }, + "resource_name": { + "kind": "string", + "identifier": true + }, + "invoker": { + "kind": "string", + "identifier": true + } + }, + "sample": [ + { + "line": "2013-07-30T09:40:25 DEBUG main_process.main PM - 1768839331504132353247612213662950165988626018 - - Pipeline manager '' sending to Leads. Invoker 'admin': PREPARE {'parent_rid': '1768839331504132353247612213662950165988626018', 'resource_name': u'Leads', 'input_views': {}, 'parameters': {u'DELIMITER': u',', u'INPUTFILE': u'file://tutorial/data/leads.csv'}, 'output_views': {u'Output1': {'method': 'GET'}}, 'context_name': u'', 'snap_control_version': '1.2'}" + } + ] + } +}
\ No newline at end of file diff --git a/src/formats/sssd_log.json b/src/formats/sssd_log.json new file mode 100644 index 0000000..04f3a77 --- /dev/null +++ b/src/formats/sssd_log.json @@ -0,0 +1,38 @@ +{ + "$schema": "https://lnav.org/schemas/format-v1.schema.json", + "sssd_log": { + "title": "SSSD log format", + "description": "Log format used by the System Security Services Daemon", + "url": "http://fedorahosted.org/sssd", + "regex": { + "core": { + "pattern": "^\\((?<timestamp>\\S{3,8} \\S{3,8} ( \\d|\\d{2}) \\d{2}:\\d{2}:\\d{2} \\d{4})\\) \\[sssd\\] \\[(?<function>\\w+)\\] \\((?<debug_level>0x[0-9a-fA-F]{4})\\): (?<body>.*)$" + }, + "module": { + "pattern": "^\\((?<timestamp>\\S{3,8} \\S{3,8} ( \\d|\\d{2}) \\d{2}:\\d{2}:\\d{2} \\d{4})\\) \\[sssd(?<module>\\[.*?\\])\\] \\[(?<function>\\w+)\\] \\((?<debug_level>0x[0-9a-fA-F]{4})\\): (?<body>.*)$" + } + }, + "value": { + "module": { + "kind": "string" + }, + "function": { + "kind": "string" + }, + "debug_level": { + "kind": "string" + }, + "body": { + "kind": "string" + } + }, + "sample": [ + { + "line": "(Tue Mar 31 06:03:46 2015) [sssd[be[default]]] [sysdb_search_by_name] (0x0400): No such entry" + }, + { + "line": "(Tue Mar 31 05:58:38 2015) [sssd] [start_service] (0x0100): Queueing service LDAP for startup" + } + ] + } +}
\ No newline at end of file diff --git a/src/formats/strace_log.json b/src/formats/strace_log.json new file mode 100644 index 0000000..46c3a44 --- /dev/null +++ b/src/formats/strace_log.json @@ -0,0 +1,44 @@ +{ + "$schema": "https://lnav.org/schemas/format-v1.schema.json", + "strace_log": { + "title": "Strace", + "description": "The strace output format.", + "url": "http://en.wikipedia.org/wiki/Strace", + "multiline": false, + "regex": { + "std": { + "pattern": "^(?<timestamp>\\d{2}:\\d{2}:\\d{2}\\.\\d{6}) (?<syscall>\\w+)\\((?<body>.*)\\)\\s+=\\s+(?<rc>[-\\w]+)(?: (?<errno>\\w+) \\([^\\)]+\\))?(?: <(?<duration>\\d+\\.\\d+)>)?$" + } + }, + "level-field": "errno", + "level": { + "error": ".+" + }, + "value": { + "syscall": { + "kind": "string", + "identifier": true, + "rewriter": ":pipe-line-to explain-syscall.sh ${syscall}" + }, + "rc": { + "kind": "integer", + "foreign-key": true + }, + "duration": { + "kind": "float" + }, + "errno": { + "kind": "string", + "identifier": true + } + }, + "sample": [ + { + "line": "08:09:33.814936 execve(\"/bin/ls\", [\"ls\"], [/* 38 vars */]) = 0 <0.000264>" + }, + { + "line": "08:09:33.815943 access(\"/etc/ld.so.nohwcap\", F_OK) = -1 ENOENT (No such file or directory) <0.000019>" + } + ] + } +}
\ No newline at end of file diff --git a/src/formats/sudo_log.json b/src/formats/sudo_log.json new file mode 100644 index 0000000..d2ee72c --- /dev/null +++ b/src/formats/sudo_log.json @@ -0,0 +1,48 @@ +{ + "$schema": "https://lnav.org/schemas/format-v1.schema.json", + "sudo_log": { + "title": "sudo", + "description": "The sudo privilege management tool.", + "url": "", + "regex": { + "std": { + "module-format": true, + "pattern": "^(?<login>\\S+)\\s*: (?:(?<error_msg>[^;]+);)?\\s*TTY=(?<tty>[^;]+)\\s+;\\s*PWD=(?<pwd>[^;]+)\\s+;\\s*USER=(?<user>[^;]+)\\s+;\\s*COMMAND=(?<command>.*)$" + } + }, + "level-field": "error_msg", + "level": { + "error": ".+" + }, + "value": { + "login": { + "kind": "string", + "identifier": true + }, + "error_msg": { + "kind": "string" + }, + "tty": { + "kind": "string" + }, + "pwd": { + "kind": "string" + }, + "user": { + "kind": "string", + "identifier": true + }, + "command": { + "kind": "string" + } + }, + "sample": [ + { + "line": "stack : 3 incorrect password attempts ; TTY=ttys005 ; PWD=/Users/stack/ClionProjects/lbuild ; USER=root ; COMMAND=/bin/ls" + }, + { + "line": "stack : TTY=ttys005 ; PWD=/Users/stack/ClionProjects/lbuild ; USER=root ; COMMAND=/bin/ls" + } + ] + } +}
\ No newline at end of file diff --git a/src/formats/syslog_log.json b/src/formats/syslog_log.json new file mode 100644 index 0000000..9207f74 --- /dev/null +++ b/src/formats/syslog_log.json @@ -0,0 +1,99 @@ +{ + "$schema": "https://lnav.org/schemas/format-v1.schema.json", + "syslog_log": { + "title": "Syslog", + "description": "The system logger format found on most posix systems.", + "url": "http://en.wikipedia.org/wiki/Syslog", + "regex": { + "std": { + "pattern": "^(?<timestamp>(?:\\S{3,8}\\s+\\d{1,2} \\d{2}:\\d{2}:\\d{2}|\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}(?:\\.\\d{3,6})?(?:Z|(?:\\+|-)\\d{2}:\\d{2})))(?: (?<log_hostname>[a-zA-Z0-9:][^ ]+[a-zA-Z0-9]))?(?: \\[CLOUDINIT\\])?(?:(?: syslogd [\\d\\.]+|(?: (?<log_syslog_tag>(?<log_procname>(?:[^\\[: ]+|[^ :]+))(?:\\[(?<log_pid>\\d+)\\](?: \\([^\\)]+\\))?)?))):\\s*(?<body>.*)$|:?(?:(?: ---)? last message repeated \\d+ times?(?: ---)?))" + }, + "rfc5424": { + "pattern": "^<(?<log_pri>\\d+)>(?<syslog_version>\\d+) (?<timestamp>\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}(?:\\.\\d{6})?(?:[^ ]+)?) (?<log_hostname>[^ ]+|-) (?<log_syslog_tag>(?<log_procname>[^ ]+|-) (?<log_pid>[^ ]+|-) (?<log_msgid>[^ ]+|-)) (?<log_struct>\\[(?:[^\\]\"]|\"(?:\\.|[^\"])+\")*\\]|-|)\\s+(?<body>.*)" + } + }, + "level-field": "body", + "level": { + "error": "(?:(?:(?<![a-zA-Z]))(?:(?i)error(?:s)?)(?:(?![a-zA-Z]))|failed|failure)", + "warning": "(?:(?:(?i)warn)|not responding|init: cannot execute)" + }, + "opid-field": "log_syslog_tag", + "multiline": true, + "module-field": "log_procname", + "value": { + "log_pri": { + "kind": "integer", + "foreign-key": true, + "description": "The priority level of the message" + }, + "syslog_version": { + "kind": "integer", + "foreign-key": true, + "description": "The version of the syslog format used for this message" + }, + "log_hostname": { + "kind": "string", + "collate": "ipaddress", + "identifier": true, + "description": "The name of the host that generated the message" + }, + "log_procname": { + "kind": "string", + "identifier": true, + "description": "The name of the process that generated the message" + }, + "log_pid": { + "kind": "string", + "identifier": true, + "action-list": [ + "dump_pid" + ], + "description": "The ID of the process that generated the message" + }, + "log_syslog_tag": { + "kind": "string", + "identifier": true, + "description": "The combination of the procname and pid" + }, + "log_msgid": { + "kind": "string", + "identifier": true + }, + "log_struct": { + "kind": "struct" + } + }, + "action": { + "dump_pid": { + "label": "Show Process Info", + "capture-output": true, + "cmd": [ + "dump-pid.sh" + ] + } + }, + "sample": [ + { + "line": "Apr 28 04:02:03 tstack-centos5 syslogd 1.4.1: restart." + }, + { + "line": "Jun 27 01:47:20 Tims-MacBook-Air.local configd[17]: network changed: v4(en0-:192.168.1.8) DNS- Proxy- SMB" + }, + { + "line": "Jun 20 17:26:13 ip-10-188-149-5 [CLOUDINIT] util.py[DEBUG]: Restoring selinux mode for /var/lib/cloud (recursive=False)" + }, + { + "line": "<46>1 2017-04-27T07:50:47.381967+02:00 logserver rsyslogd - - [origin software=\"rsyslogd\" swVersion=\"8.4.2\" x-pid=\"900\" x-info=\"http://www.rsyslog.com\"] start" + }, + { + "line": "<30>1 2017-04-27T07:59:12+02:00 nextcloud dhclient - - - DHCPREQUEST on eth0 to 192.168.1.1 port 67" + }, + { + "line": "<78>1 2017-04-27T08:09:01+02:00 nextcloud CRON 1472 - - (root) CMD ( [ -x /usr/lib/php5/sessionclean ] && /usr/lib/php5/sessionclean)" + }, + { + "line": "Aug 1 00:00:03 Tim-Stacks-iMac com.apple.xpc.launchd[1] (com.apple.mdworker.shared.0C000000-0700-0000-0000-000000000000[50989]): Service exited due to SIGKILL | sent by mds[198]" + } + ] + } +}
\ No newline at end of file diff --git a/src/formats/tcf_log.json b/src/formats/tcf_log.json new file mode 100644 index 0000000..ad42d78 --- /dev/null +++ b/src/formats/tcf_log.json @@ -0,0 +1,51 @@ +{ + "$schema": "https://lnav.org/schemas/format-v1.schema.json", + "tcf_log": { + "title": "TCF Log", + "description": "Target Communication Framework log", + "url": [ + "http://wiki.eclipse.org/TCF", + "http://git.eclipse.org/c/tcf/org.eclipse.tcf.git/tree/target_explorer/plugins/org.eclipse.tm.te.tcf.log.core/src/org/eclipse/tm/te/tcf/log/core/internal/listener/ChannelTraceListener.java?id=b6e81bb8405f99dda2764b22cff876fa00f734f5#n144" + ], + "regex": { + "std": { + "pattern": "^TCF (?<timestamp>\\d{2}:\\d{2}.\\d{3}): (?:Server-Properties: (?:.*)|channel server|\\w+: (?<dir>--->|<---) (?<type>\\w)(?: (?<token>\\w+))?(?: (?<service>\\w+))?(?: (?<name>\\w+))?(?: (?<msg>.*))?(?: <eom>))(?<body>.*)$" + } + }, + "value": { + "dir": { + "kind": "string" + }, + "type": { + "kind": "string", + "identifier": true + }, + "token": { + "kind": "string", + "identifier": true + }, + "service": { + "kind": "string", + "identifier": true + }, + "name": { + "kind": "string", + "identifier": true + }, + "msg": { + "kind": "json" + } + }, + "sample": [ + { + "line": "TCF 29:47.191: Server-Properties: {\"Name\":\"TCF Protocol Logger\",\"OSName\":\"Linux 3.2.0-60-generic\",\"UserName\":\"xavier\",\"AgentID\":\"1fde3dd1-d4be-4f79-8090-6f8d212f03bf\",\"TransportName\":\"TCP\",\"Proxy\":\"\",\"ValueAdd\":\"1\",\"Port\":\"1534\"}" + }, + { + "line": "TCF 30:11.475: 0: <--- R 2 [\"P1\"] <eom>" + }, + { + "line": "TCF 30:11.475: 0: ---> C 4 RunControl getChildren \"P1\" <eom>" + } + ] + } +}
\ No newline at end of file diff --git a/src/formats/tcsh_history.json b/src/formats/tcsh_history.json new file mode 100644 index 0000000..86f7cbd --- /dev/null +++ b/src/formats/tcsh_history.json @@ -0,0 +1,18 @@ +{ + "$schema": "https://lnav.org/schemas/format-v1.schema.json", + "tcsh_history": { + "title": "TCSH History", + "description": "The tcsh history file format.", + "convert-to-local-time": true, + "regex": { + "std": { + "pattern": "^#(?<timestamp>\\+\\d+)\\n?(?<body>.*)?$" + } + }, + "sample": [ + { + "line": "#+1375138067\necho HELLO=BAR" + } + ] + } +}
\ No newline at end of file diff --git a/src/formats/unifi_iptables_log.json b/src/formats/unifi_iptables_log.json new file mode 100644 index 0000000..907266a --- /dev/null +++ b/src/formats/unifi_iptables_log.json @@ -0,0 +1,154 @@ +{ + "$schema": "https://lnav.org/schemas/format-v1.schema.json", + "unifi_iptables_log": { + "title": "UniFi iptables log", + "description": "The UniFi gateway iptables logger format (for /var/log/iptables).", + "url": "https://www.halolinux.us/firewalls/firewall-log-messages-what-do-they-mean.html", + "regex": { + "kernel-udp": { + "pattern": "^(?<timestamp>[A-Z][a-z]{2}\\s+\\d+\\s+\\d+:\\d+:\\d+) (?<host>[^\\s]+)\\s(?:\\[(?<rule_name>[^\\]]+)\\]\\s*)?(?:ALIEN BLOCK: )?DESCR=\"(?<DESCR>.*?)\"?\\sIN=(?<IP_IN>(?:\\d|\\w)*) OUT=(?<IP_OUT>(?:\\d|\\w)*) MAC=(?:(?<MAC>(?:[0-9a-f]{2}:){5}[0-9a-f]{2})(?::(?<MAC_SRC>[^\\s]+)))? SRC=(?<SRC>(?:[\\d\\.])+) DST=(?<DST>(?:[\\d\\.])+) LEN=(?<LEN>(?:\\d+)) TOS=(?<TOS>(?:[0-9A-F])+) PREC=(?<PREC>0x(?:[0-9A-F])+) TTL=(?<TTL>\\d+) ID=(?<ID>\\d+) ((?<DF>(?:DF)) )?PROTO=(?<PROTO>UDP) SPT=(?<SPT>\\d+) DPT=(?<DPT>\\d+) LEN=(?<LEN_UDP>\\d+)\\s*(?<body>.*)$" + }, + "kernel-tcp": { + "pattern": "^(?<timestamp>[A-Z][a-z]{2}\\s+\\d+\\s+\\d+:\\d+:\\d+) (?<host>[^\\s]+)\\s(?:\\[(?<rule_name>[^\\]]+)\\]\\s*)?(?:ALIEN BLOCK: )?DESCR=\"(?<DESCR>.*?)\"?\\sIN=(?<IP_IN>(?:\\d|\\w)*) OUT=(?<IP_OUT>(?:\\d|\\w)*) MAC=(?:(?<MAC>(?:[0-9a-f]{2}:){5}[0-9a-f]{2})(?::(?<MAC_SRC>[^\\s]+))) SRC=(?<SRC>(?:[\\d\\.])+) DST=(?<DST>(?:[\\d\\.])+) LEN=(?<LEN>(?:\\d+)) TOS=(?<TOS>(?:[0-9A-F])+) PREC=(?<PREC>0x(?:[0-9A-F])+) TTL=(?<TTL>\\d+) ID=(?<ID>\\d+) ((?<DF>(?:DF)) )?PROTO=(?<PROTO>TCP) SPT=(?<SPT>\\d+) DPT=(?<DPT>\\d+) SEQ=(?<SEQ>\\d+) ACK=(?<ACK>\\d+) WINDOW=(?<WINDOW>\\d+) (?<body>.*)$" + }, + "kernel-other-proto": { + "pattern": "^(?<timestamp>[A-Z][a-z]{2}\\s+\\d+\\s+\\d+:\\d+:\\d+) (?<host>[^\\s]+)\\s(?:\\[(?<rule_name>[^\\]]+)\\]\\s*)?(?:ALIEN BLOCK: )?DESCR=\"(?<DESCR>.*?)\"?\\sIN=(?<IP_IN>(?:\\d|\\w)*) OUT=(?<IP_OUT>(?:\\d|\\w)*) MAC=(?:(?<MAC>(?:[0-9a-f]{2}:){5}[0-9a-f]{2})(?::(?<MAC_SRC>[^\\s]+)))? SRC=(?<SRC>(?:[\\d\\.])+) DST=(?<DST>(?:[\\d\\.])+) LEN=(?<LEN>(?:\\d+)) TOS=(?<TOS>(?:[0-9A-F])+) PREC=(?<PREC>0x(?:[0-9A-F])+) TTL=(?<TTL>\\d+) ID=(?<ID>\\d+) ((?<DF>(?:DF)) )?PROTO=(?<PROTO>(?!TCP|UDP)(?:\\w+))(?<body>.*)$" + } + }, + "opid-field": "SEQ", + "multiline": false, + "timestamp-format": [ + "%b %d %H:%M:%S" + ], + "value": { + "host" : { + "kind": "string", + "identifier": true, + "hidden": true + }, + "rule_name" : { + "kind": "string", + "identifier": true + }, + "DESCR" : { + "kind": "string", + "identifier": false + }, + "dhcp_op" : { + "kind": "string", + "identifier": true + }, + "dhcp_mac" : { + "kind": "string", + "identifier": true + }, + "dhcp_iface" : { + "kind": "string" + }, + "dhcp_ip" : { + "kind": "string", + "collate": "ipaddress", + "identifier": true + }, + "SEQ" : { + "kind": "integer", + "identifier": true + }, + "IP_IN" : { + "kind": "string", + "identifier": false + }, + "IP_OUT" : { + "kind": "string", + "identifier": false + }, + "MAC" : { + "kind": "string", + "identifier": true, + "hidden": true + }, + "MAC_SRC" : { + "kind": "string", + "identifier": false, + "hidden": true + }, + "SRC" : { + "kind": "string", + "collate": "ipaddress", + "identifier": true + }, + "SPT" : { + "kind": "integer", + "identifier": true + }, + "DST" : { + "kind": "string", + "collate": "ipaddress", + "identifier": true + }, + "DPT" : { + "kind": "integer", + "identifier": true + }, + "LEN" : { + "kind": "integer" + }, + "TOS" : { + "kind": "string", + "hidden": true + }, + "PREC" : { + "kind": "string", + "hidden": true + }, + "TTL" : { + "kind": "integer", + "hidden": true + }, + "PROTO" : { + "kind": "string", + "identifier": true + }, + "LEN_UDP" : { + "kind": "integer" + }, + "WINDOW" : { + "kind": "integer", + "hidden": true + }, + "RES" : { + "kind": "string", + "hidden": true + }, + "SYN" : { + "kind": "string", + "hidden": true + }, + "URGP" : { + "kind": "integer", + "hidden": true + }, + "body" : { + "kind": "string" + } + }, + "highlights": { + "rule_name": { + "pattern": "(\\[.*-[DR]-.*\\])", + "color": "Red", + "underline": true + } + }, + "sample": [ + { + "line": "Apr 7 21:49:03 UDM-Pro [WAN_IN-RET-3006] DESCR=\"[WAN_IN] PortForward Allow [HTTPS (cli IN=eth8 OUT=br96 MAC=24:5a:4c:a2:b1:09:48:2c:d0:ab:93:3f:08:00 SRC=194.230.158.35 DST=192.168.96.10 LEN=60 TOS=00 PREC=0x00 TTL=49 ID=51880 DF PROTO=TCP SPT=38618 DPT=10443 SEQ=2560518888 ACK=0 WINDOW=65535 SYN URGP=0 MARK=0" + }, + { + "line": "Apr 7 23:24:35 UDM-Pro [PREROUTING-DNAT-13] DESCR=\"PortForward DNAT [HTTPS (client-cer IN=eth8 OUT= MAC=24:5a:4c:a2:b1:09:48:2c:d0:ab:93:3f:08:00 SRC=103.203.57.14 DST=192.168.10.2 LEN=40 TOS=00 PREC=0x00 TTL=241 ID=54321 PROTO=TCP SPT=43609 DPT=443 SEQ=3913455798 ACK=0 WINDOW=65535 SYN URGP=0 MARK=0" + }, + { + "line": "Apr 7 21:40:58 UDM-Pro [LAN_IN-D-4001] DESCR=\"🛑Drop all InterVLAN traffic\" IN=br96 OUT=tlprt0 MAC=24:5a:4c:a2:b1:0b:24:5e:be:46:df:c8:08:00 SRC=192.168.96.10 DST=192.168.2.1 LEN=40 TOS=00 PREC=0x00 TTL=63 ID=50760 DF PROTO=TCP SPT=8084 DPT=55504 SEQ=1967017102 ACK=3747652110 WINDOW=131 ACK FIN URGP=0 MARK=0" + } + ] + } +} diff --git a/src/formats/unifi_log.json b/src/formats/unifi_log.json new file mode 100644 index 0000000..369e753 --- /dev/null +++ b/src/formats/unifi_log.json @@ -0,0 +1,204 @@ +{ + "$schema": "https://lnav.org/schemas/format-v1.schema.json", + "unifi_log": { + "title": "UniFi log", + "description": "The UniFi gateway messages logger format (for /var/log/messages).", + "url": "https://www.halolinux.us/firewalls/firewall-log-messages-what-do-they-mean.html", + "regex": { + "kernel-udp": { + "pattern": "^(?<timestamp>[A-Z][a-z]{2}\\s+\\d+\\s+\\d+:\\d+:\\d+) (?<host>[^\\s]+) (?<facility>\\w+)\\.(?<level>\\w+) (?<module>kernel): \\[(?:\\s*(?<ellapsed>\\d+\\.\\d+))\\]\\s(?:\\[(?<rule_name>[^\\]]+)\\]\\s*)?(?:ALIEN BLOCK: )?IN=(?<IP_IN>(?:\\d|\\w)*) OUT=(?<IP_OUT>(?:\\d|\\w)*) MAC=(?:(?<MAC>(?:[0-9a-f]{2}:){5}[0-9a-f]{2})(?::(?<MAC_SRC>[^\\s]+)))? SRC=(?<SRC>(?:[\\d\\.])+) DST=(?<DST>(?:[\\d\\.])+) LEN=(?<LEN>(?:\\d+)) TOS=(?<TOS>0x(?:[0-9A-F])+) PREC=(?<PREC>0x(?:[0-9A-F])+) TTL=(?<TTL>\\d+) ID=(?<ID>\\d+) (?<DF>(?:DF) )?PROTO=(?<PROTO>UDP) SPT=(?<SPT>\\d+) DPT=(?<DPT>\\d+) LEN=(?<LEN_UDP>\\d+)\\s*(?<body>.*)$" + }, + "kernel-tcp": { + "pattern": "^(?<timestamp>[A-Z][a-z]{2}\\s+\\d+\\s+\\d+:\\d+:\\d+) (?<host>[^\\s]+) (?<facility>\\w+)\\.(?<level>\\w+) (?<module>kernel): \\[(?:\\s*(?<ellapsed>\\d+\\.\\d+))\\]\\s(?:\\[(?<rule_name>[^\\]]+)\\]\\s*)?(?:ALIEN BLOCK: )?IN=(?<IP_IN>(?:\\d|\\w)*) OUT=(?<IP_OUT>(?:\\d|\\w)*) MAC=(?:(?<MAC>(?:[0-9a-f]{2}:){5}[0-9a-f]{2})(?::(?<MAC_SRC>[^\\s]+))) SRC=(?<SRC>(?:[\\d\\.])+) DST=(?<DST>(?:[\\d\\.])+) LEN=(?<LEN>(?:\\d+)) TOS=(?<TOS>0x(?:[0-9A-F])+) PREC=(?<PREC>0x(?:[0-9A-F])+) TTL=(?<TTL>\\d+) ID=(?<ID>\\d+) (?<DF>(?:DF) )?PROTO=(?<PROTO>TCP) SPT=(?<SPT>\\d+) DPT=(?<DPT>\\d+) WINDOW=(?<WINDOW>\\d+) RES=(?<RES>0x(?:[0-9A-F])+) (?<SYN>(?:SYN) )?(?:URGP=(?<URGP>\\d+)\\s*)?(?<body>.*)$" + }, + "kernel-other-proto": { + "pattern": "^(?<timestamp>[A-Z][a-z]{2}\\s+\\d+\\s+\\d+:\\d+:\\d+) (?<host>[^\\s]+) (?<facility>\\w+)\\.(?<level>\\w+) (?<module>kernel): \\[(?:\\s*(?<ellapsed>\\d+\\.\\d+))\\]\\s(?:\\[(?<rule_name>[^\\]]+)\\]\\s*)?(?:ALIEN BLOCK: )?IN=(?<IP_IN>(?:\\d|\\w)*) OUT=(?<IP_OUT>(?:\\d|\\w)*) MAC=(?:(?<MAC>(?:[0-9a-f]{2}:){5}[0-9a-f]{2})(?::(?<MAC_SRC>[^\\s]+)))? SRC=(?<SRC>(?:[\\d\\.])+) DST=(?<DST>(?:[\\d\\.])+) LEN=(?<LEN>(?:\\d+)) TOS=(?<TOS>0x(?:[0-9A-F])+) PREC=(?<PREC>0x(?:[0-9A-F])+) TTL=(?<TTL>\\d+) ID=(?<ID>\\d+) (?<DF>(?:DF) )?PROTO=(?<PROTO>(?!TCP|UDP)(?:\\w+))(?<body>.*)$" + }, + "kernel-other": { + "pattern": "^(?<timestamp>[A-Z][a-z]{2}\\s+\\d+\\s+\\d+:\\d+:\\d+) (?<host>[^\\s]+) (?<facility>\\w+)\\.(?<level>\\w+) (?<module>kernel): (?:\\[(?:\\s*(?<ellapsed>\\d+\\.\\d+))\\]\\s)?(?!IN|ALIEN BLOCK)(?<body>[^\\[].*)$" + }, + "dnsmasq-dhcp": { + "pattern": "^(?<timestamp>[A-Z][a-z]{2}\\s+\\d+\\s+\\d+:\\d+:\\d+) (?<host>[^\\s]+) (?<facility>\\w+)\\.(?<level>\\w+) (?<module>dnsmasq-dhcp[A-Za-z0-9\\.\\-]*)(?:\\[(?<ID>\\d+)\\])?: (?<dhcp_op>DHCP[^(]+)(?:\\((?<dhcp_iface>[^)]*)\\)) (?:(?<dhcp_ip>(?:\\d{1,3}\\.){3}\\d{1,3}) )?(?<dhcp_mac>(?:[0-9a-f]{2}:)+[0-9a-f]{2})(?: (?<body>.*))?$" + }, + "other": { + "pattern": "^(?<timestamp>[A-Z][a-z]{2}\\s+\\d+\\s+\\d+:\\d+:\\d+) (?<host>[^\\s]+) (?<facility>\\w+)\\.(?<level>\\w+) (?<module>(?!kernel|dnsmasq-dhcp)[A-Za-z0-9\\.\\-]*)(?:\\[(?<ID>\\d+)\\])?: (?:\\[apply-config\\])?(?:\\[(?:\\s*(?<ellapsed>\\d+\\.\\d+))\\]\\s)?(?<body>.*)$" + } + }, + "level-field": "level", + "level": { + "critical": "emerg", + "error": "err", + "warning": "warn", + "info": "notice" + }, + "opid-field": "ID", + "multiline": false, + "module-field": "module", + "timestamp-format": [ + "%b %d %H:%M:%S" + ], + "value": { + "level": { + "kind": "string", + "identifier": true + }, + "facility": { + "kind": "string", + "identifier": false + }, + "module": { + "kind": "string", + "identifier": false + }, + "ellapsed": { + "kind": "float", + "identifier": false, + "hidden": true + }, + "rule_name": { + "kind": "string", + "identifier": true + }, + "host": { + "kind": "string", + "identifier": true, + "hidden": true + }, + "dhcp_op": { + "kind": "string", + "identifier": true + }, + "dhcp_mac": { + "kind": "string", + "identifier": true + }, + "dhcp_iface": { + "kind": "string" + }, + "dhcp_ip": { + "kind": "string", + "collate": "ipaddress", + "identifier": true + }, + "ID": { + "kind": "integer", + "identifier": true + }, + "IP_IN": { + "kind": "string", + "identifier": false + }, + "IP_OUT": { + "kind": "string", + "identifier": false + }, + "MAC": { + "kind": "string", + "identifier": true, + "hidden": true + }, + "MAC_SRC": { + "kind": "string", + "identifier": false, + "hidden": true + }, + "SRC": { + "kind": "string", + "collate": "ipaddress", + "identifier": true + }, + "SPT": { + "kind": "integer", + "identifier": true + }, + "DST": { + "kind": "string", + "collate": "ipaddress", + "identifier": true + }, + "DPT": { + "kind": "integer", + "identifier": true + }, + "LEN": { + "kind": "integer" + }, + "TOS": { + "kind": "string", + "hidden": true + }, + "PREC": { + "kind": "string", + "hidden": true + }, + "TTL": { + "kind": "integer", + "hidden": true + }, + "PROTO": { + "kind": "string", + "identifier": true + }, + "LEN_UDP": { + "kind": "integer" + }, + "WINDOW": { + "kind": "integer", + "hidden": true + }, + "RES": { + "kind": "string", + "hidden": true + }, + "SYN": { + "kind": "string", + "hidden": true + }, + "URGP": { + "kind": "integer", + "hidden": true + }, + "body": { + "kind": "string" + } + }, + "sample": [ + { + "line": "Mar 2 23:24:28 UDM-Pro user.warn kernel: [1293979.679369] IN=br46 OUT= MAC=24:5a:4c:a2:b1:0b:74:7a:90:9f:e4:ff:08:00 SRC=192.168.46.5 DST=8.8.8.8 LEN=68 TOS=0x00 PREC=0x00 TTL=255 ID=34103 DF PROTO=UDP SPT=65450 DPT=53 LEN=48" + }, + { + "line": "Mar 2 23:24:28 UDM-Pro user.warn kernel: [ 979.679369] [DNAT-br46-udp]IN=br46 OUT= MAC=24:5a:4c:a2:b1:0b:74:7a:90:9f:e4:ff:08:00 SRC=192.168.46.5 DST=8.8.8.8 LEN=68 TOS=0x00 PREC=0x00 TTL=255 ID=34103 DF PROTO=UDP SPT=65450 DPT=53 LEN=48" + }, + { + "line": "Mar 2 23:00:01 UDM-Pro user.warn kernel: [1293512.217894] [FW-A-LAN_LOCAL_U-2013]IN=br96 OUT= MAC=24:5a:4c:a2:b1:0b:24:5e:be:46:df:c8:08:00 SRC=192.168.96.10 DST=192.168.16.1 LEN=40 TOS=0x00 PREC=0x20 TTL=64 ID=44654 DF PROTO=TCP SPT=55144 DPT=22 WINDOW=837 RES=0x00 ACK URGP=0" + }, + { + "line": "Mar 4 19:25:13 UDM-Pro user.warn kernel: [ 1170.826385] ALIEN BLOCK: IN=eth8 OUT= MAC=24:5a:4c:a2:b1:09:48:2c:d0:ab:93:3f:08:00 SRC=185.128.41.50 DST=192.168.10.2 LEN=40 TOS=0x00 PREC=0x00 TTL=244 ID=51242 PROTO=TCP SPT=56958 DPT=7001 WINDOW=1024 RES=0x00 SYN URGP=0" + }, + { + "line": "Mar 2 23:27:40 UDM-Pro authpriv.notice dropbear[29787]: Pubkey auth succeeded for 'root' with key sha1!! 0e:16:76:2b:89:b3:c0:c7:14:a4:00:be:8f:9b:38:9a:12:fd:20:48 from 192.168.96.27:56718" + }, + { + "line": "Mar 4 19:26:07 UDM-Pro user.warn kernel: [ 1225.134675] conntrack: generic helper won't handle protocol 47. Please consider loading the specific helper module." + }, + { + "line": "Mar 4 19:39:06 UDM-Pro user.notice syswrapper: [apply-config] using fast apply" + }, + { + "line": "Feb 27 23:59:39 UDM-Pro user.notice dpi-flow-stats: ubnt-dpi-util: fingerprint_overrides API failed with HTTP -1" + }, + { + "line": "Mar 4 19:09:18 UDM-Pro user.notice getsig.sh: alien enabled, starting update." + }, + { + "line": "Mar 4 19:22:19 UDM-Pro daemon.info dnsmasq-dhcp[7431]: DHCPREQUEST(br96) 192.168.96.31 a0:57:e3:00:78:9e" + }, + { + "line": "Mar 6 22:38:20 UDM-Pro user.warn kernel: [ 6509.503966] [FW-A-LAN_LOCAL_U-2147483647]IN=br96 OUT= MAC= SRC=192.168.96.1 DST=233.89.188.1 LEN=32 TOS=0x00 PREC=0xC0 TTL=1 ID=0 DF PROTO=2" + }, + { + "line": "Feb 28 14:25:54 UDM-Pro daemon.err mcad: mcad[2910]: ace_reporter.reporter_fail(): initial contact failed #6, url=http://localhost:8080/inform, rc=7" + } + ] + } +} diff --git a/src/formats/uwsgi_log.json b/src/formats/uwsgi_log.json new file mode 100644 index 0000000..cabc436 --- /dev/null +++ b/src/formats/uwsgi_log.json @@ -0,0 +1,108 @@ +{ + "$schema": "https://lnav.org/schemas/format-v1.schema.json", + "uwsgi_log": { + "title": "Uwsgi Log", + "description": "The uwsgi log format.", + "multiline": false, + "regex": { + "std": { + "pattern": "^\\[pid: (?<s_pid>\\d+)\\|app: (?<s_app>[\\-\\d]+)\\|req: (?<s_req>[\\-\\d]+)/(?<s_worker_reqs>\\d+)\\] (?<c_ip>[^ ]+) \\((?<cs_username>[^\\)]*)\\) \\{(?<cs_vars>\\d+) vars in (?<cs_bytes>\\d+) bytes\\} \\[(?<timestamp>[^\\]]+)\\] (?<cs_method>[A-Z]+) (?<cs_uri_stem>[^ \\?]+)(?:\\?(?<cs_uri_query>[^ ]*))? => generated (?<sc_bytes>\\d+) bytes in (?<s_runtime>\\d+) (?<rt_unit>\\w+) \\((?<cs_version>[^ ]+) (?<sc_status>\\d+)\\) (?<sc_headers>\\d+) headers in (?<sc_header_bytes>\\d+) bytes \\((?<s_switches>\\d+) switches on core (?<s_core>\\d+)\\)(?<body>.*)" + } + }, + "level-field": "sc_status", + "level": { + "error": "^[^123]" + }, + "opid-field": "s_pid", + "value": { + "s_pid": { + "kind": "string", + "identifier": true + }, + "s_app": { + "kind": "string", + "identifier": true + }, + "s_req": { + "kind": "integer", + "foreign-key": true + }, + "s_worker_reqs": { + "kind": "integer", + "foreign-key": true + }, + "c_ip": { + "kind": "string", + "collate": "ipaddress", + "identifier": true + }, + "cs_username": { + "kind": "string", + "identifier": true + }, + "cs_vars": { + "kind": "integer" + }, + "cs_bytes": { + "kind": "integer" + }, + "cs_method": { + "kind": "string", + "identifier": true + }, + "cs_uri_stem": { + "kind": "string", + "identifier": true + }, + "cs_uri_query": { + "kind": "string" + }, + "sc_bytes": { + "kind": "integer" + }, + "s_runtime": { + "kind": "float", + "description": "hello, world", + "unit": { + "field": "rt_unit", + "scaling-factor": { + "msecs": { + "op": "divide", + "value": 1000.0 + }, + "micros": { + "op": "divide", + "value": 1000000.0 + } + } + } + }, + "cs_version": { + "kind": "string", + "identifier": true + }, + "sc_status": { + "kind": "integer", + "foreign-key": true + }, + "sc_headers": { + "kind": "integer" + }, + "sc_header_bytes": { + "kind": "integer" + }, + "s_switches": { + "kind": "integer" + }, + "s_core": { + "kind": "string", + "identifier": true + } + }, + "sample": [ + { + "line": "[pid: 24386|app: 0|req: 482950/4125645] 86.221.170.65 () {44 vars in 1322 bytes} [Tue Jan 3 05:01:31 2012] GET /contest/log_presence/shhootter/?_=1325592089910 => generated 192 bytes in 21 msecs (HTTP/1.1 200) 4 headers in 188 bytes (1 switches on core 0)" + } + ] + } +}
\ No newline at end of file diff --git a/src/formats/vdsm_log.json b/src/formats/vdsm_log.json new file mode 100644 index 0000000..52529e3 --- /dev/null +++ b/src/formats/vdsm_log.json @@ -0,0 +1,67 @@ +{ + "$schema": "https://lnav.org/schemas/format-v1.schema.json", + "vdsm_log": { + "title": "Vdsm Logs", + "description": "Vdsm log format", + "url": "http://www.ovirt.org/develop/developer-guide/vdsm/log-files/", + "regex": { + "v4.1": { + "pattern": "(?s)^(?<timestamp>\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2},\\d{3}(?:\\+\\d{4})?) (?<level>\\w+)\\s+\\((?<thread>.+?)\\) \\[(?<logger>.+?)\\]\\s+?(?<body>.*?)\\s+?\\((?<src_file>\\w+?):(?<src_line>\\d+)\\)(\\n(?<traceback>Traceback.*?)(?=\\n(?P=timestamp)|$))?" + }, + "v4": { + "pattern": "^(?<tid>[^:]+)::(?<level>[^:]+)::(?<timestamp>\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}(?:,\\d{3})?)::(?<module>[^:]+)::(?<src_line>[^:]+)::(?<logger>[^:]+)::\\((?<func>[^\\)]+)\\)(?<body>.*)" + } + }, + "level-field": "level", + "level": { + "error": "ERROR", + "debug": "DEBUG", + "info": "INFO", + "warning": "WARNING|WARN", + "critical": "CRIT", + "trace": "TRACE" + }, + "value": { + "tid": { + "kind": "string", + "identifier": true + }, + "module": { + "kind": "string", + "identifier": true + }, + "src_line": { + "kind": "integer", + "foreign-key": true + }, + "logger": { + "kind": "string", + "identifier": true + }, + "func": { + "kind": "string", + "identifier": true + }, + "thread": { + "kind": "string", + "identifier": true + }, + "src_file": { + "kind": "string", + "identifier": true + }, + "traceback": { + "kind": "string", + "identifier": true + } + }, + "sample": [ + { + "line": "Thread-1950::INFO::2011-12-07 12:14:15,018::dispatcher::94::Storage.Dispatcher.Protect::(run) Run and protect: getDeviceList, args: ( storageType=2)" + }, + { + "line": "2017-03-06 14:49:05,167+0200 INFO (vm/9e5dd42e) [virt.vm] (vmId='9e5dd42e-5177-4da3-a6ce-87fa2052d315') (vm:2104)" + } + ] + } +}
\ No newline at end of file diff --git a/src/formats/vmk_log.json b/src/formats/vmk_log.json new file mode 100644 index 0000000..5d93def --- /dev/null +++ b/src/formats/vmk_log.json @@ -0,0 +1,51 @@ +{ + "$schema": "https://lnav.org/schemas/format-v1.schema.json", + "vmk_log": { + "title": "VMKernel Logs", + "description": "The VMKernel's log format", + "url": "", + "regex": { + "std": { + "pattern": "^(?<timestamp>\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}Z) cpu(?<cpu>\\d+):(?<world_id>\\d+)(?: opID=(?<opid>[^\\)]+))?\\)((?:(?<level>WARNING|ALERT)|(?<subsystem>[^:]+)): )?(?<body>.*)" + } + }, + "level-field": "level", + "level": { + "error": "ALERT", + "warning": "WARNING" + }, + "max-unrecognized-lines": 15000, + "opid-field": "opid", + "value": { + "cpu": { + "kind": "integer", + "identifier": true, + "foreign-key": true + }, + "world_id": { + "kind": "integer", + "identifier": true, + "foreign-key": true + }, + "subsystem": { + "kind": "string", + "identifier": true + }, + "opid": { + "kind": "string", + "identifier": true + } + }, + "sample": [ + { + "line": "2014-11-14T19:19:51.559Z cpu7:35233)VC: 2002: Device rescan time 704 msec (total number of devices 91)" + }, + { + "line": "2015-04-01T22:22:35.038Z cpu22:44012977)ALERT: This is what an alert looks like." + }, + { + "line": "2022-06-02T02:16:57.414Z cpu31:1001392590 opID=827cfaf)<unk>: UWVMKSyscall: ForkExec:2408: hostd-worker: Found params <group=hostd-tmp,mem=10>" + } + ] + } +}
\ No newline at end of file diff --git a/src/formats/vmw_log.json b/src/formats/vmw_log.json new file mode 100644 index 0000000..afe1128 --- /dev/null +++ b/src/formats/vmw_log.json @@ -0,0 +1,241 @@ +{ + "$schema": "https://lnav.org/schemas/format-v1.schema.json", + "vmw_log": { + "title": "VMware Logs", + "description": "One of the log formats used in VMware's ESXi and vCenter software.", + "url": "https://kb.vmware.com/kb/2004201", + "regex": { + "6.0+": { + "pattern": "^(?:\\[#\\d+\\] )?(?<timestamp>\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}(?:Z|[-+]\\d{2}:\\d{2})) (?<level>\\w+)(?:\\(\\d+\\)+)? (?<prc>[\\w\\-]+)\\[(?<tid>\\w+)\\]:? (?:\\w+ -\\[\\d+\\] )?\\[(?<src>\\w+@\\d+)(?:\\s+sub=(?<sub>.*?(?!\\w+=)))?(?:\\s+item=(?<item>[\\w\\.\\-@/:]+))?(?: req=(?<req>[^ \\]]+))?(?: opI(?:D|d)=(?<opid>(?:req=)?[\\w@ \\-\\.:]+?(?!\\w+=)))?(?: sid=(?<sid>[^ \\]]+))?(?: user=(?<user>[^ \\]<]+(?:<[^>]+>)?))?(?: update=(?<vpxa_update>\\d+))?(?:\\s+reason=(?<reason>[^\\]]+))?\\]\\s*(?<body>.*)$" + }, + "6.0+-nosrc": { + "pattern": "^(?<timestamp>\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}(?:Z|[-+]\\d{2}:\\d{2})) (?<level>\\w+)(?:\\(\\d+\\)+)? (?<prc>[\\w\\-]+)\\[(?<tid>\\w+)\\]:? \\[(?:opI(?:D|d)=(?<opid>[^\\]]+))\\]\\s*(?<body>.*)$" + }, + "section": { + "pattern": "^(?<timestamp>\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}(?:Z|[-+]\\d{2}:\\d{2})) (?:- last log rotation time, \\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}(?:Z|[-+]\\d{2}:\\d{2}))?\\s*(ESX KMX Agent started.|(?:- time the service was last started(?: \\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}.\\d{3}Z)?, )?Section for (?:[^,]+), pid=(?<tid>\\w+).*)" + }, + "esx-section": { + "pattern": "^(?<timestamp>\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}(?:Z|[-+]\\d{2}:\\d{2})) (?<level>\\w+)(?:\\(\\d+\\)+) (?<prc>[\\w\\-]+)\\[(?<tid>\\w+)\\]: (?:Logs rotated. \\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}(?:Z|[-+]\\d{2}:\\d{2}))?(?:- last log rotation time, \\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}(?:Z|[-+]\\d{2}:\\d{2}))?\\s*(ESX KMX Agent started.|(?:- time the service was last started(?: \\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}.\\d{3}Z)?, )?Section for (?:[^,]+), pid=(?:\\w+).*)" + }, + "5.0+": { + "pattern": "^(?<timestamp>\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}(?:Z|[-+]\\d{2}:\\d{2})) \\[(?<tid>\\w+) (?<level>\\w+) '(?<comp>[^']+)'(?: opID=(?<opid>[^ \\]]+))?(?: user=(?<user>[^ \\]]+))?\\](?<body>.*)(?:\\n.*)?$" + }, + "pre-5.0": { + "pattern": "^\\[(?<timestamp>\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}\\.\\d{3}) (?<tid>\\w+) (?<level>\\w+) '(?<comp>[^']+)'(?: opID=(?<opid>[^ \\]]+))?(?: user=(?<user>[^ \\]]+))?\\](?<body>.*)(?:\\n.*)?$" + }, + "ls-log": { + "pattern": "^\\[(?<timestamp>\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2},\\d{3}) (?<tid>[\\w\\-]+)\\s+(?<level>\\w+)\\s+(?<comp>[^\\]]+)\\]\\s+(?<body>.*)" + }, + "hdr-ftr": { + "pattern": "^(?<timestamp>\\d{4}-\\d{2}-\\d{2}(T| )\\d{2}:\\d{2}:\\d{2}(?:.|,)\\d{3}(?:Z|[-+]\\d{2}:\\d{2})) \\[(?<prc>[^\\[]+)\\[(?<tid>\\w+)\\]:\\s+(?<body>.*)\\]$" + }, + "pylog": { + "pattern": "^(?<timestamp>\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}(?:\\.\\d{3})?(?:Z|[-+]\\d{2}:\\d{2})) (?<prc>[^:]+):\\s+(?<tid>\\d+):\\s+(?<comp>[^:]+):(?<line>\\d+)?\\s+(?<level>\\w+):?\\s+(?<body>.*)(?:\\n.*)?$" + }, + "vum-log4cpp": { + "pattern": "^\\[(?<timestamp>\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}:\\d{3}) '(?<category>[^']*)' (?<tid>\\d+) (?<level>[a-zA-Z]+)\\]\\s+(?>\\[(?<file>\\S+), (?<line>\\d+)\\])? (?<body>.*$)" + }, + "pylog2": { + "pattern": "^(?<prc>[^:]+):(?<timestamp>\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2},\\d{3})\\[(?<tid>\\w+)\\](?<file>[^:]+):(?<line>\\d+) \\[(?<level>[a-zA-Z]+)\\]\\s+(?<body>.*)$" + }, + "pylog3": { + "pattern": "^(?<prc>[^:]+): (?<tid>\\d+): (?<timestamp>\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2},\\d{3}) (?<file>[^:]+):(?<line>\\d+) (?<level>[a-zA-Z]+)\\s+(?<body>.*)$" + } + }, + "level-field": "level", + "level": { + "info": "^(?i)(?:info|In)$", + "debug": "^(?i)(?:debug|Db)$", + "error": "^(?i)(error|Er)$", + "warning": "^(?i)(warning|warn|Wa)$", + "trace": "^(?i)verbose$", + "critical": "^(?i)(?:crit|Cr)$", + "fatal": "^(?i)(?:alert|fatal|panic|Al|Em)$" + }, + "opid-field": "opid", + "value": { + "prc": { + "kind": "string", + "identifier": true + }, + "tid": { + "kind": "string", + "identifier": true + }, + "src": { + "kind": "string", + "identifier": true + }, + "comp": { + "kind": "string", + "identifier": true + }, + "line": { + "kind": "integer", + "identifier": true + }, + "sub": { + "kind": "string", + "identifier": true + }, + "item": { + "kind": "string", + "identifier": true + }, + "opid": { + "kind": "string", + "identifier": true + }, + "req": { + "kind": "string", + "identifier": true + }, + "sid": { + "kind": "string", + "identifier": true + }, + "user": { + "kind": "string", + "identifier": true + }, + "vpxa_update": { + "kind": "integer", + "identifier": true + }, + "reason": { + "kind": "string" + }, + "file": { + "kind": "string", + "identifier": true + } + }, + "search-table": { + "vpxd_session_stats": { + "pattern": "/SessionStats/SessionPool/Session/Id='(?<SessionId>[^']+)'/Username='(?<Username>[^']+)'/ClientIP='(?<ClientIP>[^']+)'(?<ProfileKey>[^ ]+) (?<ProfileValue>[^\\n]+)", + "glob": "*/vpxd-profile*" + }, + "vpxd_session_pool_stats": { + "pattern": "/SessionStats/SessionPool/Id='(?<SessionId>[^']+)'/Username='(?<Username>[^']+)'(?<ProfileKey>[^ ]+) (?<ProfileValue>[^\\n]+)", + "glob": "*/vpxd-profile*" + }, + "vpx_lro_begin": { + "pattern": "\\[VpxLRO\\] -- BEGIN (?<lro_id>\\S+) -- (?<entity>\\S*) -- (?<operation>\\S*) -- (?:(?<SessionId>[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})(?:\\((?<SessionSubId>[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})\\))?)?" + }, + "vpx_lro_finish": { + "pattern": "\\[VpxLRO\\] -- FINISH (?<lro_id>\\S+)" + }, + "vpx_lro_error": { + "pattern": "\\[VpxLRO\\] -- ERROR (?<lro_id>\\S+) -- (?:(?<SessionId>[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})(?:\\((?<SessionSubId>[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})\\))?)? -- (?<entity>\\S*) -- (?<operation>\\S*):\\s*(?<error>.*)", + "level": "error" + } + }, + "tags": { + "test-failure": { + "description": "Tag for gtest test-case failures", + "paths": [ + { + "glob": "*/test.log" + } + ], + "pattern": "^Expected equality of these values:" + } + }, + "sample": [ + { + "line": "2021-05-24T20:31:05.671Z - last log rotation time, 2021-05-24T09:30:02.683Z - time the service was last started, Section for VMware ESX, pid=1000080910, version=7.0.3, build=0, option=DEBUG" + }, + { + "line": "[2011-04-01 15:14:34.203 F5A5AB90 info 'vm:/vmfs/volumes/4d6579ec-23f981cb-465c-00237da0cfee/Vmotion-test/Vmotion-test.vmx' opID=F6FC49D5-000007E6-d] VMotionPrepare: dstMgmtIp=10.21.49.138" + }, + { + "line": "2014-11-04T15:53:31.075+05:30 verbose vpxd[05160] [Originator@6876 sub=PropertyProvider opID=ProcessAlarmFiring-427c3c55] RecordOp ASSIGN: declaredAlarmState[\"alarm-1.host-23\"], host-23. Applied change to temp map." + }, + { + "line": "2020-05-11T22:32:22.932Z [/etc/init.d/vvold[1000211787]: WaitVvoldToComeUp /var/run/vmware/.vmware-vvol.started created]" + }, + { + "line": "2020-05-11T22:32:22.736Z info -[1000212040] [Originator@6876 sub=Default] Successfully registered SIGHUP handler" + }, + { + "line": "2014-01-17T04:55:50.347Z [7F03ECE76700 verbose 'Default' opID=2140bc71] [VpxVmomi] Invoke done: vmodl.query.PropertyCollector.waitForUpdatesEx session: c580b3ef-0011-88a5-b2af-7ca7e74114c8" + }, + { + "line": "2014-11-04T12:46:42.990+05:30 Section for VMware VirtualCenter, pid=6432, version=6.0.0, build=2255588, option=BETA" + }, + { + "line": "[2013-01-16 02:26:25,500 pool-3-thread-1 INFO com.vmware.vim.license.service.impl.ServiceImpl] License Accounting Service initialized" + }, + { + "line": "2020-05-11T22:32:38.751Z info vsansystem[1000212707] [Originator@6876 sub=Libs opId=vsan-d810] VsanConfigStore: Get object host from vsan" + }, + { + "line": "2020-05-11T22:32:56Z lifecycle: 1000212917: lifecyclectl:58 ERROR Failed to delete pidfile(/var/vmware/lifecycle/lifecycle.apply.pid) at boot Traceback (most recent call last): File \"/usr/lib/vmware/lifecycle/bin/lifecyclectl.py\", line 55, in main os.remove(PID_APPLY_FILE) FileNotFoundError: [Errno 2] No such file or directory: '/var/vmware/lifecycle/lifecycle.apply.pid'" + }, + { + "line": "2020-05-11T22:32:09Z esxupdate: 1000211352: root: INFO: Command = profile.setacceptance" + }, + { + "line": "2020-05-11T22:36:11.202Z verbose hostd[1000212678] [Originator@6876 sub=Vimsvc.Ticket 52 66 af 87 76 b0 ab 21-dd 2b 28 27 3a 39 fd fd opID=ec9d7fe8] Ticket issued for root" + }, + { + "line": "[2020-05-26 19:46:43:259 'ConfigurationMgr' 139700477753600 INFO] [configurationMgr, 155] Configuration manager created...vcidbInit" + }, + { + "line": "HsmService:2021-05-19 17:36:35,978[MainThread]hsmService:737 [INFO] HSM Service is initialized" + }, + { + "line": "ImageService: 139853338433792: 2021-05-19 17:36:35,675 imageService:769 INFO Image service is initialized" + }, + { + "line": "2021-06-03T16:50:32.890Z - time the service was last started 2021-06-03T16:50:32.884Z, Section for VMware Update Manager, pid=5916, version=7.0.3, build=0, option=DEBUG" + }, + { + "line": "2021-06-16T14:59:52.362Z info vsand[1000083783] [opID=MainThread VsanMgmtSvcMain::_PatchPyVmomi] Python 3.8.8 (default, Jun 1 2021, 03:16:39) [GCC 4.6.3] on VMkernel, optimize: 0, recursionLimit: 300" + }, + { + "line": "2022-06-01T13:24:08.243Z Section for vdtc, pid=4774, version=e.x.p, build=19889761, option=Release" + }, + { + "line": "2022-06-02T12:21:26.285Z info vmware-vum-server[00932] [Originator@6876 sub=SessionAuthData reason=Timeout expired] [vciSessionAuthData 94] Session [521601f2-6eb3-0475-7be8-c30eed1040ab] is closed" + }, + { + "line": "2022-06-01T20:17:31.352Z info vpxd[27059] [Originator@6876 sub=drmLogger item=FixNIOCViolation opID=SWI-5441881e] FixNIOCViolation worker starts for host [vim.HostSystem:host-588,esx-3-23.vlcm.com]" + }, + { + "line": "2022-06-01T14:23:06.536Z info vpxd[27349] [Originator@6876 sub=Http2Session #28] Starting a Http2Session (server): <io_obj p:0x00007eff5867c618, h:477, <TCP '127.0.0.1 : 8093'>, <TCP '127.0.0.1 : 50504'>>" + }, + { + "line": "2022-06-02T08:34:54.811Z info vpxd[44568] [Originator@6876 sub=SoapAdapter[0].HTTPService] Max buffered response size is 104857600 bytes" + }, + { + "line": "2022-06-02T08:34:55.613Z info vpxd[44568] [Originator@6876 sub=SSL SoapAdapter[1].HTTPService] Max buffered response size is 104857600 bytes" + }, + { + "line": "2022-06-02T08:34:55.753Z info vpxd[44568] [Originator@6876 sub=Req@vsan/VMC M5] Created version logger for vsan/VMC M5 (vsan.version.version8)" + }, + { + "line": "2022-06-02T08:34:55.421Z info vpxd[44568] [Originator@6876 sub=Req@vsan/vSAN 7.0U2] Created version logger for vsan/vSAN 7.0U2 (vsan.version.version16)" + }, + { + "line": "2022-06-02T08:33:42.327Z info StatsMonitor[40881] [Originator@6876 sub=LinuxStatsProvider(00005575a7a395f0)] Created" + }, + { + "line": "2022-06-02T08:33:42.692Z info rhttpproxy[40998] [Originator@6876 sub=WorkQueue.rhttpproxy.packetsniffer[0000562e3c06fee0].queue] Created: WorkQueue.rhttpproxy.packetsniffer[0000562e3c06fee0].queue, type = serial, priority = 16, itemWeight = 1" + }, + { + "line": "2022-06-02T02:56:51.118Z Db(167) Hostd[1001392583]: [Originator@6876 sub=AdapterServer] Vdt span started, activation is <<527066d1-5c96-4854-ebdb-fec666747e0e, <TCP '127.0.0.1 : 8307'>, <TCP '127.0.0.1 : 28140'>>, ha-root-pool, vim.ManagedEntity.GetName>, method name is GetName", + "level": "debug" + }, + { + "line": "2022-06-02T02:56:51.640Z In(14) vmsyslogd[1001390391]: Logs rotated. 2022-06-02T02:54:42.721Z - time the service was last started 2022-06-02T02:54:42.708Z, Section for VMware ESX, pid=1001391976, version=8.0.0, build=19833347, option=BETA" + }, + { + "line": "2022-06-02T02:15:22.987Z In(166) Hostd[1001392061]: info -[1001392061] [Originator@6876 sub=Default] Supported VMs 640" + }, + { + "line": "2022-06-02T03:20:05.107Z Db(167) Hostd[1001392035]: [Originator@6876 sub=AdapterServer opID=531c52d7-9d8a sid=52806149 user=vpxuser:<no user>] New request: target='vim.HostSystem:ha-host', method='retrieveInternalCapability', session='52806149-fe15-f6ff-7685-353ae5d93dcc'" + } + ] + } +}
\ No newline at end of file diff --git a/src/formats/vmw_py_log.json b/src/formats/vmw_py_log.json new file mode 100644 index 0000000..25bb3e8 --- /dev/null +++ b/src/formats/vmw_py_log.json @@ -0,0 +1,42 @@ +{ + "$schema": "https://lnav.org/schemas/format-v1.schema.json", + "vmw_py_log": { + "title": "VMware vSphere log format", + "description": "The log format for some VMware vSphere services", + "url": "http://kb.vmware.com/kb/2000988", + "regex": { + "std": { + "pattern": "^(?<timestamp>\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}(?:\\.\\d{3})?(?: (?:AM|PM) UTC)?) \\[(?<pid>\\d+)\\](?<level>ERROR|WARNING|INFO|DEBUG):(?<module>[\\w\\-\\.]+):(?<body>.*$)" + } + }, + "level-field": "level", + "timestamp-field": "timestamp", + "level": { + "error": "ERROR", + "debug": "DEBUG", + "info": "INFO", + "warning": "WARNING" + }, + "value": { + "pid": { + "kind": "integer", + "identifier": true, + "foreign-key": true, + "description": "The ID of the process that generated the message" + }, + "module": { + "kind": "string", + "identifier": true, + "description": "The name of the module that generated the message" + } + }, + "sample": [ + { + "line": "2015-04-24T21:09:29.296 [25376]INFO:somemodule:Something very INFOrmative." + }, + { + "line": "2022-06-01T13:23:25.515 [2376]DEBUG:vmware.vherd.base.detwist:method = com.vmware.appliance.version1.networking.interfaces.list, args = ()" + } + ] + } +}
\ No newline at end of file diff --git a/src/formats/vmw_vc_svc_log.json b/src/formats/vmw_vc_svc_log.json new file mode 100644 index 0000000..ed507f7 --- /dev/null +++ b/src/formats/vmw_vc_svc_log.json @@ -0,0 +1,48 @@ +{ + "$schema": "https://lnav.org/schemas/format-v1.schema.json", + "vmw_vc_svc_log": { + "title": "VMware Go Log", + "description": "Log files for go-based logs", + "url": "https://docs.vmware.com/en/VMware-vSphere/7.0/vmware-vsphere-with-tanzu/GUID-2A989D79-463C-4EC8-A5F2-CDC3A2C827FB.html", + "regex": { + "std": { + "pattern": "^(?<timestamp>\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}Z)\\s+(?<level>\\w+)\\s+(?<module>\\w+)\\s\\[(?<srcfile>[^:]+):(?<srcline>\\d+)\\](\\s+\\[opID=(?<opid>[^\\]]+)\\])?\\s+(?<body>.*)" + } + }, + "opid-field": "opid", + "value": { + "body": { + "kind": "string" + }, + "opid": { + "kind": "string" + }, + "srcfile": { + "kind": "string", + "identifier": true + }, + "srcline": { + "kind": "string", + "identifier": true + }, + "module": { + "kind": "string", + "identifier": true + } + }, + "sample": [ + { + "line": "2022-06-02T12:25:11.537Z info wcp [eamagency/util.go:148] [opID=vCLS] Going to read service config", + "level": "info" + }, + { + "line": "2022-06-02T10:54:45.001Z debug wcp [auth/session.go:156] Removing expired sessions", + "level": "debug" + }, + { + "line": "2022-06-02T08:35:17.031Z info vlcm [logger/teelogger.go:47] [opID=vapi] Loading Introspection Services", + "level": "info" + } + ] + } +} diff --git a/src/formats/xmlrpc_log.json b/src/formats/xmlrpc_log.json new file mode 100644 index 0000000..31638b3 --- /dev/null +++ b/src/formats/xmlrpc_log.json @@ -0,0 +1,43 @@ +{ + "$schema": "https://lnav.org/schemas/format-v1.schema.json", + "xmlrpc_log": { + "title": "RHN server XMLRPC log format", + "description": "Generated by Satellite's XMLRPC component", + "url": "https://access.redhat.com/products/red-hat-satellite", + "regex": { + "main": { + "pattern": "^(?<timestamp>\\d{4}/\\d{2}/\\d{2} \\d{2}:\\d{2}:\\d{2} [+-]?\\d{2}:\\d{2}) (?<pid>\\d+) (?<client_ip>\\S+): (?<module>\\w+)/(?<function>.*)(?<arguments>\\(.*?\\))?(?<body>.*)$" + } + }, + "value": { + "pid": { + "kind": "integer", + "identifier": true + }, + "client_ip": { + "kind": "string", + "identifier": true + }, + "module": { + "kind": "string" + }, + "function": { + "kind": "string" + } + }, + "sample": [ + { + "line": "2015/05/24 07:48:21 -05:00 767 10.206.22.17: xmlrpc/up2date.listChannels(1000011979,)" + }, + { + "line": "2015/05/24 07:48:22 -05:00 1377 10.184.37.105: xmlrpc/registration.welcome_message('lang: None',)" + }, + { + "line": "2015/05/24 07:48:22 -05:00 759 10.49.10.30: xmlrpc/registration.register_osad" + }, + { + "line": "2015/05/24 07:48:22 -05:00 759 10.49.10.30: rhnServer/server_certificate.valid('Server id ID-1000019942 not found in database',)" + } + ] + } +}
\ No newline at end of file |