diff options
Diffstat (limited to 'doc/schemas/logstash')
| -rw-r--r-- | doc/schemas/logstash/README | 95 | ||||
| -rw-r--r-- | doc/schemas/logstash/kibana4-dashboard.json | 123 | ||||
| -rw-r--r-- | doc/schemas/logstash/log-courier.conf | 56 | ||||
| -rw-r--r-- | doc/schemas/logstash/logstash-radius.conf | 256 | ||||
| -rwxr-xr-x | doc/schemas/logstash/radius-mapping.sh | 100 |
5 files changed, 630 insertions, 0 deletions
diff --git a/doc/schemas/logstash/README b/doc/schemas/logstash/README new file mode 100644 index 0000000..2f36eb6 --- /dev/null +++ b/doc/schemas/logstash/README @@ -0,0 +1,95 @@ +Example configuration for logstash/elasticsearch +================================================ + +So you've got all these RADIUS logs, but how do you analyse them? What is the +easiest way to query the logs, find out when a client connected or disconnected, +or view the top ten clients logging into the system over the last six hours? + +The elastic stack is designed and built to do just that. elasticsearch is a +search engine; logstash is commonly used to feed data in, and kibana the web +interface to query the logs in near real time. + +Installing the elastic stack is beyond the scope of this document, but can be done +in a short amount of time by any competent sysadmin. Then comes getting the +logs in. + +This directory contains the following files as a starting point for feeding +RADIUS logs into elasticsearch via logstash, then sample dashboards for Kibana +to explore the data. + +Files +----- + +Please note that all files should be reviewed before use to determine if they +are suitable for your configuration/system, especially if you are integrating +this into an existing logstash/elasticsearch setup. + +radius-mapping.sh + + Each elasticsearch index needs a mapping to describe how fields are stored. + If one is not provided then all is not lost as elasticsearch will build one + on the fly. However, this may not be optimal, especially for RADIUS data, as + all fields will be analyzed making some visualisations hard or impossible + (such as showing top N clients). + + This shell script (which just runs curl) pushes a template mapping into the + elasticsearch cluster. + +logstash-radius.conf + + A sample configuration file for logstash that parses RADIUS 'detail' files. + It processes these by joining each record onto one line, then splitting the + tab-delimited key-value pairs out. Some additional data is then extracted + from certain key attributes. + + The logstash config will need to be edited at least to set the input method: + for experimentation the given input (file) may be used. If logstash is running + on the RADIUS server itself then this example input may be appropriate, + otherwise a different input such as log-courier or filebeat may be better to + get the data over the network to logstash. + + It would be best to use an input method that can join the multiple lines of + the detail file together and feed them to logstash as a single entry, rather + than using the logstash multiline codec. + +log-courier.conf + + An example configuration for the log-courier feeder. + +kibana4-dashboard.json + + Basic RADIUS dashboard (for Kibana 4 to Kibana 6). + + To import the dashboard first create a new index called "radius-*" in + Settings/Indices. Then go to Kibana's Settings page, "Objects" and "Import". + Once imported open the "RADIUS detail" dashboard. + + +Example usage +------------- + +Install mapping (only needs to be done once): + + $ ./radius-mapping.sh + +Edit logstash-radius.conf to point to the correct file, then feed a detail file +in: + + # /usr/share/logstash/bin/logstash --path.settings=/etc/logstash -f logstash-radius.conf + +To view debug output, append `--log.level=debug`. + + +See also +-------- + +elasticsearch web site: http://www.elastic.co/ + +The configuration examples presented here have been tested with the +following software versions: + + elasticsearch 6.7.0 + logstash 6.7.0 + kibana 6.7.0 + kibana 5.1.2 + kibana 4.1.11 diff --git a/doc/schemas/logstash/kibana4-dashboard.json b/doc/schemas/logstash/kibana4-dashboard.json new file mode 100644 index 0000000..6c379df --- /dev/null +++ b/doc/schemas/logstash/kibana4-dashboard.json @@ -0,0 +1,123 @@ +[ + { + "_id": "RADIUS-data", + "_type": "search", + "_source": { + "title": "RADIUS data", + "description": "", + "hits": 0, + "columns": [ + "User-Name", + "Calling-Station-Id", + "Called-Station-Id", + "Framed-IP-Address", + "NAS-Identifier" + ], + "sort": [ + "@timestamp", + "desc" + ], + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"index\":\"radius-*\",\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\"}},\"highlight\":{\"pre_tags\":[\"@kibana-highlighted-field@\"],\"post_tags\":[\"@/kibana-highlighted-field@\"],\"fields\":{\"*\":{}},\"fragment_size\":2147483647},\"filter\":[]}" + } + } + }, + { + "_id": "RADIUS-detail", + "_type": "dashboard", + "_source": { + "title": "RADIUS detail", + "hits": 0, + "description": "", + "panelsJSON": "[{\"col\":5,\"id\":\"RADIUS-unique-User-Name-by-day\",\"row\":1,\"size_x\":4,\"size_y\":4,\"type\":\"visualization\"},{\"col\":1,\"columns\":[\"User-Name\",\"Calling-Station-Id\",\"Called-Station-Id\",\"Framed-IP-Address\",\"NAS-Identifier\"],\"id\":\"RADIUS-data\",\"row\":5,\"size_x\":8,\"size_y\":4,\"sort\":[\"@timestamp\",\"desc\"],\"type\":\"search\"},{\"col\":1,\"id\":\"RADIUS-accounting-packets-histogram\",\"row\":1,\"size_x\":4,\"size_y\":4,\"type\":\"visualization\"},{\"col\":9,\"id\":\"RADIUS-table-topN-data-transferred-by-User-Name\",\"row\":1,\"size_x\":4,\"size_y\":4,\"type\":\"visualization\"},{\"id\":\"RADIUS-Sessions-per-NAS\",\"type\":\"visualization\",\"size_x\":4,\"size_y\":4,\"col\":9,\"row\":5}]", + "version": 1, + "timeRestore": true, + "timeTo": "now", + "timeFrom": "now-7d", + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"filter\":[{\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\"}}}]}" + } + } + }, + { + "_id": "RADIUS-Accounting-Start-data", + "_type": "search", + "_source": { + "title": "RADIUS Accounting-Start data", + "description": "", + "hits": 0, + "columns": [ + "User-Name", + "Calling-Station-Id", + "Called-Station-Id", + "Framed-IP-Address", + "NAS-Identifier" + ], + "sort": [ + "@timestamp", + "desc" + ], + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"index\":\"radius-*\",\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\"}},\"highlight\":{\"pre_tags\":[\"@kibana-highlighted-field@\"],\"post_tags\":[\"@/kibana-highlighted-field@\"],\"fields\":{\"*\":{}},\"fragment_size\":2147483647},\"filter\":[{\"meta\":{\"negate\":false,\"index\":\"radius-*\",\"key\":\"Acct-Status-Type\",\"value\":\"Start\",\"disabled\":false},\"query\":{\"match\":{\"Acct-Status-Type\":{\"query\":\"Start\",\"type\":\"phrase\"}}}}]}" + } + } + }, + { + "_id": "RADIUS-unique-User-Name-by-day", + "_type": "visualization", + "_source": { + "title": "RADIUS unique User-Name by day", + "visState": "{\n \"type\": \"histogram\",\n \"params\": {\n \"shareYAxis\": true,\n \"addTooltip\": true,\n \"addLegend\": true,\n \"scale\": \"linear\",\n \"mode\": \"stacked\",\n \"times\": [],\n \"addTimeMarker\": false,\n \"defaultYExtents\": false,\n \"setYExtents\": false,\n \"yAxis\": {}\n },\n \"aggs\": [\n {\n \"id\": \"1\",\n \"type\": \"cardinality\",\n \"schema\": \"metric\",\n \"params\": {\n \"field\": \"User-Name\"\n }\n },\n {\n \"id\": \"2\",\n \"type\": \"date_histogram\",\n \"schema\": \"segment\",\n \"params\": {\n \"field\": \"@timestamp\",\n \"interval\": \"d\",\n \"customInterval\": \"2h\",\n \"min_doc_count\": 1,\n \"extended_bounds\": {}\n }\n },\n {\n \"id\": \"3\",\n \"type\": \"terms\",\n \"schema\": \"group\",\n \"params\": {\n \"field\": \"User-Name\",\n \"size\": 50,\n \"order\": \"desc\",\n \"orderBy\": \"1\"\n }\n }\n ],\n \"listeners\": {}\n}", + "description": "", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\n \"filter\": []\n}" + }, + "savedSearchId": "RADIUS-data" + } + }, + { + "_id": "RADIUS-accounting-packets-histogram", + "_type": "visualization", + "_source": { + "title": "RADIUS accounting packets histogram", + "visState": "{\"type\":\"histogram\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":true,\"scale\":\"linear\",\"mode\":\"stacked\",\"times\":[],\"addTimeMarker\":false,\"defaultYExtents\":false,\"setYExtents\":false,\"yAxis\":{}},\"aggs\":[{\"id\":\"1\",\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"auto\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{}}},{\"id\":\"3\",\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"Acct-Status-Type\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}", + "description": "", + "savedSearchId": "RADIUS-data", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"filter\":[]}" + } + } + }, + { + "_id": "RADIUS-table-topN-data-transferred-by-User-Name", + "_type": "visualization", + "_source": { + "title": "RADIUS table topN data transferred by User-Name", + "visState": "{\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMeticsAtAllLevels\":false},\"aggs\":[{\"id\":\"1\",\"type\":\"max\",\"schema\":\"metric\",\"params\":{\"field\":\"Acct-Output-Octets_long\"}},{\"id\":\"2\",\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"User-Name\",\"size\":10,\"order\":\"desc\",\"orderBy\":\"1\"}},{\"id\":\"3\",\"type\":\"max\",\"schema\":\"metric\",\"params\":{\"field\":\"Acct-Input-Octets_long\"}}],\"listeners\":{}}", + "description": "", + "savedSearchId": "RADIUS-data", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"filter\":[]}" + } + } + }, + { + "_id": "RADIUS-Sessions-per-NAS", + "_type": "visualization", + "_source": { + "title": "RADIUS Sessions per NAS", + "visState": "{\"type\":\"pie\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":true,\"isDonut\":false},\"aggs\":[{\"id\":\"1\",\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"NAS-Identifier\",\"size\":20,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}", + "description": "", + "savedSearchId": "RADIUS-Accounting-Start-data", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"filter\":[]}" + } + } + } +] diff --git a/doc/schemas/logstash/log-courier.conf b/doc/schemas/logstash/log-courier.conf new file mode 100644 index 0000000..20d106f --- /dev/null +++ b/doc/schemas/logstash/log-courier.conf @@ -0,0 +1,56 @@ +# Example log-courier configuration file for RADIUS detail files. +# +# This has been tested with log-courier version 2.0.4 +# +{ + "general": { + "persist directory": "/var/lib/log-courier", + "log syslog": true, + "log stdout": false + }, + + "network": { + "transport": "tcp", + + # Servers to connect to. + # + "servers": [ + "logstash1.example:5140", + "logstash2.example:5140" + ] + }, + + "files": [ + { + # Match RADIUS detail files, but not anything that has + # been gzipped. + # + "paths": [ "/var/log/radius/radacct/*/detail-????????" ], + + # Add a type:"radiusdetail" field to the data so that + # logstash can tell what type of data this is (in case + # log-courier is being used for other data as well). + # + "fields": { + "type": "radiusdetail" + }, + + # Stop watching a file if nothing has been written in 12h. + # + "dead time": "12h", + + # Process multilines. If this is being used then the + # "multiline" section should be commented out from the + # logstash configuration. Logstash can then also be run + # with multiple workers (using -w). + # + "codecs": [ + { + "name": "multiline", + "patterns": [ "^[A-Z\t]" ], + "what": "next" + } + ] + } + ] +} diff --git a/doc/schemas/logstash/logstash-radius.conf b/doc/schemas/logstash/logstash-radius.conf new file mode 100644 index 0000000..f473179 --- /dev/null +++ b/doc/schemas/logstash/logstash-radius.conf @@ -0,0 +1,256 @@ +# logstash configuration to process RADIUS detail files +# +# Matthew Newton +# April 2019 +# +# This config has been tested with logstash version 6.7.0 +# +# RADIUS "detail" files are textual representations of the RADIUS +# packets, and are written to disk by e.g. FreeRADIUS. They look +# something like the following, with the timestamp on the first +# line then all attributes/values tab-indented. +# +# Tue Mar 10 15:32:24 2015 +# Packet-Type = Access-Request +# User-Name = "test@example.com" +# Calling-Station-Id = "01-02-03-04-05-06" +# Called-Station-Id = "aa-bb-cc-dd-ee-ff:myssid" +# NAS-Port = 10 +# NAS-IP-Address = 10.9.0.4 +# NAS-Identifier = "Wireless-Controller-1" +# Service-Type = Framed-User +# NAS-Port-Type = Wireless-802.11 +# + + + +# Example input - read data from a file. For example, to read in a +# detail file with this input you could use: +# +# # /usr/share/logstash/bin/logstash --path.settings=/etc/logstash -f logstash-radius.conf --log.level=debug +# + +input { + file { + path => "/var/log/radius/radacct/*/detail-*" + exclude => "*.gz" + + # Note when testing that logstash will remember where + # it got to and continue from there. + start_position => "beginning" + + # Set the type, for below. + type => radiusdetail + + # It is preferable to use a log feeder that can join + # multiple lines together, rather than using multiline + # here. For an example, see the log-courier + # configuration in this directory. + + # If you didn't read the above, go back and read it again. + + # If that is not possible you may be able to use the + # following section. Note that if you are using the + # "stdin" input, the file is chunked into 16k blobs, + # so every 16k a detail record is likely to be chopped + # in half. If you are using the "file" input (as in this + # example), the blank links between records are not + # passed through so the regex here has to be aware of + # that. Basically, do multiline as early as possible + # in your log feeder client not here and you'll avoid + # most issues that are likely to come up. + + codec => multiline { + pattern => "^\t" + negate => false + what => "previous" + } + + # If you really want to use the "stdin" input, this + # will work better, but be aware of the comments + # above. + + #codec => multiline { + # pattern => "^[A-Z\t]" + # negate => false + # what => "next" + #} + } +} + +# Moving into production will likely need something more reliable. +# There are many input methods, an example here using log-courier +# (which supports client-site multiline processing and does not +# lose log events if logstash is restarted). You could also +# investigate e.g. filebeat from Elastic. + +# input { +# courier { +# port => 5140 +# transport => "tcp" +# +# # Don't set the type here, as it's set in the +# # log-courier config instead. +# #type => radiusdetail +# } +# } + + + +# Filter stage. Here we take the raw logs and process them into +# something structured ready to index. Each attribute is stored as +# a separate field in the output document. + +filter { + + if [type] == "radiusdetail" { + + # Pull off the timestamp at the start of the + # detail record. Note there may be additional data + # after it that has been added by the local admin, + # so stop at a newline OR a tab. + + grok { + match => [ "message", "^(?<timestamp>[^\n\t]+)[\n\t]" ] + } + + # Create the @timestamp field. + + date { + match => [ "timestamp", "EEE MMM dd HH:mm:ss yyyy", + "EEE MMM d HH:mm:ss yyyy" ] + } + + # Split the attributes and values into fields. + # This is the bulk of processing that adds all of + # the RADIUS attributes as elasticsearch fields. + + # Note issue https://github.com/logstash-plugins/logstash-filter-kv/issues/10 + # currently means that all spaces will be stripped + # from all fields. If this is a problem, adjust the + # trim setting. + + kv { + field_split => "\n" + source => "message" + trim_value => "\" " + trim_key => "\t " + } + + # Now we try and add some useful additional + # information. If certain fields can be broken + # down into components then do that here and add + # the data as sub-fields. For example, + # Called-Station-Id might be able to be broken + # down to Called-Station-Id_mac and Called-Station-Id_ssid + # on some wireless systems, or to _ip and _port + # with a VPN. + + # Multiple calls to grok otherwise it can stop + # processing once it has matched one field, but + # e.g. you want to pull both IP and port out of + # the same field in two different regex's. + + # Pull out some IP addresses as field_ip: + + grok { + break_on_match => false + tag_on_failure => [] + match => [ + "Framed-IP-Address", "^(?<Framed-IP-Address_ip>\d+\.\d+\.\d+\.\d+$)", + "NAS-IP-Address", "^(?<NAS-IP-Address_ip>\d+\.\d+\.\d+\.\d+$)", + "Calling-Station-Id", "^(?<Calling-Station-Id_ip>\d+\.\d+\.\d+\.\d+)", + "Called-Station-Id", "^(?<Called-Station-Id_ip>\d+\.\d+\.\d+\.\d+)" + ] + } + + # Split User-Name, Operator-Name, and pull out + # some IP ports if they are there: + + grok { + break_on_match => false + tag_on_failure => [] + match => [ + "User-Name", "^(?<User-Name_username>[^@]+)?(?:@(?<User-Name_realm>[^@]+))$", + "Operator-Name", "^(?<Operator-Name_id>.)(?<Operator-Name_value>.+)$", + + "Calling-Station-Id", "\[(?<Calling-Station-Id_port>\d+)\]$", + "Called-Station-Id", "\[(?<Called-Station-Id_port>\d+)\]$" + ] + } + + # Extract MAC addresses (and SSIDs if there). + # MAC address matching here is lazy, but should be + # good enough. + + grok { + break_on_match => false + tag_on_failure => [] + match => [ + "Calling-Station-Id", "^(?<Calling-Station-Id_mac>[a-fA-F0-9:-]{17})$", + "Calling-Station-Id", "^(?<Calling-Station-Id_mac>[a-fA-F0-9\.]{14})$", + "Calling-Station-Id", "^(?<Calling-Station-Id_mac>[a-fA-F0-9]{12})$", + + "Called-Station-Id", "^(?<Called-Station-Id_mac>[a-fA-F0-9:-]{17})(?::(?<Called-Station-Id_ssid>.*))?$", + "Called-Station-Id", "^(?<Called-Station-Id_mac>[a-fA-F0-9\.]{14})(?::(?<Called-Station-Id_ssid>.*))?$", + "Called-Station-Id", "^(?<Called-Station-Id_mac>[a-fA-F0-9]{12})(?::(?<Called-Station-Id_ssid>.*))?$" + ] + } + + # With the optional sanitize_mac plugin, it's + # possible to make sure all MAC addresses look the + # same, which has obvious benefits. + # + # https://github.com/mcnewton/logstash-filter-sanitize_mac + + # sanitize_mac { + # match => { + # "Called-Station-Id_mac" => "Called-Station-Id_mac" + # "Calling-Station-Id_mac" => "Calling-Station-Id_mac" + # } + # separator => "-" + # fixcase => "lower" + # } + + + # Gigawords presents an issue because the 64-bit + # value is split across two attributes. Combine + # them both back into a single attribute so that + # the full value is available to use. + + if ([Acct-Input-Octets]) { + ruby { + code => "event.set('Acct-Input-Octets_long', event.get('Acct-Input-Octets').to_i + + (event.get('Acct-Input-Gigawords') ? (event.get('Acct-Input-Gigawords').to_i * (2**32)) : 0))" + } + } + + if ([Acct-Output-Octets]) { + ruby { + code => "event.set('Acct-Output-Octets_long', event.get('Acct-Output-Octets').to_i + + (event.get('Acct-Output-Gigawords') ? (event.get('Acct-Output-Gigawords').to_i * (2**32)) : 0))" + } + } + + + # Remove the original "message" field. + + mutate { + remove_field => ["message"] + } + + } +} + + + +# Output data to the local elasticsearch cluster +# using type "detail" in index "radius-DATE". + +output { + if [type] == "radiusdetail" { + elasticsearch { + index => "radius-%{+YYYY.MM.dd}" + } + } +} diff --git a/doc/schemas/logstash/radius-mapping.sh b/doc/schemas/logstash/radius-mapping.sh new file mode 100755 index 0000000..0ee9a3f --- /dev/null +++ b/doc/schemas/logstash/radius-mapping.sh @@ -0,0 +1,100 @@ +#! /bin/sh + +# Create an elasticsearch template mapping for RADIUS data +# Matthew Newton +# April 2019 + +# This should be run on an elasticsearch node. Alternatively, +# adjust the curl URI below. + +# This version has been tested on elasticsearch 6.7.0 + +# The template will be called "radius", and will apply to all +# indices prefixed with "radius-". +# +# As not all RADIUS attributes are known to begin with it has the +# following starting point that can be modified to suit the local +# configuration: +# +# Acct-Input- or Acct-Output- attributes are numbers; +# Acct-Session-Time is a number; +# Everything else is a keyword, which is a non-analysed string. + +# Additionally, the supplied logstash config will try and extract +# MAC addresses, IP addresses and ports from the data. These are +# stored with suffixes on the respective attribute. For example, +# an attribute +# +# Called-Station-Id := "10.0.4.6[4500]" +# +# will be broken down into the following fields in elasticsearch: +# +# Called-Station-Id = "10.0.4.6[4500]" +# Called-Station-Id_ip = "10.0.4.6" +# Called-Station-Id_port = "4500" +# +# This mapping ensures that these have an appropriate data type. + + +curl -s -XPUT -H 'Content-Type: application/json' '127.0.0.1:9200/_template/radius' -d ' +{ + "template":"radius-*", + "order":0, + "mappings":{ + "doc":{ + + "properties": { + "@timestamp": { "format" : "date_optional_time", "type" : "date" }, + "@version": { "type" : "keyword" }, + "message": { "type" : "text" }, + "Acct-Session-Time": { "type" : "long" }, + "offset": { "type" : "long" } + }, + + "dynamic_templates": [ + + { "acct_io_numbers": { + "match_pattern": "regex", + "match": "^Acct-(Input|Output)-.*$", + "mapping": { + "type": "long" + } + } + }, + + { "ipv4_address": { + "path_match": "*_ip", + "mapping": { + "type": "ip" + } + } + }, + + { "network_port": { + "path_match": "*_port", + "mapping": { + "type": "integer" + } + } + }, + + { "long_number": { + "path_match": "*_long", + "mapping": { + "type": "long" + } + } + }, + + { "no_analyze_strings": { + "match": "*", + "mapping": { + "type": "keyword" + } + } + } + + ] + } + } +}' |