diff options
Diffstat (limited to '')
30 files changed, 1438 insertions, 0 deletions
diff --git a/fluent-bit/conf/fluent-bit-macos.conf b/fluent-bit/conf/fluent-bit-macos.conf new file mode 100644 index 00000000..05da7ee6 --- /dev/null +++ b/fluent-bit/conf/fluent-bit-macos.conf @@ -0,0 +1,90 @@ +[SERVICE] + # Flush + # ===== + # set an interval of seconds before to flush records to a destination + flush 1 + + # Daemon + # ====== + # instruct Fluent Bit to run in foreground or background mode. + daemon Off + + # Log_Level + # ========= + # Set the verbosity level of the service, values can be: + # + # - error + # - warning + # - info + # - debug + # - trace + # + # by default 'info' is set, that means it includes 'error' and 'warning'. + log_level info + + # Parsers File + # ============ + # specify an optional 'Parsers' configuration file + parsers_file parsers.conf + + # Plugins File + # ============ + # specify an optional 'Plugins' configuration file to load external plugins. + plugins_file plugins.conf + + # HTTP Server + # =========== + # Enable/Disable the built-in HTTP Server for metrics + http_server Off + http_listen 0.0.0.0 + http_port 2020 + + # Storage + # ======= + # Fluent Bit can use memory and filesystem buffering based mechanisms + # + # - https://docs.fluentbit.io/manual/administration/buffering-and-storage + # + # storage metrics + # --------------- + # publish storage pipeline metrics in '/api/v1/storage'. The metrics are + # exported only if the 'http_server' option is enabled. + # + storage.metrics on + + # storage.path + # ------------ + # absolute file system path to store filesystem data buffers (chunks). + # + # storage.path /tmp/storage + + # storage.sync + # ------------ + # configure the synchronization mode used to store the data into the + # filesystem. It can take the values normal or full. + # + # storage.sync normal + + # storage.checksum + # ---------------- + # enable the data integrity check when writing and reading data from the + # filesystem. The storage layer uses the CRC32 algorithm. + # + # storage.checksum off + + # storage.backlog.mem_limit + # ------------------------- + # if storage.path is set, Fluent Bit will look for data chunks that were + # not delivered and are still in the storage layer, these are called + # backlog data. This option configure a hint of maximum value of memory + # to use when processing these records. + # + # storage.backlog.mem_limit 5M + +[INPUT] + name dummy + tag dummy.local + +[OUTPUT] + name stdout + match * diff --git a/fluent-bit/conf/fluent-bit-metrics.conf b/fluent-bit/conf/fluent-bit-metrics.conf new file mode 100644 index 00000000..988e0dcb --- /dev/null +++ b/fluent-bit/conf/fluent-bit-metrics.conf @@ -0,0 +1,27 @@ +# Node Exporter Metrics + Prometheus Exporter +# ------------------------------------------- +# The following example collect host metrics on Linux and expose +# them through a Prometheus HTTP end-point. +# +# After starting the service try it with: +# +# $ curl http://127.0.0.1:2021/metrics +# +[SERVICE] + flush 1 + log_level info + +[INPUT] + name node_exporter_metrics + tag node_metrics + scrape_interval 2 + +[OUTPUT] + name prometheus_exporter + match node_metrics + listen 0.0.0.0 + port 2021 + # Add user-defined labels + # ----------------------- + # add_label app fluent-bit + # add_label color blue diff --git a/fluent-bit/conf/fluent-bit-win32.conf b/fluent-bit/conf/fluent-bit-win32.conf new file mode 100644 index 00000000..a27ac5da --- /dev/null +++ b/fluent-bit/conf/fluent-bit-win32.conf @@ -0,0 +1,91 @@ +[SERVICE] + # Flush + # ===== + # set an interval of seconds before to flush records to a destination + flush 1 + + # Daemon + # ====== + # instruct Fluent Bit to run in foreground or background mode. + daemon Off + + # Log_Level + # ========= + # Set the verbosity level of the service, values can be: + # + # - error + # - warning + # - info + # - debug + # - trace + # + # by default 'info' is set, that means it includes 'error' and 'warning'. + log_level info + + # Parsers File + # ============ + # specify an optional 'Parsers' configuration file + parsers_file parsers.conf + + # Plugins File + # ============ + # specify an optional 'Plugins' configuration file to load external plugins. + plugins_file plugins.conf + + # HTTP Server + # =========== + # Enable/Disable the built-in HTTP Server for metrics + http_server Off + http_listen 0.0.0.0 + http_port 2020 + + # Storage + # ======= + # Fluent Bit can use memory and filesystem buffering based mechanisms + # + # - https://docs.fluentbit.io/manual/administration/buffering-and-storage + # + # storage metrics + # --------------- + # publish storage pipeline metrics in '/api/v1/storage'. The metrics are + # exported only if the 'http_server' option is enabled. + # + storage.metrics on + + # storage.path + # ------------ + # absolute file system path to store filesystem data buffers (chunks). + # + # storage.path /tmp/storage + + # storage.sync + # ------------ + # configure the synchronization mode used to store the data into the + # filesystem. It can take the values normal or full. + # + # storage.sync normal + + # storage.checksum + # ---------------- + # enable the data integrity check when writing and reading data from the + # filesystem. The storage layer uses the CRC32 algorithm. + # + # storage.checksum off + + # storage.backlog.mem_limit + # ------------------------- + # if storage.path is set, Fluent Bit will look for data chunks that were + # not delivered and are still in the storage layer, these are called + # backlog data. This option configure a hint of maximum value of memory + # to use when processing these records. + # + # storage.backlog.mem_limit 5M + +[INPUT] + Name winlog + Channels Setup,Windows PowerShell + Interval_Sec 1 + +[OUTPUT] + name stdout + match * diff --git a/fluent-bit/conf/fluent-bit.conf b/fluent-bit/conf/fluent-bit.conf new file mode 100644 index 00000000..bf3269f3 --- /dev/null +++ b/fluent-bit/conf/fluent-bit.conf @@ -0,0 +1,93 @@ +[SERVICE] + # Flush + # ===== + # set an interval of seconds before to flush records to a destination + flush 1 + + # Daemon + # ====== + # instruct Fluent Bit to run in foreground or background mode. + daemon Off + + # Log_Level + # ========= + # Set the verbosity level of the service, values can be: + # + # - error + # - warning + # - info + # - debug + # - trace + # + # by default 'info' is set, that means it includes 'error' and 'warning'. + log_level info + + # Parsers File + # ============ + # specify an optional 'Parsers' configuration file + parsers_file parsers.conf + + # Plugins File + # ============ + # specify an optional 'Plugins' configuration file to load external plugins. + plugins_file plugins.conf + + # HTTP Server + # =========== + # Enable/Disable the built-in HTTP Server for metrics + http_server Off + http_listen 0.0.0.0 + http_port 2020 + + # Storage + # ======= + # Fluent Bit can use memory and filesystem buffering based mechanisms + # + # - https://docs.fluentbit.io/manual/administration/buffering-and-storage + # + # storage metrics + # --------------- + # publish storage pipeline metrics in '/api/v1/storage'. The metrics are + # exported only if the 'http_server' option is enabled. + # + storage.metrics on + + # storage.path + # ------------ + # absolute file system path to store filesystem data buffers (chunks). + # + # storage.path /tmp/storage + + # storage.sync + # ------------ + # configure the synchronization mode used to store the data into the + # filesystem. It can take the values normal or full. + # + # storage.sync normal + + # storage.checksum + # ---------------- + # enable the data integrity check when writing and reading data from the + # filesystem. The storage layer uses the CRC32 algorithm. + # + # storage.checksum off + + # storage.backlog.mem_limit + # ------------------------- + # if storage.path is set, Fluent Bit will look for data chunks that were + # not delivered and are still in the storage layer, these are called + # backlog data. This option configure a hint of maximum value of memory + # to use when processing these records. + # + # storage.backlog.mem_limit 5M + +[INPUT] + name cpu + tag cpu.local + + # Read interval (sec) Default: 1 + interval_sec 1 + +[OUTPUT] + name stdout + match * diff --git a/fluent-bit/conf/in_disk.conf b/fluent-bit/conf/in_disk.conf new file mode 100644 index 00000000..55d0b3cd --- /dev/null +++ b/fluent-bit/conf/in_disk.conf @@ -0,0 +1,61 @@ +[SERVICE] + # Flush + # ===== + # Set an interval of seconds before to flush records to a destination + Flush 5 + + # Daemon + # ====== + # Instruct Fluent Bit to run in foreground or background mode. + Daemon Off + + # Log_Level + # ========= + # Set the verbosity level of the service, values can be: + # + # - error + # - warning + # - info + # - debug + # - trace + # + # By default 'info' is set, that means it includes 'error' and 'warning'. + Log_Level info + + # HTTP Monitoring Server + # ====================== + # + # HTTP_Monitor: enable/disable the HTTP Server to monitor + # Fluent Bit internals. + # HTTP_Port : specify the TCP port of the HTTP Server + HTTP_Monitor Off + HTTP_Port 2020 + +# Head Input +# ========== +[INPUT] + Name disk + Tag disk.all + + # Dev_Name + # ==== + # To limit the target disk/partition. + # If not set, the plugin gathers from all of disks and partitions. + # Dev_Name sda + + # Total Interval + # = Interval Sec + ( Interval Nsec / 1000 / 1000 / 1000 ) + # + # Interval Sec + # ==== + # Read interval (sec) Default: 1 + Interval_Sec 1 + + # Interval NSec + # ==== + # Read interval (nsec) Default: 0 + Interval_NSec 0 + +[OUTPUT] + Name stdout + Match disk.* diff --git a/fluent-bit/conf/in_dummy.conf b/fluent-bit/conf/in_dummy.conf new file mode 100644 index 00000000..abdb2069 --- /dev/null +++ b/fluent-bit/conf/in_dummy.conf @@ -0,0 +1,54 @@ +[SERVICE] + # Flush + # ===== + # Set an interval of seconds before to flush records to a destination + Flush 5 + + # Daemon + # ====== + # Instruct Fluent Bit to run in foreground or background mode. + Daemon Off + + # Log_Level + # ========= + # Set the verbosity level of the service, values can be: + # + # - error + # - warning + # - info + # - debug + # - trace + # + # By default 'info' is set, that means it includes 'error' and 'warning'. + Log_Level info + + # HTTP Monitoring Server + # ====================== + # + # HTTP_Monitor: enable/disable the HTTP Server to monitor + # Fluent Bit internals. + # HTTP_Port : specify the TCP port of the HTTP Server + HTTP_Monitor Off + HTTP_Port 2020 + +[INPUT] + Name dummy + Tag dummy.data + + # Dummy + # ==== + # JSON string. + # Default : {"message":"dummy"} + Dummy {"this is":"dummy data"} + + # Rate + # ==== + # Rate = Message / second + # Default : 1 + Rate 1 + +[OUTPUT] + Name stdout + Match * + +
\ No newline at end of file diff --git a/fluent-bit/conf/in_forward.conf b/fluent-bit/conf/in_forward.conf new file mode 100644 index 00000000..2b40f256 --- /dev/null +++ b/fluent-bit/conf/in_forward.conf @@ -0,0 +1,51 @@ +[SERVICE] + # Flush + # ===== + # Set an interval of seconds before to flush records to a destination + Flush 5 + + # Daemon + # ====== + # Instruct Fluent Bit to run in foreground or background mode. + Daemon Off + + # Log_Level + # ========= + # Set the verbosity level of the service, values can be: + # + # - error + # - warning + # - info + # - debug + # - trace + # + # By default 'info' is set, that means it includes 'error' and 'warning'. + Log_Level info + + # HTTP Monitoring Server + # ====================== + # + # HTTP_Monitor: enable/disable the HTTP Server to monitor + # Fluent Bit internals. + # HTTP_Port : specify the TCP port of the HTTP Server + HTTP_Monitor Off + HTTP_Port 2020 + +[INPUT] + Name forward + + # The Listen interface, by default we listen on all of them + Listen 0.0.0.0 + + # Default TCP listener port + Port 24224 + + # Buffer (Kilobytes) + # ------------------ + # Specify the size of the receiver buffer. Incoming records + # must be inside this limit. By default 512KB. + Buffer 512000 + +[OUTPUT] + Name stdout + Match ** diff --git a/fluent-bit/conf/in_head.conf b/fluent-bit/conf/in_head.conf new file mode 100644 index 00000000..e3f6ab32 --- /dev/null +++ b/fluent-bit/conf/in_head.conf @@ -0,0 +1,86 @@ +[SERVICE] + # Flush + # ===== + # Set an interval of seconds before to flush records to a destination + Flush 5 + + # Daemon + # ====== + # Instruct Fluent Bit to run in foreground or background mode. + Daemon Off + + # Log_Level + # ========= + # Set the verbosity level of the service, values can be: + # + # - error + # - warning + # - info + # - debug + # - trace + # + # By default 'info' is set, that means it includes 'error' and 'warning'. + Log_Level info + + # HTTP Monitoring Server + # ====================== + # + # HTTP_Monitor: enable/disable the HTTP Server to monitor + # Fluent Bit internals. + # HTTP_Port : specify the TCP port of the HTTP Server + HTTP_Monitor Off + HTTP_Port 2020 + +# Head Input +# ========== +[INPUT] + Name head + Tag head.path_to_file + + # File + # ==== + # File path. e.g. /proc/uptime (required) + # + File /path/to/file + + # Buf_Size + # ==== + # Buffer size to read file. Default: 256 + Buf_Size 256 + + # Total Interval + # = Interval Sec + ( Interval Nsec / 1000 / 1000 / 1000 ) + # + # Interval Sec + # ==== + # Read interval (sec) Default: 1 + Interval_Sec 1 + + # Interval NSec + # ==== + # Read interval (nsec) Default: 0 + Interval_NSec 0 + + # Add Path + # ==== + # if true, append file path to each record. Default: false + Add_Path true + + # Key + # ==== + # Rename key Default: head + Key head + + # Lines + # ==== + # Lines to read. If sets, in_head works like 'head -n' + Lines 10 + + # Split_line + # ==== + # If true, in_head splits lines into k-v pairs + Split_line true + +[OUTPUT] + Name stdout + Match head.* diff --git a/fluent-bit/conf/in_mqtt.conf b/fluent-bit/conf/in_mqtt.conf new file mode 100644 index 00000000..3e3bc42e --- /dev/null +++ b/fluent-bit/conf/in_mqtt.conf @@ -0,0 +1,46 @@ +[SERVICE] + # Flush + # ===== + # Set an interval of seconds before to flush records to a destination + Flush 5 + + # Daemon + # ====== + # Instruct Fluent Bit to run in foreground or background mode. + Daemon Off + + # Log_Level + # ========= + # Set the verbosity level of the service, values can be: + # + # - error + # - warning + # - info + # - debug + # - trace + # + # By default 'info' is set, that means it includes 'error' and 'warning'. + Log_Level info + + # HTTP Monitoring Server + # ====================== + # + # HTTP_Monitor: enable/disable the HTTP Server to monitor + # Fluent Bit internals. + # HTTP_Port : specify the TCP port of the HTTP Server + HTTP_Monitor Off + HTTP_Port 2020 + +[INPUT] + Name mqtt + Tag mqtt.local + + # The Listen interface, by default we listen on all of them + Listen 0.0.0.0 + + # Default MQTT TCP port + Port 1883 + +[OUTPUT] + Name stdout + Match * diff --git a/fluent-bit/conf/in_netif.conf b/fluent-bit/conf/in_netif.conf new file mode 100644 index 00000000..dc1299f9 --- /dev/null +++ b/fluent-bit/conf/in_netif.conf @@ -0,0 +1,53 @@ +[SERVICE] + # Flush + # ===== + # Set an interval of seconds before to flush records to a destination + Flush 5 + + # Daemon + # ====== + # Instruct Fluent Bit to run in foreground or background mode. + Daemon Off + + # Log_Level + # ========= + # Set the verbosity level of the service, values can be: + # + # - error + # - warning + # - info + # - debug + # - trace + # + # By default 'info' is set, that means it includes 'error' and 'warning'. + Log_Level info + + # HTTP Monitoring Server + # ====================== + # + # HTTP_Monitor: enable/disable the HTTP Server to monitor + # Fluent Bit internals. + # HTTP_Port : specify the TCP port of the HTTP Server + HTTP_Monitor Off + HTTP_Port 2020 + +[INPUT] + Name netif + Tag eth0.traffic + + # Interface + # ==== + # Specify network interface to count packet. Required. + Interface eth0 + + # Verbose + # ==== + # count up all status. + # Default : false + Verbose false + +[OUTPUT] + Name stdout + Match * + +
\ No newline at end of file diff --git a/fluent-bit/conf/in_proc.conf b/fluent-bit/conf/in_proc.conf new file mode 100644 index 00000000..f1d742d0 --- /dev/null +++ b/fluent-bit/conf/in_proc.conf @@ -0,0 +1,74 @@ +[SERVICE] + # Flush + # ===== + # Set an interval of seconds before to flush records to a destination + Flush 5 + + # Daemon + # ====== + # Instruct Fluent Bit to run in foreground or background mode. + Daemon Off + + # Log_Level + # ========= + # Set the verbosity level of the service, values can be: + # + # - error + # - warning + # - info + # - debug + # - trace + # + # By default 'info' is set, that means it includes 'error' and 'warning'. + Log_Level info + + # HTTP Monitoring Server + # ====================== + # + # HTTP_Monitor: enable/disable the HTTP Server to monitor + # Fluent Bit internals. + # HTTP_Port : specify the TCP port of the HTTP Server + HTTP_Monitor Off + HTTP_Port 2020 + +# Head Input +# ========== +[INPUT] + Name proc + Tag process.fluent-bit + + # Proc_Name + # ==== + # Process Name. e.g. sshd + # + Proc_Name fluent-bit + + + # Alert + # ==== + # if true, flush records when process doesn't work + # Default : false + Alert false + + # Mem + # ==== + # if true, reports memory usage of the process. + # Default : true + Alert false + + # Total Interval + # = Interval Sec + ( Interval Nsec / 1000 / 1000 / 1000 ) + # + # Interval Sec + # ==== + # Read interval (sec) Default :1 + Interval_Sec 1 + + # Interval NSec + # ==== + # Read interval (nsec) Default :0 + Interval_NSec 0 + +[OUTPUT] + Name stdout + Match process.* diff --git a/fluent-bit/conf/in_xbee.conf b/fluent-bit/conf/in_xbee.conf new file mode 100644 index 00000000..a978ed24 --- /dev/null +++ b/fluent-bit/conf/in_xbee.conf @@ -0,0 +1,46 @@ +[SERVICE] + # Flush + # ===== + # Set an interval of seconds before to flush records to a destination + Flush 5 + + # Daemon + # ====== + # Instruct Fluent Bit to run in foreground or background mode. + Daemon Off + + # Log_Level + # ========= + # Set the verbosity level of the service, values can be: + # + # - error + # - warning + # - info + # - debug + # - trace + # + # By default 'info' is set, that means it includes 'error' and 'warning'. + Log_Level trace + +# XBee Input +# ========== +# This configuration file specify the information to be used +# when gathering data from XBee input plugin. All key fields +# in the 'XBEE' section are mandatory. +[INPUT] + Name xbee + Tag xbee + + # File + # ==== + # Filename of serial port. e.g. /dev/ttyS0, /dev/ttyAMA0 + File /dev/ttyUSB0 + + # Baudrate + # ======== + # Specify the bitrate to communicate using the port. + Baudrate 9600 + +[OUTPUT] + Name stdout + Match * diff --git a/fluent-bit/conf/kube.conf b/fluent-bit/conf/kube.conf new file mode 100644 index 00000000..ab7ef150 --- /dev/null +++ b/fluent-bit/conf/kube.conf @@ -0,0 +1,20 @@ +[SERVICE] + Flush 1 + Daemon Off + Log_Level info + Parsers_File parsers.conf + +[INPUT] + Name tail + Path /var/log/containers/*.log + Parser docker + Tag kube.* + Mem_Buf_Limit 5MB + +[FILTER] + Name kubernetes + Match kube.* + +[OUTPUT] + Name stdout + Match * diff --git a/fluent-bit/conf/kube_elasticsearch.conf b/fluent-bit/conf/kube_elasticsearch.conf new file mode 100644 index 00000000..50ca2e10 --- /dev/null +++ b/fluent-bit/conf/kube_elasticsearch.conf @@ -0,0 +1,23 @@ +[SERVICE] + Flush 5 + Daemon Off + Log_Level info + Parsers_File parsers.conf + +[INPUT] + Name tail + Path /var/log/containers/*.log + Parser docker + Tag kube.* + Mem_Buf_Limit 5MB + +[FILTER] + Name kubernetes + Match kube.* + +[OUTPUT] + Name es + Match * + Logstash_Format On + Retry_Limit False + Logstash_Prefix_Key es_index diff --git a/fluent-bit/conf/out_file.conf b/fluent-bit/conf/out_file.conf new file mode 100644 index 00000000..a6e4cca7 --- /dev/null +++ b/fluent-bit/conf/out_file.conf @@ -0,0 +1,36 @@ +[SERVICE] + Flush 5 + Daemon Off + Log_Level info + +[INPUT] + Name mem + Tag mem.usage + +# filter_stdout to show streaming data +[FILTER] + Name stdout + Match * + +# default format : JSON +[OUTPUT] + Name file + Match * + Path mem.json + +# csv format +[OUTPUT] + Name file + Match * + Path mem.csv + Format csv + #Delimiter , + +# ltsv format +[OUTPUT] + Name file + Match * + Path mem.ltsv + Format ltsv + #Label_Delimiter : + #Delimiter \t diff --git a/fluent-bit/conf/out_flowcounter.conf b/fluent-bit/conf/out_flowcounter.conf new file mode 100644 index 00000000..717688fb --- /dev/null +++ b/fluent-bit/conf/out_flowcounter.conf @@ -0,0 +1,39 @@ +[SERVICE] + # Flush + # ===== + # Set an interval of seconds before to flush records to a destination + Flush 1 + + # Log_Level + # ========= + # Set the verbosity level of the service, values can be: + # + # - error + # - warning + # - info + # - debug + # - trace + # + # By default 'info' is set, that means it includes 'error' and 'warning'. + Log_Level info + +[INPUT] + Name random + Tag flowcounter_test + +# To count records. +[OUTPUT] + Name flowcounter + + # Unit + # ==== + # Unit of measurement. + # second/minute/hour/day Default: minute + Unit second + + Match flowcounter_test + +# To display records. +[OUTPUT] + Name stdout + Match flowcounter_test diff --git a/fluent-bit/conf/out_kafka-rest.conf b/fluent-bit/conf/out_kafka-rest.conf new file mode 100644 index 00000000..5582fa64 --- /dev/null +++ b/fluent-bit/conf/out_kafka-rest.conf @@ -0,0 +1,20 @@ +[SERVICE] + Flush 1 + Daemon Off + Log_Level debug + +[INPUT] + Name cpu + +[OUTPUT] + Name kafka-rest + Match * + Host 127.0.0.1 + Port 8082 + Time_Key @timestamp + Tag_Key _fluent-tag + Include_Tag_Key On + Tag_Key My_Tag_Key + Topic fluent-bit-test + # Partition 0 + # Message_Key abc diff --git a/fluent-bit/conf/out_td.conf b/fluent-bit/conf/out_td.conf new file mode 100644 index 00000000..ab7bdfa6 --- /dev/null +++ b/fluent-bit/conf/out_td.conf @@ -0,0 +1,65 @@ +[SERVICE] + # Flush + # ===== + # Set an interval of seconds before to flush records to a destination + Flush 5 + + # Daemon + # ====== + # Instruct Fluent Bit to run in foreground or background mode. + Daemon Off + + # Log_Level + # ========= + # Set the verbosity level of the service, values can be: + # + # - error + # - warning + # - info + # - debug + # - trace + # + # By default 'info' is set, that means it includes 'error' and 'warning'. + Log_Level info + + # HTTP Monitoring Server + # ====================== + # + # HTTP_Monitor: enable/disable the HTTP Server to monitor + # Fluent Bit internals. + # HTTP_Port : specify the TCP port of the HTTP Server + HTTP_Monitor Off + HTTP_Port 2020 + + +[INPUT] + Name cpu + Tag cpu + +# TreasureData Output +# =================== +# This configuration file specify the information to be used +# when flushing information to TreasureData.com service. All +# key fields in the 'TD' section are mandatory. +[OUTPUT] + Name td + Match * + + # API + # === + # The TreasureData API key. To obtain this please log into your + # TreasureData Console: + # + # 1. Go to https://console.treasuredata.com/ + # 2. API Keys box: copy the API key hash + API 5a00/e75beccae41e9f83dfa1wa35ddfbb0dcd8a8d980 + + # Database + # ======== + # Specify the name of your database, it must exists. + Database fluentbit + + # Table + # ===== + # Specify the database table name where the records will be stored + Table cpu_samples diff --git a/fluent-bit/conf/parser_custom.conf b/fluent-bit/conf/parser_custom.conf new file mode 100644 index 00000000..51e81985 --- /dev/null +++ b/fluent-bit/conf/parser_custom.conf @@ -0,0 +1,15 @@ + [PARSER] + # https://rubular.com/r/6ZCuwV4Xa7nfA3 + Name rabbitmq + Format regex + Regex (?<date>[^ ]+)\s(?<time>[^ ]+)\s\[(?<log_level>[^ \]]*)\]\s(?<PID>[^ ]*)\s(?<msg>((([a-zA-Z]*\s+)+[^ ]*)+)+) + [PARSER] + # https://rubular.com/r/jWfJIOMKr2LgcO + Name neo4j + Format regex + Regex (?<date>[^ ]*) (?<time>[^ ]*) (?<log_level>[^ ]*)\s(?<msg>([^ ]*\s+[^ ]*)+) + [PARSER] + # https://rubular.com/r/U8VbByp0oRPLU6 + Name external-dns + Format regex + Regex ([^ ])\"(?<time>[^ ]+)\"\s([^ ]+)\=(?<log_level>[.+a-zA-Z]+)\s([^ ]+)\"(?<msg>([^ ]*\s+[^ ]*\s[a-zA-Z0-9]*)+) diff --git a/fluent-bit/conf/parsers.conf b/fluent-bit/conf/parsers.conf new file mode 100644 index 00000000..71706545 --- /dev/null +++ b/fluent-bit/conf/parsers.conf @@ -0,0 +1,126 @@ +[PARSER] + Name apache + Format regex + Regex ^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^\"]*?)(?: +\S*)?)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$ + Time_Key time + Time_Format %d/%b/%Y:%H:%M:%S %z + +[PARSER] + Name apache2 + Format regex + Regex ^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>.*)")?$ + Time_Key time + Time_Format %d/%b/%Y:%H:%M:%S %z + +[PARSER] + Name apache_error + Format regex + Regex ^\[[^ ]* (?<time>[^\]]*)\] \[(?<level>[^\]]*)\](?: \[pid (?<pid>[^\]]*)\])?( \[client (?<client>[^\]]*)\])? (?<message>.*)$ + +[PARSER] + Name nginx + Format regex + Regex ^(?<remote>[^ ]*) (?<host>[^ ]*) (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^\"]*?)(?: +\S*)?)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)") + Time_Key time + Time_Format %d/%b/%Y:%H:%M:%S %z + +[PARSER] + # https://rubular.com/r/IhIbCAIs7ImOkc + Name k8s-nginx-ingress + Format regex + Regex ^(?<host>[^ ]*) - (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^\"]*?)(?: +\S*)?)?" (?<code>[^ ]*) (?<size>[^ ]*) "(?<referer>[^\"]*)" "(?<agent>[^\"]*)" (?<request_length>[^ ]*) (?<request_time>[^ ]*) \[(?<proxy_upstream_name>[^ ]*)\] (\[(?<proxy_alternative_upstream_name>[^ ]*)\] )?(?<upstream_addr>[^ ]*) (?<upstream_response_length>[^ ]*) (?<upstream_response_time>[^ ]*) (?<upstream_status>[^ ]*) (?<reg_id>[^ ]*).*$ + Time_Key time + Time_Format %d/%b/%Y:%H:%M:%S %z + +[PARSER] + Name json + Format json + Time_Key time + Time_Format %d/%b/%Y:%H:%M:%S %z + +[PARSER] + Name docker + Format json + Time_Key time + Time_Format %Y-%m-%dT%H:%M:%S.%L + Time_Keep On + # -- + # Since Fluent Bit v1.2, if you are parsing Docker logs and using + # the Kubernetes filter, it's not longer required to decode the + # 'log' key. + # + # Command | Decoder | Field | Optional Action + # =============|==================|================= + #Decode_Field_As json log + +[PARSER] + Name docker-daemon + Format regex + Regex time="(?<time>[^ ]*)" level=(?<level>[^ ]*) msg="(?<msg>[^ ].*)" + Time_Key time + Time_Format %Y-%m-%dT%H:%M:%S.%L + Time_Keep On + +[PARSER] + Name syslog-rfc5424 + Format regex + Regex ^\<(?<pri>[0-9]{1,5})\>1 (?<time>[^ ]+) (?<host>[^ ]+) (?<ident>[^ ]+) (?<pid>[-0-9]+) (?<msgid>[^ ]+) (?<extradata>(\[(.*?)\]|-)) (?<message>.+)$ + Time_Key time + Time_Format %Y-%m-%dT%H:%M:%S.%L%z + Time_Keep On + +[PARSER] + Name syslog-rfc3164-local + Format regex + Regex ^\<(?<pri>[0-9]+)\>(?<time>[^ ]* {1,2}[^ ]* [^ ]*) (?<ident>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? *(?<message>.*)$ + Time_Key time + Time_Format %b %d %H:%M:%S + Time_Keep On + +[PARSER] + Name syslog-rfc3164 + Format regex + Regex /^\<(?<pri>[0-9]+)\>(?<time>[^ ]* {1,2}[^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? *(?<message>.*)$/ + Time_Key time + Time_Format %b %d %H:%M:%S + Time_Keep On + +[PARSER] + Name mongodb + Format regex + Regex ^(?<time>[^ ]*)\s+(?<severity>\w)\s+(?<component>[^ ]+)\s+\[(?<context>[^\]]+)]\s+(?<message>.*?) *(?<ms>(\d+))?(:?ms)?$ + Time_Format %Y-%m-%dT%H:%M:%S.%L + Time_Keep On + Time_Key time + +[PARSER] + # https://rubular.com/r/0VZmcYcLWMGAp1 + Name envoy + Format regex + Regex ^\[(?<start_time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^\"]*?)(?: +\S*)?)? (?<protocol>\S+)" (?<code>[^ ]*) (?<response_flags>[^ ]*) (?<bytes_received>[^ ]*) (?<bytes_sent>[^ ]*) (?<duration>[^ ]*) (?<x_envoy_upstream_service_time>[^ ]*) "(?<x_forwarded_for>[^ ]*)" "(?<user_agent>[^\"]*)" "(?<request_id>[^\"]*)" "(?<authority>[^ ]*)" "(?<upstream_host>[^ ]*)" + Time_Format %Y-%m-%dT%H:%M:%S.%L%z + Time_Keep On + Time_Key start_time + +[PARSER] + # https://rubular.com/r/17KGEdDClwiuDG + Name istio-envoy-proxy + Format regex + Regex ^\[(?<start_time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^\"]*?)(?: +\S*)?)? (?<protocol>\S+)" (?<response_code>[^ ]*) (?<response_flags>[^ ]*) (?<response_code_details>[^ ]*) (?<connection_termination_details>[^ ]*) (?<upstream_transport_failure_reason>[^ ]*) (?<bytes_received>[^ ]*) (?<bytes_sent>[^ ]*) (?<duration>[^ ]*) (?<x_envoy_upstream_service_time>[^ ]*) "(?<x_forwarded_for>[^ ]*)" "(?<user_agent>[^\"]*)" "(?<x_request_id>[^\"]*)" (?<authority>[^ ]*)" "(?<upstream_host>[^ ]*)" (?<upstream_cluster>[^ ]*) (?<upstream_local_address>[^ ]*) (?<downstream_local_address>[^ ]*) (?<downstream_remote_address>[^ ]*) (?<requested_server_name>[^ ]*) (?<route_name>[^ ]*) + Time_Format %Y-%m-%dT%H:%M:%S.%L%z + Time_Keep On + Time_Key start_time + +[PARSER] + # http://rubular.com/r/tjUt3Awgg4 + Name cri + Format regex + Regex ^(?<time>[^ ]+) (?<stream>stdout|stderr) (?<logtag>[^ ]*) (?<message>.*)$ + Time_Key time + Time_Format %Y-%m-%dT%H:%M:%S.%L%z + Time_Keep On + +[PARSER] + Name kube-custom + Format regex + Regex (?<tag>[^.]+)?\.?(?<pod_name>[a-z0-9](?:[-a-z0-9]*[a-z0-9])?(?:\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*)_(?<namespace_name>[^_]+)_(?<container_name>.+)-(?<docker_id>[a-z0-9]{64})\.log$ diff --git a/fluent-bit/conf/parsers_ambassador.conf b/fluent-bit/conf/parsers_ambassador.conf new file mode 100644 index 00000000..3ab00d37 --- /dev/null +++ b/fluent-bit/conf/parsers_ambassador.conf @@ -0,0 +1,6 @@ +# Ambassador - open source Kubernetes-native API gateway for microservices built on the Envoy Proxy https://www.getambassador.io + +[PARSER] + Name ambassador + Format regex + Regex ^(?<type>\S+) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>(?:[^\"]|\\.)*?)(?: +\S*)?) (?<protocol>\S+)?" (?<response_code>\S+) (?<response_flags>\S+) (?<bytes_received>\S+) (?<bytes_sent>\S+) (?<duration>\S+) (?<x_envoy_upstream_service_time>\S+) "(?<x_forwarded_for>[^\"]*)" "(?<user_agent>[^\"]*)" "(?<x_request_id>[^\"]*)" "(?<authority>[^\"]*)" "(?<upstream_host>[^\"]*)" diff --git a/fluent-bit/conf/parsers_cinder.conf b/fluent-bit/conf/parsers_cinder.conf new file mode 100644 index 00000000..b67ee99a --- /dev/null +++ b/fluent-bit/conf/parsers_cinder.conf @@ -0,0 +1,10 @@ + +[PARSER] + # http://rubular.com/r/IvZVElTgNl + Name ceph + Format regex + Regex ^(?<log_time>[^ ][-.\d\+:T]+[ ]*[.:\d]*)\s+(?<message>.*)$ + Time_Format %Y-%m-%d %H:%M:%S.%L + Time_Keep Off + Time_Key log_time + diff --git a/fluent-bit/conf/parsers_extra.conf b/fluent-bit/conf/parsers_extra.conf new file mode 100644 index 00000000..afe48c1a --- /dev/null +++ b/fluent-bit/conf/parsers_extra.conf @@ -0,0 +1,172 @@ +# Extra set of common parsers + +[PARSER] + # http://rubular.com/r/cCVd1HLCAO + Name crowbar + Format regex + Regex ^.*\[(?<log_time>[^ ][-.\d\+:]+T[:\d]*)([^\]])*?\]\s+?(?<severity>[^ ]\w+)([\s-]*):?\s+(?<message>.*) + Time_Format %Y-%m-%dT%H:%M:%S + Time_Keep Off + Time_Key log_time + +[PARSER] + # http://rubular.com/r/frDgnElXW9 + Name chefclient + Format regex + Regex ^\[(?<log_time>[^ ][-.\d\+:]+T[:\d]*)([^\]])*?\]\s+(?<severity>[^ ]\w+):\s+(?<message>.*)$ + Time_Format %Y-%m-%dT%H:%M:%S + Time_Keep Off + Time_Key log_time + +[PARSER] + Name mysql_error + Format regex + #Regex ^(?<log_time>[^ +][ -:0-9TZ]+|[[:upper:]][[:lower:]]{2})(\+\d+:\d+[TZ]*){0,1}\s*(?<myid>[^ ]\d+)\s+\[(?<severity>[^ ]\w+)\](\s+(?<subsystem>[^ ]\w+):){0,1}\s+(?<message>.*)$ + Regex ^(?<log_time>[^ +][-\d]+[\ T]*[:\dZ]+)\s*(?<myid>[^ ]\d+)\s+\[(?<severity>[^ ]\w+)\](\s+(?<subsystem>[^ ]\w+):){0,1}\s+(?<message>.*)$ + Time_Format %Y-%m-%d %H:%M:%S + Time_Keep Off + Time_Key log_time + +[PARSER] + Name mysql_slow + Format regex + Regex ^# User\@Host:\s+(?<user>[^\@][\w\[\]]+)[@\s]+(?<dbhost>[^ ][-.\w]+)\s+(\[(?<dbhost_address>[.\d]+)\]){0,1}\s+(?<message>.*)$ + +[PARSER] + Name pacemaker + Format regex + Regex ^\s*(?<log_time>[^ ]* {1,2}[^ ]* [^ ]*) \[(?<pid>\d+)\] (?<node>[\-\w]*)\s*(?<component>\w*):\s+(?<severity>\w+):\s+(?<message>.*)$ + #Time_Format %Y-%m-%dT%H:%M:%S + Time_Format %b %d %H:%M:%S + Time_Keep Off + Time_Key log_time + #Types pid:integer + +[PARSER] + Name rabbitmq + Format regex + Regex ^=(?<severity>[^ ]\w+)\s+REPORT[=\s]*(?<log_time>[^ =][-:.\d\w]+)[\s=]+(?<message>.*)$ + Time_Format %d-%b-%Y::%H:%M:%S + Time_Keep Off + Time_Key log_time + +[PARSER] + Name http_statement + Format regex + Regex ^.*((?<req_method>GET|POST|PUT|DELETE|CONNECT|OPTIONS|HEAD[^ ]\w+)\s*(?<req_path>[^ ][-._?=%&\/[:alnum:]]*)\s*(?<req_protocol>[^ ][.\/\dHTFSP]+){0,1})(['"\s]*){0,1}((\s*status:\s*(?<req_status>[^ ]\d+)){0,1}(\s*len:\ (?<req_len>[^ ]\d+)){0,1}(\s*time:\s*(?<req_log_time>[^ ][.\d]+)){0,1}(\s*microversion:\s*(?<req_mver>[^ ][.\d]+)){0,1}){0,1}$ + +[PARSER] + Name universal + Format regex + Regex ^(?<message>.*)$ + +[PARSER] + Name uuid + Format regex + Regex (?<uuid>[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-5][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}) +#UUID v1 : +#/^[0-9A-F]{8}-[0-9A-F]{4}-[1][0-9A-F]{3}-[89AB][0-9A-F]{3}-[0-9A-F]{12}$/i +#UUID v2 : +#/^[0-9A-F]{8}-[0-9A-F]{4}-[2][0-9A-F]{3}-[89AB][0-9A-F]{3}-[0-9A-F]{12}$/i +#UUID v3 : +#/^[0-9A-F]{8}-[0-9A-F]{4}-[3][0-9A-F]{3}-[89AB][0-9A-F]{3}-[0-9A-F]{12}$/i +#UUID v4 : +#/^[0-9A-F]{8}-[0-9A-F]{4}-[4][0-9A-F]{3}-[89AB][0-9A-F]{3}-[0-9A-F]{12}$/i +#UUID v5 : +#/^[0-9A-F]{8}-[0-9A-F]{4}-[5][0-9A-F]{3}-[89AB][0-9A-F]{3}-[0-9A-F]{12}$/i + +# Parse IP Tables rules - this one regex should capture pretty much any IP Tables rule and split it into the various fields +[PARSER] + Name iptables + Format regex + Regex \[(?<rule_chain>\w*)-(?<rule_name>\w*)-(?<accept_or_drop>\w*)\]IN=(?<in_interface>[\w.]+)? OUT=(?<out_interface>[\w.]+)? MAC=(?<mac_address>[\w:]+)? SRC=(?<source>(?:[0-9]{1,3}\.){3}[0-9]{1,3}) DST=(?<dest>(?:[0-9]{1,3}\.){3}[0-9]{1,3}) LEN=(?<pkt_len>\d+) TOS=(?<pkt_tos>[\w\d]+) PREC=(?<pkt_prec>[\w\d]+) TTL=(?<pkt_ttl>\d+) ID=(?<pkt_id>\d+)\s?(?<pkg_frg>[A-Z\s].?)\s?PROTO=(?<protocol>[\w\d]+) (SPT=(?<source_port>.*) DPT=(?<dest_port>.*) (LEN=(?<proto_pkt_len>\w+)?)?(WINDOW=(?<proto_window_size>\d+) RES=(?<pkt_res>\w+)? (?<pkt_type>\w+)\s((?<pkt_flag>\w+)?)\s?URGP=(?<pkg_urgency>\d))? )?(TYPE=(?<pkt_icmp_type>\d+) CODE=(?<pkt_icmp_code>\d+) ID=(?<pkt_icmp_id>\d+) SEQ=(?<pkt_icmp_seq>\d+) )?$ + Types source_port:integer,dest_port:integer,pkt_ttl:integer,pkt_tos:integer,pkt_len:integer + +# Various parsers for Couchbase Server logs + +[PARSER] + Name couchbase_json_log_nanoseconds + Format json + Time_Key timestamp + Time_Format %Y-%m-%dT%H:%M:%S.%L + Time_Keep On + # Do not remove the time field from the output we ship + +[PARSER] + Name couchbase_rebalance_report + Format json + Time_Key timestamp + Time_Format %Y-%m-%dT%H:%M:%SZ + Time_Keep On + +# The level may have optional brackets around it +[PARSER] + Name couchbase_simple_log + Format regex + Regex ^(?<timestamp>\d+-\d+-\d+T\d+:\d+:\d+\.\d+(\+|-)\d+:\d+)\s+\[(?<level>\w+)\](?<message>.*)$ + Time_Key timestamp + Time_Format %Y-%m-%dT%H:%M:%S.%L%z + Time_Keep On + +[PARSER] + Name couchbase_simple_log_space_separated + Format regex + Regex ^(?<timestamp>\d+-\d+-\d+T\d+:\d+:\d+\.\d+(\+|-)\d+:\d+)\s+(?<level>\w+)\s+(?<message>.*)$ + Time_Key timestamp + Time_Format %Y-%m-%dT%H:%M:%S.%L%z + Time_Keep On + +# Slight change in time format to use Z at end instead of offset: +# 2021-03-09T17:32:02.136Z INFO ... +# https://rubular.com/r/EpG3M1dHb5AnTC +[PARSER] + Name couchbase_simple_log_utc + Format regex + Regex ^(?<timestamp>\d+-\d+-\d+T\d+:\d+:\d+\.\d+Z)\s+(?<level>\w+)(?<message>.*)$ + Time_Key timestamp + Time_Format %Y-%m-%dT%H:%M:%S.%LZ + Time_Keep On + +# Cope with two different log formats, e.g.: +# 2021/03/09 17:32:15 cbauth: ... +# 2021-03-09T17:32:15.303+00:00 [INFO] ... +# https://rubular.com/r/XUt7xQqEJnrF2M +[PARSER] + Name couchbase_simple_log_mixed + Format regex + Regex ^(?<timestamp>\d+(-|/)\d+(-|/)\d+(T|\s+)\d+:\d+:\d+(\.\d+(\+|-)\d+:\d+|))\s+((\[)?(?<level>\w+)(\]|:))(?<message>.*)$ + Time_Key timestamp + Time_Keep On +# We cannot parse the time as different formats directly, it could be done downstream and/or left as current time + +[PARSER] + Name couchbase_erlang_multiline + Format regex + # For some reason this cannot parse an ending close bracket ] followed by a new line immediately + #Regex \[(?<logger>\w+):(?<level>\w+),(?<timestamp>\d+-\d+-\d+T\d+:\d+:\d+.\d+Z),.*\](?<message>.*)$ + Regex \[(?<logger>\w+):(?<level>\w+),(?<timestamp>\d+-\d+-\d+T\d+:\d+:\d+.\d+Z),(?<message>.*)$ + Time_Key timestamp + Time_Format %Y-%m-%dT%H:%M:%S.%L + Time_Keep On + +# 2021-03-09T17:32:25.339+00:00 INFO CBAS.bootstrap.AnalyticsNCApplication [main] ... +# https://rubular.com/r/9jh1oKtXBN5GEV +# Can include an exception stack trace or a thread dump as well but ignoring these for now +[PARSER] + Name couchbase_java_multiline + Format regex + Regex ^(?<timestamp>\d+-\d+-\d+T\d+:\d+:\d+\.\d+(\+|-)\d+:\d+)\s+(?<level>\w+)\s+(?<class>.*)\s+\[(?<thread>.*)\]\s+(?<message>.*)$ + Time_Key timestamp + Time_Format %Y-%m-%dT%H:%M:%S.%L%z + Time_Keep On + +# A slight modification of the usual Apache/Apache2 parsers +[PARSER] + Name couchbase_http + Format regex + Regex ^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<timestamp>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*) - (?<client>.*)$ + Time_Key timestamp + Time_Format %d/%b/%Y:%H:%M:%S %z + Time_Keep On + +# End of Couchbase Server parsers diff --git a/fluent-bit/conf/parsers_java.conf b/fluent-bit/conf/parsers_java.conf new file mode 100644 index 00000000..b03729a0 --- /dev/null +++ b/fluent-bit/conf/parsers_java.conf @@ -0,0 +1,6 @@ +[PARSER] + Name java_multiline + Format regex + Regex /^(?<time>\d{4}-\d{1,2}-\d{1,2} \d{1,2}:\d{1,2}:\d{1,2}) \[(?<thread>.*)\] (?<level>[^\s]+)(?<message>.*)/ + Time_Key time + Time_Format %Y-%m-%d %H:%M:%S diff --git a/fluent-bit/conf/parsers_kafka.conf b/fluent-bit/conf/parsers_kafka.conf new file mode 100644 index 00000000..db4384a6 --- /dev/null +++ b/fluent-bit/conf/parsers_kafka.conf @@ -0,0 +1,16 @@ +[PARSER] + # Confluent Schema Registry 7.1.1 default format + Name confluent-schema-registry + Format regex + Time_Key time + Time_Format %Y-%m-%d %H:%M:%S,%L + Regex ^\[(?<time>\d{4}-\d{1,2}-\d{1,2} \d{2}:\d{2}:\d{2},\d{1,4})] (?<level>[A-Z]{1,8}) (?<src>[\d.]+) - - \[(?<date>.*)] \"(?<method>[A-Z]+) (?<path>\/[\S]+|\/) (?<http_version>HTTP\/[\d]\.[\d])\" (?<code>[\d]{1,3}) (?<size>[\d]+) \"-\" \"(?<agent>.*)\" (?<extra_info>.*) + + +[PARSER] + # Confluent Schema Registry 7.1.1 default format - with _sr_ prefix to allow `nest` + Name confluent-schema-registry-prefixed + Format regex + Time_Key _sr_time + Time_Format %Y-%m-%d %H:%M:%S,%L + Regex ^\[(?<_sr_time>\d{4}-\d{1,2}-\d{1,2} \d{2}:\d{2}:\d{2},\d{1,4})] (?<_sr_level>[A-Z]{1,8}) (?<_sr_src>[\d.]+) - - \[(?<_sr_date>.*)] \"(?<_sr_method>[A-Z]+) (?<_sr_path>\/[\S]+|\/) (?<_sr_http_version>HTTP\/[\d]\.[\d])\" (?<_sr_code>[\d]{1,3}) (?<_sr_size>[\d]+) \"-\" \"(?<_sr_agent>.*)\" (?<_sr_extra_info>.*) diff --git a/fluent-bit/conf/parsers_mult.conf b/fluent-bit/conf/parsers_mult.conf new file mode 100644 index 00000000..af9025f4 --- /dev/null +++ b/fluent-bit/conf/parsers_mult.conf @@ -0,0 +1,26 @@ +[PARSER] + Name mult_first + Format regex + Regex Started (?<method>[^ ]+) "(?<path>[^"]+)" for (?<host>[^ ]+) at (?<time>[^ ]+ [^ ]+ [^ ]+) + Time_Key time + Time_Format %Y-%m-%d %H:%M:%S %z + +[PARSER] + Name mult_1 + Format regex + Regex /Processing by (?<controller>[^\u0023]+)\u0023(?<controller_method>[^ ]+) as (?<format>[^ ]+?)$/ + +[PARSER] + Name mult_2 + Format regex + Regex ( Parameters: (?<parameters>[^ ]+))? + +[PARSER] + Name mult_3 + Format regex + Regex / Rendered (?<template>[^ ]+) within (?<layout>.+) \([\d\.]+ms\)/ + +[PARSER] + Name mult_4 + Format regex + Regex Completed (?<code>[^ ]+) [^ ]+ in (?<runtime>[\d\.]+)ms \(Views: (?<view_runtime>[\d\.]+)ms \| ActiveRecord: (?<ar_runtime>[\d\.]+)ms\) diff --git a/fluent-bit/conf/parsers_multiline.conf b/fluent-bit/conf/parsers_multiline.conf new file mode 100644 index 00000000..8ac34806 --- /dev/null +++ b/fluent-bit/conf/parsers_multiline.conf @@ -0,0 +1,6 @@ +[MULTILINE_PARSER] + name exception_test + type regex + flush_timeout 1000 + rule "start_state" "/(Dec \d+ \d+\:\d+\:\d+)(.*)/" "cont" + rule "cont" "/^\s+at.*/" "cont" diff --git a/fluent-bit/conf/parsers_openstack.conf b/fluent-bit/conf/parsers_openstack.conf new file mode 100644 index 00000000..e1bbc803 --- /dev/null +++ b/fluent-bit/conf/parsers_openstack.conf @@ -0,0 +1,53 @@ +# Example openstack parsers, see source for recent updates: +# https://github.com/salt-formulas/salt-formula-fluentbit/tree/master/metadata/service/config/parser + +# Take these parsers as an example, as they may not fully fit all possible +# openstack deployments from all vendors, etc.. + +[PARSER] + Name cinder + Format regex + Regex ^(?<log_time>[^ ][-.\d\+:T]+[ ]*[.:\d]*)\s+(?<pid>[^ ]\d+)\s+(?<severity>[^ ][.-_\w]+)\s+(?<component>[^ ][.-_\w]+)(\s+\[(-|(?<req_id>[^ ][-\w]*) (?<req_user>[^ ][-\w]*) (?<req_project>[^ ][-\w]*) (?<req_domain>[^ ][-\w]*) (?<req_user_domain>[^ ][-\w]*) (?<req_project_domain>[^ ][-\w]*))\]){1}\s+(?<message>.*)$ + Time_Format %Y-%m-%d %H:%M:%S.%L + Time_Keep Off + Time_Key log_time + +[PARSER] + Name glance + Format regex + Regex ^(?<log_time>[^ ][-.\d\+:T]+[ ]*[.:\d]*)\s+(?<pid>[^ ]\d+)\s+(?<severity>[^ ][.-_\w]+)\s+(?<component>[^ ][.-_\w]+)(\s+\[(-|(?<req_id>[^ ][-\w]*) (?<req_user>[^ ][-\w]*) (?<req_project>[^ ][-\w]*) (?<req_domain>[^ ][-\w]*) (?<req_user_domain>[^ ][-\w]*) (?<req_project_domain>[^ ][-\w]*))\]){1}\s+(?<message>.*)$ + Time_Format %Y-%m-%d %H:%M:%S.%L + Time_Keep Off + Time_Key log_time + +[PARSER] + Name heat + Format regex + Regex ^(?<log_time>[^ ][-.\d\+:T]+[ ]*[.:\d]*)\s+(?<pid>[^ ]\d+)\s+(?<severity>[^ ][.-_\w]+)\s+(?<component>[^ ][.-_\w]+)(\s+\[(-|(?<req_id>[^ ][-\w]*) (?<req_user>[^ ][-\w]*) (?<req_project>[^ ][-\w]*) (?<req_domain>[^ ][-\w]*) (?<req_user_domain>[^ ][-\w]*) (?<req_project_domain>[^ ][-\w]*))\]){1}\s+(?<message>.*)$ + Time_Format %Y-%m-%d %H:%M:%S.%L + Time_Keep Off + Time_Key log_time + +[PARSER] + Name keystone + Format regex + Regex ^(?<log_time>[^ ][-.\d\+:T]+[ ]*[.:\d]*)\s+(?<pid>[^ ]\d+)\s+(?<severity>[^ ][.-_\w]+)\s+(?<component>[^ ][.-_\w]+)(\s+\[(-|(?<req_id>[^ ][-\w]*) (?<req_user>[^ ][-\w]*) (?<req_project>[^ ][-\w]*) (?<req_domain>[^ ][-\w]*) (?<req_user_domain>[^ ][-\w]*) (?<req_project_domain>[^ ][-\w]*))\]){1}\s+(?<message>.*)$ + Time_Format %Y-%m-%d %H:%M:%S.%L + Time_Keep Off + Time_Key log_time + +[PARSER] + Name neutron + Format regex + Regex ^(?<log_time>[^ ][-.\d\+:T]+[ ]*[.:\d]*)\s+(?<pid>[^ ]\d+)\s+(?<severity>[^ ][.-_\w]+)\s+(?<component>[^ ][.-_\w]+)(\s+\[(-|(?<req_id>[^ ][-\w]*) (?<req_user>[^ ][-\w]*) (?<req_project>[^ ][-\w]*) (?<req_domain>[^ ][-\w]*) (?<req_user_domain>[^ ][-\w]*) (?<req_project_domain>[^ ][-\w]*))\]){1}\s+(?<message>.*)$ + Time_Format %Y-%m-%d %H:%M:%S.%L + Time_Keep Off + Time_Key log_time + +[PARSER] + Name nova + Format regex + Regex ^(?<log_time>[^ ][-.\d\+:T]+[ ]*[.:\d]*)\s+(?<pid>[^ ]\d+)\s+(?<severity>[^ ][.-_\w]+)\s+(?<component>[^ ][.-_\w]+)(\s+\[(-|(?<req_id>[^ ][-\w]*) (?<req_user>[^ ][-\w]*) (?<req_project>[^ ][-\w]*) (?<req_domain>[^ ][-\w]*) (?<req_user_domain>[^ ][-\w]*) (?<req_project_domain>[^ ][-\w]*))\]){1}\s+(?<message>.*)$ + Time_Format %Y-%m-%d %H:%M:%S.%L + Time_Keep Off + Time_Key log_time diff --git a/fluent-bit/conf/plugins.conf b/fluent-bit/conf/plugins.conf new file mode 100644 index 00000000..55c5a8d3 --- /dev/null +++ b/fluent-bit/conf/plugins.conf @@ -0,0 +1,2 @@ +[PLUGINS] + # Path /path/to/out_gstdout.so diff --git a/fluent-bit/conf/rate_limit.conf b/fluent-bit/conf/rate_limit.conf new file mode 100644 index 00000000..9fdc8311 --- /dev/null +++ b/fluent-bit/conf/rate_limit.conf @@ -0,0 +1,25 @@ +[SERVICE] + Flush 1 + Daemon Off + Log_Level info + Parsers_File parsers.conf + +[INPUT] + Name tail + Path /var/log/containers/*.log + Parser docker + Tag kube.* + Mem_Buf_Limit 5MB + +[FILTER] + Name kubernetes + Match kube.* +# This filter calls the rate limit script +[FILTER] + Name lua + Match kube.* + script rate_limit.lua + call rate_limit +[OUTPUT] + Name stdout + Match *
\ No newline at end of file |