Merging upstream version 1.45.3+dfsg.

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

5 files changed, 118 insertions, 0 deletions

diff --git a/src/fluent-bit/examples/perf_test/.env b/src/fluent-bit/examples/perf_test/.env
new file mode 100644
index 000000000..c24399e96
--- /dev/null
+++ b/src/fluent-bit/examples/perf_test/.env
@@ -0,0 +1,5 @@
+FB_BASELINE_IMAGE=fluent/fluent-bit:latest
+LOG_PREFIX=test
+LOG_COUNT=100
+LOG_RATE=20
+LOG_SIZE=1000
diff --git a/src/fluent-bit/examples/perf_test/docker-compose.yml b/src/fluent-bit/examples/perf_test/docker-compose.yml
new file mode 100644
index 000000000..9f67440ad
--- /dev/null
+++ b/src/fluent-bit/examples/perf_test/docker-compose.yml
@@ -0,0 +1,38 @@
+version: "3.4"
+
+services:
+
+  fb-baseline:
+    image: ${FB_BASELINE_IMAGE}
+    volumes:
+      - log-volume:/logs:ro
+      - ./fluent-bit.conf:/fluent-bit/etc/fluent-bit.conf:ro
+
+  fb-delta:
+    build:
+      context: ../..
+      dockerfile: dockerfiles/Dockerfile
+      target: production
+    volumes:
+      - log-volume:/logs:ro
+      - ./fluent-bit.conf:/fluent-bit/etc/fluent-bit.conf:ro
+    ports:
+      - "2020:2020"
+
+  data-generator:
+    image: fluentbitdev/fluent-bit-ci:benchmark
+    command: bash /scripts/entrypoint.sh
+    volumes:
+      - log-volume:/logs:rw
+      - ./scripts:/scripts:ro
+    environment:
+      LOG_PREFIX: ${LOG_PREFIX}
+      LOG_COUNT: ${LOG_COUNT}
+      LOG_SIZE: ${LOG_SIZE}
+      # For multiline set this to >0
+      LINE_COUNT: ${LINE_COUNT}
+      # For multiline set this to the sleep period between log entries rather than the rate
+      LOG_RATE: ${LOG_RATE}
+
+volumes:
+  log-volume:
diff --git a/src/fluent-bit/examples/perf_test/fluent-bit.conf b/src/fluent-bit/examples/perf_test/fluent-bit.conf
new file mode 100644
index 000000000..3defa36e2
--- /dev/null
+++ b/src/fluent-bit/examples/perf_test/fluent-bit.conf
@@ -0,0 +1,27 @@
+[SERVICE]
+    Daemon Off
+    # Log_Level debug
+    HTTP_Server On
+    # HTTP_Listen 0.0.0.0
+    HTTP_Port 2020
+    flush        1
+    storage.metrics           on
+    storage.path              /data/
+    storage.sync              normal
+    storage.checksum          off
+    storage.backlog.mem_limit 1G
+    storage.max_chunks_up     128
+
+[INPUT]
+    Name dummy
+    fixed_timestamp on
+
+[INPUT]
+    Name tail
+    Path /logs/*
+    # Intended for use with multiline data generator
+    multiline.parser java
+
+[OUTPUT]
+    Name null
+    Match *
diff --git a/src/fluent-bit/examples/perf_test/scripts/entrypoint.sh b/src/fluent-bit/examples/perf_test/scripts/entrypoint.sh
new file mode 100755
index 000000000..4b53d4c70
--- /dev/null
+++ b/src/fluent-bit/examples/perf_test/scripts/entrypoint.sh
@@ -0,0 +1,25 @@
+#!/bin/bash
+set -eu
+
+LOG_DIR=${LOG_DIR:-/logs}
+LOG_PREFIX=${LOG_PREFIX:-multi}
+LOG_COUNT=${LOG_COUNT:-100}
+LOG_RATE=${LOG_RATE:-20}
+LOG_SIZE=${LOG_SIZE:-1000}
+LINE_COUNT=${LINE_COUNT:-0}
+
+rm -vfr "${LOG_DIR:?}/$LOG_PREFIX*"
+
+for i in $(seq "$LOG_COUNT")
+do
+    export OUTPUT_LOGFILE="$LOG_DIR/$LOG_PREFIX-$i.log"
+    if [[ "$LINE_COUNT" -gt 0 ]]; then
+        /scripts/multi-line-log-generator.sh &
+    else
+        echo "Creating $OUTPUT_LOGFILE"
+        # Far too much debug
+        /run_log_generator.py --log-size-in-bytes "$LOG_SIZE" --log-rate "$LOG_RATE" --log-agent-input-type tail --tail-file-path "$OUTPUT_LOGFILE" &> /dev/null &
+    fi
+done
+
+wait
diff --git a/src/fluent-bit/examples/perf_test/scripts/multi-line-log-generator.sh b/src/fluent-bit/examples/perf_test/scripts/multi-line-log-generator.sh
new file mode 100755
index 000000000..1e6236d48
--- /dev/null
+++ b/src/fluent-bit/examples/perf_test/scripts/multi-line-log-generator.sh
@@ -0,0 +1,23 @@
+#!/bin/bash
+set -eu
+
+
+OUTPUT_LOGFILE=${OUTPUT_LOGFILE:-/logs/test.log}
+rm -fv "$OUTPUT_LOGFILE"
+
+LOG_RATE=${LOG_RATE:-0.2}
+LINE_COUNT=${LINE_COUNT:-100}
+
+echo "Sleep for $LOG_RATE and create $OUTPUT_LOGFILE with $LINE_COUNT+1 lines per entry"
+
+while true; do
+    cat >> "$OUTPUT_LOGFILE" << EOF
+Exception in thread "main" java.lang.RuntimeException: A test exception
+EOF
+    for _ in $(seq "$LINE_COUNT"); do
+cat >> "$OUTPUT_LOGFILE" << EOF
+  at com.stackify.stacktrace.StackTraceExample.methodB(StackTraceExample.java:13)
+EOF
+    done
+    sleep "$LOG_RATE"
+done