diff options
| author | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-04-19 02:57:58 +0000 |
|---|---|---|
| committer | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-04-19 02:57:58 +0000 |
| commit | be1c7e50e1e8809ea56f2c9d472eccd8ffd73a97 (patch) | |
| tree | 9754ff1ca740f6346cf8483ec915d4054bc5da2d /fluent-bit/tests/runtime_shell | |
| parent | Initial commit. (diff) | |
| download | netdata-be1c7e50e1e8809ea56f2c9d472eccd8ffd73a97.tar.xz netdata-be1c7e50e1e8809ea56f2c9d472eccd8ffd73a97.zip | |
Adding upstream version 1.44.3.upstream/1.44.3upstream
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'fluent-bit/tests/runtime_shell')
33 files changed, 1907 insertions, 0 deletions
diff --git a/fluent-bit/tests/runtime_shell/.gitignore b/fluent-bit/tests/runtime_shell/.gitignore new file mode 100644 index 00000000..bf8c81f4 --- /dev/null +++ b/fluent-bit/tests/runtime_shell/.gitignore @@ -0,0 +1,2 @@ +runtime_shell.env + diff --git a/fluent-bit/tests/runtime_shell/CMakeLists.txt b/fluent-bit/tests/runtime_shell/CMakeLists.txt new file mode 100644 index 00000000..79c96de5 --- /dev/null +++ b/fluent-bit/tests/runtime_shell/CMakeLists.txt @@ -0,0 +1,29 @@ +configure_file( + "${CMAKE_CURRENT_SOURCE_DIR}/runtime_shell.env.in" + "${CMAKE_CURRENT_SOURCE_DIR}/runtime_shell.env" + ) + +set(UNIT_TESTS_SH + in_dummy_expect.sh + in_tail_expect.sh + in_http_tls_expect.sh + in_syslog_tcp_tls_expect.sh + in_syslog_tcp_plaintext_expect.sh + in_syslog_udp_plaintext_expect.sh + in_syslog_uds_dgram_plaintext_expect.sh + in_syslog_uds_stream_plaintext_expect.sh + ) + +# Prepare list of unit tests +foreach(script ${UNIT_TESTS_SH}) + add_test(NAME ${script} + COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/${script} + ) + + set_tests_properties(${script} PROPERTIES ENVIRONMENT + "FLB_ROOT=${PROJECT_SOURCE_DIR};\ +FLB_RUNTIME_SHELL_PATH=${CMAKE_CURRENT_SOURCE_DIR};\ +FLB_RUNTIME_SHELL_CONF=${CMAKE_CURRENT_SOURCE_DIR}/conf;\ +FLB_BIN=${CMAKE_BINARY_DIR}/bin/fluent-bit" + ) +endforeach() diff --git a/fluent-bit/tests/runtime_shell/common.sh b/fluent-bit/tests/runtime_shell/common.sh new file mode 100644 index 00000000..1f4ec663 --- /dev/null +++ b/fluent-bit/tests/runtime_shell/common.sh @@ -0,0 +1,26 @@ +#!/bin/sh + +# This function is meant to be used alongside with +# in_expect_base.conf or a similar setup where +# fluent-bit is set up to create a file as soon +# as it starts. +# +# I would rather use pidof but it's not viable. +# +wait_for_fluent_bit() { + result=1 + + for retry in `seq 10` + do + if test -f $1 + then + sleep 1 + result=0 + break + fi + + sleep 1 + done + + echo "$result" +} diff --git a/fluent-bit/tests/runtime_shell/conf/in_dummy_expect.conf b/fluent-bit/tests/runtime_shell/conf/in_dummy_expect.conf new file mode 100644 index 00000000..1b72d64e --- /dev/null +++ b/fluent-bit/tests/runtime_shell/conf/in_dummy_expect.conf @@ -0,0 +1,25 @@ +[SERVICE] + Flush 1 + Grace 2 + Log_Level info + +[INPUT] + Name dummy + Dummy {"r1": "someval", "s1": {"s2": null}, "r2": {"x": 0}} + Samples 1 + +[FILTER] + Name expect + Match * + Log_Level debug + # Rules + key_exists $r1 + key_not_exists $r0 + key_val_is_null $s1['s2'] + key_val_is_not_null $r2['x'] + key_val_eq $r1 someval + action exit + +[OUTPUT] + Name exit + Match * diff --git a/fluent-bit/tests/runtime_shell/conf/in_expect_base.conf b/fluent-bit/tests/runtime_shell/conf/in_expect_base.conf new file mode 100644 index 00000000..760b47d5 --- /dev/null +++ b/fluent-bit/tests/runtime_shell/conf/in_expect_base.conf @@ -0,0 +1,28 @@ +[SERVICE] + Flush 1 + Grace 1 + Log_Level error + Parsers_File ${FLB_ROOT}/conf/parsers.conf + +[INPUT] + name dummy + samples 1 + +[OUTPUT] + name file + match dummy.* + file ${SIGNAL_FILE_PATH} + mkdir on + +[FILTER] + Name expect + Match target_input + Log_Level debug + # Rules + key_exists $message + key_val_eq $message Hello! + action exit + +[OUTPUT] + Name exit + Match target_input diff --git a/fluent-bit/tests/runtime_shell/conf/in_http_tls_expect.conf b/fluent-bit/tests/runtime_shell/conf/in_http_tls_expect.conf new file mode 100644 index 00000000..2b66dff8 --- /dev/null +++ b/fluent-bit/tests/runtime_shell/conf/in_http_tls_expect.conf @@ -0,0 +1,13 @@ +@INCLUDE in_expect_base.conf + +[INPUT] + name http + tag target_input + listen ${LISTENER_HOST} + port ${LISTENER_PORT} + tls.verify no + tls on + tls.vhost ${LISTENER_VHOST} + tls.debug 4 + tls.crt_file ${FLB_RUNTIME_SHELL_PATH}/tls/certificate.pem + tls.key_file ${FLB_RUNTIME_SHELL_PATH}/tls/private_key.pem
\ No newline at end of file diff --git a/fluent-bit/tests/runtime_shell/conf/in_syslog_tcp_plaintext_expect.conf b/fluent-bit/tests/runtime_shell/conf/in_syslog_tcp_plaintext_expect.conf new file mode 100644 index 00000000..d47f6e33 --- /dev/null +++ b/fluent-bit/tests/runtime_shell/conf/in_syslog_tcp_plaintext_expect.conf @@ -0,0 +1,8 @@ +@INCLUDE in_expect_base.conf + +[INPUT] + name syslog + tag target_input + listen ${LISTENER_HOST} + port ${LISTENER_PORT} + mode tcp
\ No newline at end of file diff --git a/fluent-bit/tests/runtime_shell/conf/in_syslog_tcp_tls_expect.conf b/fluent-bit/tests/runtime_shell/conf/in_syslog_tcp_tls_expect.conf new file mode 100644 index 00000000..afa28e6f --- /dev/null +++ b/fluent-bit/tests/runtime_shell/conf/in_syslog_tcp_tls_expect.conf @@ -0,0 +1,14 @@ +@INCLUDE in_expect_base.conf + +[INPUT] + name syslog + tag target_input + listen ${LISTENER_HOST} + port ${LISTENER_PORT} + mode tcp + tls.verify no + tls on + tls.vhost ${LISTENER_VHOST} + tls.debug 4 + tls.crt_file ${FLB_RUNTIME_SHELL_PATH}/tls/certificate.pem + tls.key_file ${FLB_RUNTIME_SHELL_PATH}/tls/private_key.pem
\ No newline at end of file diff --git a/fluent-bit/tests/runtime_shell/conf/in_syslog_udp_plaintext_expect.conf b/fluent-bit/tests/runtime_shell/conf/in_syslog_udp_plaintext_expect.conf new file mode 100644 index 00000000..ae92b39e --- /dev/null +++ b/fluent-bit/tests/runtime_shell/conf/in_syslog_udp_plaintext_expect.conf @@ -0,0 +1,8 @@ +@INCLUDE in_expect_base.conf + +[INPUT] + name syslog + tag target_input + listen ${LISTENER_HOST} + port ${LISTENER_PORT} + mode udp
\ No newline at end of file diff --git a/fluent-bit/tests/runtime_shell/conf/in_syslog_uds_dgram_plaintext_expect.conf b/fluent-bit/tests/runtime_shell/conf/in_syslog_uds_dgram_plaintext_expect.conf new file mode 100644 index 00000000..48b6c375 --- /dev/null +++ b/fluent-bit/tests/runtime_shell/conf/in_syslog_uds_dgram_plaintext_expect.conf @@ -0,0 +1,7 @@ +@INCLUDE in_expect_base.conf + +[INPUT] + name syslog + tag target_input + path ${SOCKET_PATH} + mode unix_udp
\ No newline at end of file diff --git a/fluent-bit/tests/runtime_shell/conf/in_syslog_uds_stream_plaintext_expect.conf b/fluent-bit/tests/runtime_shell/conf/in_syslog_uds_stream_plaintext_expect.conf new file mode 100644 index 00000000..cfebdd24 --- /dev/null +++ b/fluent-bit/tests/runtime_shell/conf/in_syslog_uds_stream_plaintext_expect.conf @@ -0,0 +1,7 @@ +@INCLUDE in_expect_base.conf + +[INPUT] + name syslog + tag target_input + path ${SOCKET_PATH} + mode unix_tcp
\ No newline at end of file diff --git a/fluent-bit/tests/runtime_shell/conf/in_tail_expect.conf b/fluent-bit/tests/runtime_shell/conf/in_tail_expect.conf new file mode 100644 index 00000000..7348bacc --- /dev/null +++ b/fluent-bit/tests/runtime_shell/conf/in_tail_expect.conf @@ -0,0 +1,38 @@ +[SERVICE] + Flush 1 + Grace 2 + Log_Level info + Parsers_File ${FLB_ROOT}/conf/parsers.conf + +[INPUT] + name tail + path /tmp/flb_tail_expect*.log + exclude_path /tmp/flb_*2.log + read_from_head true + parser json + refresh_interval 10 + rotate_wait 1 + docker_mode false + docker_mode_flush 4 + path_key path_key + ignore_older 0 + buffer_chunk_size 32k + buffer_max_size 32k + skip_long_lines false + exit_on_eof false + db /tmp/flb_tail_expect.db + db.sync full + +[FILTER] + Name expect + Match * + Log_Level debug + # Rules + key_exists $path_key + key_val_eq $path_key /tmp/flb_tail_expect_1.log + key_not_exists $nokey + action exit + +[OUTPUT] + Name exit + Match * diff --git a/fluent-bit/tests/runtime_shell/in_dummy_expect.sh b/fluent-bit/tests/runtime_shell/in_dummy_expect.sh new file mode 100755 index 00000000..b7547763 --- /dev/null +++ b/fluent-bit/tests/runtime_shell/in_dummy_expect.sh @@ -0,0 +1,8 @@ +#!/bin/sh + +test_in_dummy_filter_expect() { + $FLB_BIN -c $FLB_RUNTIME_SHELL_CONF/in_dummy_expect.conf +} + +# The following command launch the unit test +. $FLB_RUNTIME_SHELL_PATH/runtime_shell.env diff --git a/fluent-bit/tests/runtime_shell/in_http_tls_expect.sh b/fluent-bit/tests/runtime_shell/in_http_tls_expect.sh new file mode 100755 index 00000000..fb47e2ae --- /dev/null +++ b/fluent-bit/tests/runtime_shell/in_http_tls_expect.sh @@ -0,0 +1,29 @@ +#!/bin/sh + +. ${FLB_RUNTIME_SHELL_PATH}/common.sh + +input_generator() { + result=$(wait_for_fluent_bit) + + if test "$result" -eq "0" + then + curl -s -k \ + -H 'content-type: application/json' \ + -d '{"message": "Hello!"}' \ + "https://${LISTENER_HOST}:${LISTENER_PORT}" + fi +} + +test_in_http_tls_filter_expect() { + export SIGNAL_FILE_PATH="/tmp/fb_signal_$$" + export LISTENER_VHOST=leo.vcap.me + export LISTENER_HOST=127.0.0.1 + export LISTENER_PORT=50000 + + input_generator & + + $FLB_BIN -c $FLB_RUNTIME_SHELL_CONF/in_http_tls_expect.conf +} + +# The following command launch the unit test +. $FLB_RUNTIME_SHELL_PATH/runtime_shell.env diff --git a/fluent-bit/tests/runtime_shell/in_syslog_tcp_plaintext_expect.sh b/fluent-bit/tests/runtime_shell/in_syslog_tcp_plaintext_expect.sh new file mode 100755 index 00000000..414d4069 --- /dev/null +++ b/fluent-bit/tests/runtime_shell/in_syslog_tcp_plaintext_expect.sh @@ -0,0 +1,26 @@ +#!/bin/sh + +. ${FLB_RUNTIME_SHELL_PATH}/common.sh + +input_generator() { + result=$(wait_for_fluent_bit ${SIGNAL_FILE_PATH}) + + if test "$result" -eq "0" + then + echo '<13>1 1970-01-01T00:00:00.000000+00:00 testhost testuser - - [] Hello!' | \ + nc -w 1 $LISTENER_HOST $LISTENER_PORT + fi +} + +test_in_syslog_tcp_plaintext_filter_expect() { + export SIGNAL_FILE_PATH="/tmp/fb_signal_$$" + export LISTENER_HOST=127.0.0.1 + export LISTENER_PORT=50001 + + input_generator & + + $FLB_BIN -c $FLB_RUNTIME_SHELL_CONF/in_syslog_tcp_plaintext_expect.conf +} + +# The following command launch the unit test +. $FLB_RUNTIME_SHELL_PATH/runtime_shell.env diff --git a/fluent-bit/tests/runtime_shell/in_syslog_tcp_tls_expect.sh b/fluent-bit/tests/runtime_shell/in_syslog_tcp_tls_expect.sh new file mode 100755 index 00000000..d76c0e3c --- /dev/null +++ b/fluent-bit/tests/runtime_shell/in_syslog_tcp_tls_expect.sh @@ -0,0 +1,27 @@ +#!/bin/sh + +. ${FLB_RUNTIME_SHELL_PATH}/common.sh + +input_generator() { + result=$(wait_for_fluent_bit ${SIGNAL_FILE_PATH}) + + if test "$result" -eq "0" + then + echo '<13>1 1970-01-01T00:00:00.000000+00:00 testhost testuser - - [] Hello!' | \ + openssl s_client -connect $LISTENER_HOST:$LISTENER_PORT 2>&1 >/dev/null + fi +} + +test_in_syslog_tcp_plaintext_filter_expect() { + export SIGNAL_FILE_PATH="/tmp/fb_signal_$$" + export LISTENER_VHOST=leo.vcap.me + export LISTENER_HOST=127.0.0.1 + export LISTENER_PORT=50002 + + input_generator & + + $FLB_BIN -c $FLB_RUNTIME_SHELL_CONF/in_syslog_tcp_tls_expect.conf +} + +# The following command launch the unit test +. $FLB_RUNTIME_SHELL_PATH/runtime_shell.env diff --git a/fluent-bit/tests/runtime_shell/in_syslog_udp_plaintext_expect.sh b/fluent-bit/tests/runtime_shell/in_syslog_udp_plaintext_expect.sh new file mode 100755 index 00000000..23970a13 --- /dev/null +++ b/fluent-bit/tests/runtime_shell/in_syslog_udp_plaintext_expect.sh @@ -0,0 +1,26 @@ +#!/bin/sh + +. ${FLB_RUNTIME_SHELL_PATH}/common.sh + +input_generator() { + result=$(wait_for_fluent_bit ${SIGNAL_FILE_PATH}) + + if test "$result" -eq "0" + then + echo '<13>1 1970-01-01T00:00:00.000000+00:00 testhost testuser - - [] Hello!' | \ + nc -w 1 -u $LISTENER_HOST $LISTENER_PORT + fi +} + +test_in_syslog_tcp_plaintext_filter_expect() { + export SIGNAL_FILE_PATH="/tmp/fb_signal_$$" + export LISTENER_HOST=127.0.0.1 + export LISTENER_PORT=50003 + + input_generator & + + $FLB_BIN -c $FLB_RUNTIME_SHELL_CONF/in_syslog_udp_plaintext_expect.conf +} + +# The following command launch the unit test +. $FLB_RUNTIME_SHELL_PATH/runtime_shell.env diff --git a/fluent-bit/tests/runtime_shell/in_syslog_uds_dgram_plaintext_expect.sh b/fluent-bit/tests/runtime_shell/in_syslog_uds_dgram_plaintext_expect.sh new file mode 100755 index 00000000..2ab7703b --- /dev/null +++ b/fluent-bit/tests/runtime_shell/in_syslog_uds_dgram_plaintext_expect.sh @@ -0,0 +1,29 @@ +#!/bin/sh + +. ${FLB_RUNTIME_SHELL_PATH}/common.sh + +input_generator() { + result=$(wait_for_fluent_bit ${SIGNAL_FILE_PATH}) + + if test "$result" -eq "0" + then + echo '<13>Jan 1 00:00:00 testuser: Hello!' | nc -w 1 -U -u $SOCKET_PATH + fi +} + +test_in_syslog_uds_stream_plaintext_filter_expect() { + platform=$(uname) + + if test "$platform" != "Darwin" + then + export SIGNAL_FILE_PATH="/tmp/fb_signal_$$" + export SOCKET_PATH=/tmp/fluent_bit_syslog_uds_dgram.sock + + input_generator & + + $FLB_BIN -c $FLB_RUNTIME_SHELL_CONF/in_syslog_uds_dgram_plaintext_expect.conf + fi +} + +# The following command launch the unit test +. $FLB_RUNTIME_SHELL_PATH/runtime_shell.env diff --git a/fluent-bit/tests/runtime_shell/in_syslog_uds_stream_plaintext_expect.sh b/fluent-bit/tests/runtime_shell/in_syslog_uds_stream_plaintext_expect.sh new file mode 100755 index 00000000..b3a38cc2 --- /dev/null +++ b/fluent-bit/tests/runtime_shell/in_syslog_uds_stream_plaintext_expect.sh @@ -0,0 +1,24 @@ +#!/bin/sh + +. ${FLB_RUNTIME_SHELL_PATH}/common.sh + +input_generator() { + result=$(wait_for_fluent_bit ${SIGNAL_FILE_PATH}) + + if test "$result" -eq "0" + then + echo '<13>Jan 1 00:00:00 testuser: Hello!' | nc -w 1 -U $SOCKET_PATH + fi +} + +test_in_syslog_uds_stream_plaintext_filter_expect() { + export SIGNAL_FILE_PATH="/tmp/fb_signal_$$" + export SOCKET_PATH=/tmp/fluent_bit_syslog_uds_stream.sock + + input_generator & + + $FLB_BIN -c $FLB_RUNTIME_SHELL_CONF/in_syslog_uds_stream_plaintext_expect.conf +} + +# The following command launch the unit test +. $FLB_RUNTIME_SHELL_PATH/runtime_shell.env diff --git a/fluent-bit/tests/runtime_shell/in_tail/README.md b/fluent-bit/tests/runtime_shell/in_tail/README.md new file mode 100644 index 00000000..78c77525 --- /dev/null +++ b/fluent-bit/tests/runtime_shell/in_tail/README.md @@ -0,0 +1,126 @@ +# Fluent Bit Tail Input Plugin Tests + +The following directory contains tests for Tail input plugin behaviors. + +## run_tests.sh + +This script provide validations for offsets, database file entries and rotation under different scenarios. The following tests are available in the script + +- test_normal_rotation +- test_single_static_rotation +- test_truncate +- test_rotate_link +- test_truncate_link + +Running the script ```test_rotation.sh``` will run every test listed above, to run a single test just append it name, e.g: + +``` +./test_rotation.sh -- test_truncate +``` + +### 1. Normal Rotation + +**Unit** + +```test_normal_rotation``` + +**Description** + +Run the logger tool that creates 5 different files, write 100000 messages to each one while rotating at 256KB. + +This test enable the database backend for Tail so it also helps to validate expected entries into the 'in_tail_files' table. + +**Configuration File** + +```conf/normal_rotation.conf``` + +### 2. Single Static Rotation + +**Unit** + +```test_single_static_rotation``` + +**Description** + +Run the logger tool that creates 1 big file and let Fluent Bit process it in the static mode, before to promote it to 'events' and it gets rotated. + +**Configuration File** + +```conf/single_static_rotation.conf``` + +### 3. Truncate + +**Unit** + +```test_truncate``` + +**Description** + + Some environments still rely on truncation mode or well known as copytruncate, + this is the definition by logrotate(8): + +> Truncate the original log file to zero size in place after creating a copy, +> instead of moving the old log file and optionally creating a new one. It +> can be used when some program cannot be told to close its logfile and +> thus might continue writing (appending) to the previous log file forever. +> +> Note that there is a very small time slice between copying the file and +> truncating it, so some logging data might be lost. When this option is +> used, the create option will have no effect, as the old log file stays in +> place. + +This test checks that after a truncation the new lines added are properly +processed. + +**Configuration File** + +```conf/truncate_rotation.conf``` + +### 4. Rotate Link + +**Unit** + +```test_rotate_link``` + +**Description** + +This test checks that a monitored link, upon rotation, keeps the proper offset and database status for the real file. + + Example: + + - file with data: data.log + - monitored link: test.log + + Check the behavior upon the following rotation: test.log -> test.log.1 + +**Configuration File** + +```conf/rotate_link.conf``` + +### 5. Truncate Link + +**Unit** + +```test_truncate_link``` + +**Description** + +Test a link that gets a truncation and Fluent Bit properly use the new offset + +**Configuration File** + +```conf/truncate_link.conf``` + +### 6. Multiline Rotation + +**Unit** + +```test_multiline_rotation``` + +**Description** + +Test a multiline rotation for issue 4190. + +**Configuration File** + +```conf/multiline_rotation.conf``` diff --git a/fluent-bit/tests/runtime_shell/in_tail/conf/multiline_rotation.conf b/fluent-bit/tests/runtime_shell/in_tail/conf/multiline_rotation.conf new file mode 100644 index 00000000..3695014a --- /dev/null +++ b/fluent-bit/tests/runtime_shell/in_tail/conf/multiline_rotation.conf @@ -0,0 +1,60 @@ +[SERVICE] + flush 1 + daemon off + log_level debug + log_file ${TEST_DIR}/out.log + +[INPUT] + name tail + tag a + path ${TEST_DIR}/a.log + db ${TEST_DIR}/a.db + db.sync full + multiline.parser cri + rotate_wait 5 + refresh_interval 2 + +[INPUT] + name tail + tag b + path ${TEST_DIR}/b.log + db ${TEST_DIR}/b.db + db.sync full + multiline.parser cri + rotate_wait 5 + refresh_interval 2 + +[INPUT] + name tail + tag c + path ${TEST_DIR}/c.log + db ${TEST_DIR}/c.db + db.sync full + multiline.parser cri + rotate_wait 5 + refresh_interval 2 + +[INPUT] + name tail + tag d + path ${TEST_DIR}/d.log + db ${TEST_DIR}/d.db + db.sync full + multiline.parser cri + rotate_wait 5 + refresh_interval 2 + +[INPUT] + name tail + tag e + path ${TEST_DIR}/e.log + db ${TEST_DIR}/e.db + db.sync full + multiline.parser cri + rotate_wait 5 + refresh_interval 2 + +[OUTPUT] + name file + match * + path ${TEST_DIR} diff --git a/fluent-bit/tests/runtime_shell/in_tail/conf/normal_rotation.conf b/fluent-bit/tests/runtime_shell/in_tail/conf/normal_rotation.conf new file mode 100644 index 00000000..0265df5d --- /dev/null +++ b/fluent-bit/tests/runtime_shell/in_tail/conf/normal_rotation.conf @@ -0,0 +1,55 @@ +[SERVICE] + flush 1 + daemon off + log_level debug + log_file ${TEST_DIR}/out.log + +[INPUT] + name tail + tag a + path ${TEST_DIR}/a.log + db ${TEST_DIR}/a.db + db.sync full + rotate_wait 5 + refresh_interval 2 + +[INPUT] + name tail + tag b + path ${TEST_DIR}/b.log + db ${TEST_DIR}/b.db + db.sync full + rotate_wait 5 + refresh_interval 2 + +[INPUT] + name tail + tag c + path ${TEST_DIR}/c.log + db ${TEST_DIR}/c.db + db.sync full + rotate_wait 5 + refresh_interval 2 + +[INPUT] + name tail + tag d + path ${TEST_DIR}/d.log + db ${TEST_DIR}/d.db + db.sync full + rotate_wait 5 + refresh_interval 2 + +[INPUT] + name tail + tag e + path ${TEST_DIR}/e.log + db ${TEST_DIR}/e.db + db.sync full + rotate_wait 5 + refresh_interval 2 + +[OUTPUT] + name file + match * + path ${TEST_DIR} diff --git a/fluent-bit/tests/runtime_shell/in_tail/conf/rotate_link.conf b/fluent-bit/tests/runtime_shell/in_tail/conf/rotate_link.conf new file mode 100644 index 00000000..f0b017d4 --- /dev/null +++ b/fluent-bit/tests/runtime_shell/in_tail/conf/rotate_link.conf @@ -0,0 +1,20 @@ +[SERVICE] + flush 1 + daemon off + log_level debug + log_file ${TEST_DIR}/out.log + +[INPUT] + name tail + tag a + path ${TEST_DIR}/a.log + db ${TEST_DIR}/a.db + db.sync full + rotate_wait 5 + watcher_interval 1 + refresh_interval 2 + +[OUTPUT] + name file + match * + path ${TEST_DIR} diff --git a/fluent-bit/tests/runtime_shell/in_tail/conf/single_static_rotation.conf b/fluent-bit/tests/runtime_shell/in_tail/conf/single_static_rotation.conf new file mode 100644 index 00000000..ac883c95 --- /dev/null +++ b/fluent-bit/tests/runtime_shell/in_tail/conf/single_static_rotation.conf @@ -0,0 +1,19 @@ +[SERVICE] + flush 1 + daemon off + log_level debug + log_file ${TEST_DIR}/out.log + +[INPUT] + name tail + tag a + path ${TEST_DIR}/a.log + db ${TEST_DIR}/a.db + db.sync full + rotate_wait 5 + refresh_interval 2 + +[OUTPUT] + name file + match * + path ${TEST_DIR} diff --git a/fluent-bit/tests/runtime_shell/in_tail/conf/truncate_link.conf b/fluent-bit/tests/runtime_shell/in_tail/conf/truncate_link.conf new file mode 100644 index 00000000..6cfa5410 --- /dev/null +++ b/fluent-bit/tests/runtime_shell/in_tail/conf/truncate_link.conf @@ -0,0 +1,19 @@ +[SERVICE] + flush 1 + daemon off + log_level debug + log_file ${TEST_DIR}/out.log + +[INPUT] + name tail + tag a + path ${TEST_DIR}/a.log + db ${TEST_DIR}/a.db + db.sync full + rotate_wait 5 + refresh_interval 1 + +[OUTPUT] + name file + match * + path ${TEST_DIR} diff --git a/fluent-bit/tests/runtime_shell/in_tail/conf/truncate_rotation.conf b/fluent-bit/tests/runtime_shell/in_tail/conf/truncate_rotation.conf new file mode 100644 index 00000000..ac883c95 --- /dev/null +++ b/fluent-bit/tests/runtime_shell/in_tail/conf/truncate_rotation.conf @@ -0,0 +1,19 @@ +[SERVICE] + flush 1 + daemon off + log_level debug + log_file ${TEST_DIR}/out.log + +[INPUT] + name tail + tag a + path ${TEST_DIR}/a.log + db ${TEST_DIR}/a.db + db.sync full + rotate_wait 5 + refresh_interval 2 + +[OUTPUT] + name file + match * + path ${TEST_DIR} diff --git a/fluent-bit/tests/runtime_shell/in_tail/logger_file.py b/fluent-bit/tests/runtime_shell/in_tail/logger_file.py new file mode 100755 index 00000000..a8e2a1b3 --- /dev/null +++ b/fluent-bit/tests/runtime_shell/in_tail/logger_file.py @@ -0,0 +1,72 @@ +#!/bin/env python + +import sys +import uuid +import time +import signal +import logging +from argparse import ArgumentParser +from logging.handlers import RotatingFileHandler +from threading import Thread + +class LoggerManager: + def __init__(self, args): + # KB to bytes + self.max_bytes = (args.size * 1000) + self.backup = args.backup + self.lines = args.lines + self.delay = args.delay + self.threads = [] + + # Create a thread for every writer + for f in args.filenames: + thread = Thread(target = self.single_logger_thread, args = (f,)) + if thread is None: + print("error creating thread") + sys.exit(1) + self.threads.append(thread) + thread.start() + print("Logger thread for '" + f + "' has started") + + for th in self.threads: + th.join() + print("Logger thread finished") + + def single_logger_thread(self, name): + logger = logging.getLogger(name) + logger.setLevel(logging.DEBUG) + handler = RotatingFileHandler(name, maxBytes = self.max_bytes, + backupCount = self.backup) + logger.addHandler(handler) + rnd = uuid.uuid4() + + i = 0 + while i < self.lines: + logger.debug(rnd) + if self.delay > 0.0: + time.sleep(self.delay / 1000.0) + i = i + 1 + +def signal_handler(sig, frame): + print("stopping logger") + sys.exit(0) + +if __name__ == '__main__': + signal.signal(signal.SIGINT, signal_handler) + + # Define arguments + parser = ArgumentParser() + parser.add_argument("-b", "--backup", dest="backup", default=50, type=int) + parser.add_argument("-d", "--delay", dest="delay", default=0.1, type=float, + help="milliseconds delay between line writes") + parser.add_argument("-l", "--lines", dest="lines", default=1000, type=int) + parser.add_argument("-f", "--file", dest="filenames", action='append', required=True, + help="write logs to FILE", metavar="FILE") + parser.add_argument("-s", "--size", dest="size", type=int, + help="maximum log file size in KB before rotation", + default=256) + # Read arguments + args = parser.parse_args() + + # Start the Logger + lm = LoggerManager(args) diff --git a/fluent-bit/tests/runtime_shell/in_tail/run_tests.sh b/fluent-bit/tests/runtime_shell/in_tail/run_tests.sh new file mode 100755 index 00000000..09bb9c43 --- /dev/null +++ b/fluent-bit/tests/runtime_shell/in_tail/run_tests.sh @@ -0,0 +1,530 @@ +#!/bin/sh + +# Environment variables +FLB_BIN=`realpath ../../../build/bin/fluent-bit` +FLB_RUNTIME_SHELL_PATH=`realpath $(pwd)/../` +FLB_RUN_TEST=`realpath $FLB_RUNTIME_SHELL_PATH/../lib/shunit2/shunit2` + +# Colorize shunit2 +bold=$(tput bold) +normal=$(tput sgr0) +SHUNIT_TEST_PREFIX="$bold==========> UNIT TEST: $normal" + +# 1. Normal Rotation +# ------------------ +# Run the logger tool that creates 5 different files, write 100000 messages to each one +# while rotating at 256KB. +# +# This test enable the database backend for Tail so it also helps to validate expected +# entries into the 'in_tail_files' table. +# +# Configuration file used: conf/normal_rotation.conf + +test_normal_rotation() { + # Helper function to check monitored files + sqlite_check() + { + # Incoming parameters: + # $1: temporal directory to store data + # $2: database file name + # $3: Fluent Bit PID + # + # This function store the remaining monitored files listed in the database, + # we send the output to an .inodes for troubleshooting purposes if required + sqlite3 $1/$2 -batch \ + ".headers off" ".width 20" "SELECT inode FROM in_tail_files" > \ + $1/$2.inodes + + rows=`cat $1/$2.inodes | wc -l | tr -d -C '[0-9]'` + if [ $rows != "1" ]; then + echo "> database file $1/$2 contains $rows rows, inodes:" + cat $1/$2.inodes + echo "> open files" + ls -l /proc/$3/fd/ | grep \\.log + else + echo "> database file $1/$2 is OK" + fi + ${_ASSERT_EQUALS_} "1" $rows + } + + # Prepare test directory + export TEST_DIR=tmp_test + rm -rf $TEST_DIR + mkdir $TEST_DIR + + # Create empty files so Fluent Bit will enqueue them on start + for logfile in a b c d e ; do + touch $TEST_DIR/$logfile.log + done + + # Run Fluent Bit + $FLB_BIN -c conf/normal_rotation.conf & + FLB_PID=$! + echo "Fluent Bit started, pid=$FLB_PID" + + # Start the Logger: 5 files = 500000 log lines in total + python logger_file.py -l 100000 -s 256 -b 100 -d 0.1 \ + -f $TEST_DIR/a.log \ + -f $TEST_DIR/b.log \ + -f $TEST_DIR/c.log \ + -f $TEST_DIR/d.log \ + -f $TEST_DIR/e.log + + echo "Logger finished...wait 10 seconds" + sleep 10 + + # Count number of processed lines + write_lines=`cat $TEST_DIR/[abcdefghij].log* | wc -l` + read_lines=`cat $TEST_DIR/[abcdefghij] | wc -l` + + echo "> write lines: $write_lines" + echo "> read lines : $read_lines" + + # Check we processed same number of records + ${_ASSERT_EQUALS_} $write_lines $read_lines + + # Validate our database files has only one remaining entry per database file + for logfile in a b c d e; do + sqlite_check $TEST_DIR "$logfile.db" $FLB_PID + done + + # Stop Fluent Bit (SIGTERM) + kill -15 $FLB_PID +} + +# 2. Single Static Rotation (static process mode + rotation) +# ---------------------------------------------------------- +# Run the logger tool that creates 1 big file and let Fluent Bit process it in +# the static mode, before to promote it to 'events' and it gets rotated. +# +# Configuration file used: conf/single_static_rotation.conf + +test_single_static_rotation() { + # Write a log file of 200000 lines + + # Prepare test directory + export TEST_DIR=tmp_test + rm -rf $TEST_DIR + mkdir $TEST_DIR + + # Create empty files so Fluent Bit will enqueue them on start + touch $TEST_DIR/a.log + + # Start the Logger: 1 file with 400000 lines, we use a big size (-s) to + # avoid rotation + python logger_file.py -l 400000 -s 200000 -b 100 -d 0 \ + -f $TEST_DIR/a.log + lines=`cat $TEST_DIR/a.log | wc -l` + echo "Logger done, written lines "$lines + + # Run Fluent Bit + $FLB_BIN -c conf/single_static_rotation.conf & + FLB_PID=$! + echo "Fluent Bit started, pid=$FLB_PID" + + # Wait 3 seconds before rotation + sleep 2 + mv $TEST_DIR/a.log $TEST_DIR/a.log.1 + + lines=`cat $TEST_DIR/a | wc -l` + echo "file Rotated, mid-check: processed lines $lines" + sleep 30 + + # Count number of processed lines + write_lines=`cat $TEST_DIR/a.log.1 | wc -l` + read_lines=`cat $TEST_DIR/a | wc -l` + + echo "> write lines: $write_lines" + echo "> read lines : $read_lines" + + # Check we processed same number of records + ${_ASSERT_EQUALS_} $write_lines $read_lines + + # Validate our database files has only one remaining entry per database file + #sqlite_check $TEST_DIR "$logfile.db" $FLB_PID + + # Stop Fluent Bit (SIGTERM) + kill -15 $FLB_PID +} + +# 3. Truncate +# ----------- +# Some environments still rely on truncation mode or well known as copytruncate, +# this is the definition by logrotate(8): +# +# "Truncate the original log file to zero size in place after creating a copy, +# instead of moving the old log file and optionally creating a new one. It +# can be used when some program cannot be told to close its logfile and +# thus might continue writing (appending) to the previous log file forever. +# +# Note that there is a very small time slice between copying the file and +# truncating it, so some logging data might be lost. When this option is +# used, the create option will have no effect, as the old log file stays in +# place." +# +# This test checks that after a truncation the new lines added are properly +# processed. +# +# Configuration file used: conf/truncate_rotation.conf + +test_truncate() { + # Helper function to check monitored files + sqlite_check() + { + # Incoming parameters: + # $1: temporal directory to store data + # $2: database file name + # $3: Fluent Bit PID + # + # This function store the remaining monitored files listed in the database, + # we send the output to an .inodes for troubleshooting purposes if required + + # Get the last size of the 'a.log' file and check we have the same value + # in the database + offset=`wc -c < $TEST_DIR/a.log` + + sqlite3 $1/$2 -batch \ + ".headers off" "SELECT inode FROM in_tail_files WHERE offset=$offset" > \ + $1/$2.offset + + rows=`cat $1/$2.offset | wc -l | tr -d -C '[0-9]'` + if [ $rows != "1" ]; then + echo "> invalid database content:" + cat $1/$2.offset + echo "> open files" + ls -l /proc/$3/fd/ | grep \\.log + else + echo "> database file $1/$2 is OK" + fi + ${_ASSERT_EQUALS_} "1" $rows + } + + # Prepare test directory + export TEST_DIR=tmp_test + rm -rf $TEST_DIR + mkdir $TEST_DIR + + # Create empty files so Fluent Bit will enqueue them on start + touch $TEST_DIR/a.log + + # Start the Logger: 1 file with 200 lines, we use a big size limit (-s) to + # avoid rotation + python logger_file.py -l 200 -s 200000 -b 100 -d 0 -f $TEST_DIR/a.log + lines=`cat $TEST_DIR/a.log | wc -l` + echo "Logger done, written lines "$lines + + # Run Fluent Bit + $FLB_BIN -c conf/truncate_rotation.conf & + FLB_PID=$! + echo "Fluent Bit started, pid=$FLB_PID" + + # Wait 2 seconds before truncation + sleep 2 + pre_lines=`cat $TEST_DIR/a.log | wc -l` + truncate -s 0 $TEST_DIR/a.log + + lines=`cat $TEST_DIR/a | wc -l` + echo "file truncated, mid-check: processed lines $lines" + + # Append 100 more lines + python logger_file.py -l 100 -s 200000 -b 100 -d 0 -f $TEST_DIR/a.log + + sleep 3 + + # Count number of processed lines + write_lines=300 + read_lines=`cat $TEST_DIR/a | wc -l` + + echo "> write lines: $write_lines" + echo "> read lines : $read_lines" + + # Check we processed same number of records + ${_ASSERT_EQUALS_} $write_lines $read_lines + + sqlite_check $TEST_DIR a.db $FLB_PID + + # Stop Fluent Bit (SIGTERM) + kill -15 $FLB_PID +} + +# 4. Rotate Link +# -------------- +# This case checks that a monitored link, upon rotation, keeps the proper offset +# and database status for the real file. +# +# Example: +# +# - file with data: data.log +# - monitored link: test.log +# +# Check the behavior upon test.log -> test.log.1 behavior +# +# Configuration file used: conf/rotate_link.conf + +test_rotate_link() { + # Helper function to check monitored files + sqlite_check() + { + # Incoming parameters: + # $1: temporal directory to store data + # $2: database file name + # $3: Fluent Bit PID + # + # This function store the remaining monitored files listed in the database, + # we send the output to an .inodes for troubleshooting purposes if required + + # Get the last size of the file pointed by 'a.log.1' and check we have the + # same value in the database + offset=`wc -c < $TEST_DIR/a.log.1` + + sqlite3 $1/$2 -batch \ + ".headers off" "SELECT inode FROM in_tail_files WHERE offset=$offset \ + AND rotated=1" > $1/$2.offset + + rows=`cat $1/$2.offset | wc -l | tr -d -C '[0-9]'` + if [ $rows != "1" ]; then + echo "> invalid database content:" + cat $1/$2.offset + echo "> open files" + ls -l /proc/$3/fd/ | grep \\.log + else + echo "> offset database check $1/$2 is OK" + fi + ${_ASSERT_EQUALS_} "1" $rows + + # After rotate_wait (5 secs + watcher) we expect an empty database + sleep 6 + sqlite3 $1/$2 -batch \ + ".headers off" "SELECT inode FROM in_tail_files WHERE offset=$offset \ + AND rotated=1" > $1/$2.offset + + rows=`cat $1/$2.offset | wc -l | tr -d -C '[0-9]'` + if [ $rows != "0" ]; then + echo "> invalid database content:" + cat $1/$2.offset + echo "> open files" + ls -l /proc/$3/fd/ | grep \\.log + else + echo "> empty database check $1/$2 is OK" + fi + ${_ASSERT_EQUALS_} "0" $rows + } + + # Prepare test directory + export TEST_DIR=tmp_test + rm -rf $TEST_DIR + mkdir $TEST_DIR + + # Create empty files so Fluent Bit will enqueue them on start + touch $TEST_DIR/data.log + + # Start the Logger: 1 file with 100 lines, we use a big size limit (-s) to + # avoid rotation + python logger_file.py -l 100 -s 200000 -b 100 -d 0 -f $TEST_DIR/data.log + lines=`cat $TEST_DIR/data.log | wc -l` + ln -s data.log $TEST_DIR/a.log + echo "Logger done, written lines "$lines + + # Run Fluent Bit + $FLB_BIN -c conf/rotate_link.conf & + FLB_PID=$! + echo "Fluent Bit started, pid=$FLB_PID" + + # Wait 2 seconds and rotate file + sleep 2 + pre_lines=`cat $TEST_DIR/a.log | wc -l` + mv $TEST_DIR/a.log $TEST_DIR/a.log.1 + + lines=`cat $TEST_DIR/a | wc -l` + echo "file rotated, mid-check: processed lines $lines" + + # Append 200 more lines to the rotated link + python logger_file.py -l 200 -s 200000 -b 100 -d 0 -f $TEST_DIR/a.log.1 + + # Count number of processed lines + sleep 3 + write_lines=300 + read_lines=`cat $TEST_DIR/a | wc -l` + + echo "> write lines: $write_lines" + echo "> read lines : $read_lines" + + # Check we processed same number of records + ${_ASSERT_EQUALS_} $write_lines $read_lines + + # Check that database file have the right offset and mark the file as rotated + sqlite_check $TEST_DIR a.db $FLB_PID + + # Stop Fluent Bit (SIGTERM) + kill -15 $FLB_PID +} + +# 5. Truncate Link +# +# Test a link that gets a truncation and Fluent Bit properly use the new offset +# +# Configuration file used: conf/truncate_link.conf + +test_truncate_link() { + # Helper function to check monitored files + sqlite_check() + { + # Incoming parameters: + # $1: temporal directory to store data + # $2: database file name + # $3: Fluent Bit PID + # + # This function store the remaining monitored files listed in the database, + # we send the output to an .inodes for troubleshooting purposes if required + + # Get the last size of the 'a.log' file and check we have the same value + # in the database + offset=`wc -c < $TEST_DIR/a.log` + + sqlite3 $1/$2 -batch \ + ".headers off" "SELECT inode FROM in_tail_files WHERE offset=$offset" > \ + $1/$2.offset + + rows=`cat $1/$2.offset | wc -l | tr -d -C '[0-9]'` + if [ $rows != "1" ]; then + echo "> invalid database content:" + cat $1/$2.offset + echo "> open files" + ls -l /proc/$3/fd/ | grep \\.log + else + echo "> database file $1/$2 is OK" + fi + ${_ASSERT_EQUALS_} "1" $rows + } + + # Prepare test directory + export TEST_DIR=tmp_test + rm -rf $TEST_DIR + mkdir $TEST_DIR + + # Create empty files so Fluent Bit will enqueue them on start + touch $TEST_DIR/data.log + + # Start the Logger: 1 file with 100 lines, we use a big size limit (-s) to + # avoid rotation + python logger_file.py -l 100 -s 200000 -b 100 -d 0 -f $TEST_DIR/data.log + lines=`cat $TEST_DIR/data.log | wc -l` + ln -s data.log $TEST_DIR/a.log + echo "Logger done, written lines "$lines + + # Run Fluent Bit + $FLB_BIN -c conf/truncate_link.conf & + FLB_PID=$! + echo "Fluent Bit started, pid=$FLB_PID" + + # Wait 1 second before truncation + sleep 1 + pre_lines=`cat $TEST_DIR/a.log | wc -l` + truncate -s 0 $TEST_DIR/a.log + + sleep 2 + lines=`cat $TEST_DIR/a | wc -l` + echo "file truncated, mid-check: processed lines $lines" + + # Append 200 more lines + python logger_file.py -l 200 -s 200000 -b 100 -d 0 -f $TEST_DIR/a.log + + sleep 4 + + # Count number of processed lines + write_lines=300 + read_lines=`cat $TEST_DIR/a | wc -l` + + echo "> write lines: $write_lines" + echo "> read lines : $read_lines" + + # Check we processed same number of records + ${_ASSERT_EQUALS_} $write_lines $read_lines + + # Stop Fluent Bit (SIGTERM) + kill -15 $FLB_PID +} + +# 6. Multiline + rotation +# ------------------ +# Run the logger tool that creates 5 different files, write 100000 messages to each one +# while rotating at 256KB. +# +# This test for issue 4190 +# +# Configuration file used: conf/multiline_rotation.conf + +test_multiline_rotation() { + # Helper function to check monitored files + sqlite_check() + { + # Incoming parameters: + # $1: temporal directory to store data + # $2: database file name + # $3: Fluent Bit PID + # + # This function store the remaining monitored files listed in the database, + # we send the output to an .inodes for troubleshooting purposes if required + sqlite3 $1/$2 -batch \ + ".headers off" ".width 20" "SELECT inode FROM in_tail_files" > \ + $1/$2.inodes + + rows=`cat $1/$2.inodes | wc -l | tr -d -C '[0-9]'` + if [ $rows != "1" ]; then + echo "> database file $1/$2 contains $rows rows, inodes:" + cat $1/$2.inodes + echo "> open files" + ls -l /proc/$3/fd/ | grep \\.log + else + echo "> database file $1/$2 is OK" + fi + ${_ASSERT_EQUALS_} "1" $rows + } + + # Prepare test directory + export TEST_DIR=tmp_test + rm -rf $TEST_DIR + mkdir $TEST_DIR + + # Create empty files so Fluent Bit will enqueue them on start + for logfile in a b c d e ; do + touch $TEST_DIR/$logfile.log + done + + # Run Fluent Bit + $FLB_BIN -c conf/multiline_rotation.conf & + FLB_PID=$! + echo "Fluent Bit started, pid=$FLB_PID" + + # Start the Logger: 5 files = 500000 log lines in total + python logger_file.py -l 100000 -s 256 -b 100 -d 0.1 \ + -f $TEST_DIR/a.log \ + -f $TEST_DIR/b.log \ + -f $TEST_DIR/c.log \ + -f $TEST_DIR/d.log \ + -f $TEST_DIR/e.log + + echo "Logger finished...wait 10 seconds" + sleep 10 + + # Count number of processed lines + write_lines=`cat $TEST_DIR/[abcdefghij].log* | wc -l` + read_lines=`cat $TEST_DIR/[abcdefghij] | wc -l` + + echo "> write lines: $write_lines" + echo "> read lines : $read_lines" + + # Check we processed same number of records + ${_ASSERT_EQUALS_} $write_lines $read_lines + + # Validate our database files has only one remaining entry per database file + for logfile in a b c d e; do + sqlite_check $TEST_DIR "$logfile.db" $FLB_PID + done + + # Stop Fluent Bit (SIGTERM) + kill -15 $FLB_PID +} + +# Launch the tests +. $FLB_RUN_TEST diff --git a/fluent-bit/tests/runtime_shell/in_tail/test_rotation.sh b/fluent-bit/tests/runtime_shell/in_tail/test_rotation.sh new file mode 100644 index 00000000..889fa7b0 --- /dev/null +++ b/fluent-bit/tests/runtime_shell/in_tail/test_rotation.sh @@ -0,0 +1,542 @@ +#!/bin/sh + +# Environment variables +FLB_BIN=`realpath ../../../build/bin/fluent-bit` +FLB_RUNTIME_SHELL_PATH=`realpath $(pwd)/../` +FLB_RUN_TEST=`realpath $FLB_RUNTIME_SHELL_PATH/../lib/shunit2/shunit2` + +# Colorize shunit2 +bold=$(tput bold) +normal=$(tput sgr0) +SHUNIT_TEST_PREFIX="$bold==========> UNIT TEST: $normal" + +# 1. Normal Rotation +# ------------------ +# Run the logger tool that creates 5 different files, write 100000 messages to each one +# while rotating at 256KB. +# +# This test enable the database backend for Tail so it also helps to validate expected +# entries into the 'in_tail_files' table. +# +# Configuration file used: conf/normal_rotation.conf + +test_normal_rotation() { + # Helper function to check monitored files + sqlite_check() + { + # Incoming parameters: + # $1: temporal directory to store data + # $2: database file name + # $3: Fluent Bit PID + # + # This function store the remaining monitored files listed in the database, + # we send the output to an .inodes for troubleshooting purposes if required + sqlite3 $1/$2 -batch \ + ".headers off" ".width 20" "SELECT inode FROM in_tail_files" > \ + $1/$2.inodes + + rows=`cat $1/$2.inodes | wc -l | tr -d -C '[0-9]'` + if [ $rows != "1" ]; then + echo "> database file $1/$2 contains $rows rows, inodes:" + cat $1/$2.inodes + echo "> open files" + ls -l /proc/$3/fd/ | grep \\.log + else + echo "> database file $1/$2 is OK" + fi + ${_ASSERT_EQUALS_} "1" $rows + } + + # Prepare test directory + export TEST_DIR=tmp_test + rm -rf $TEST_DIR + mkdir $TEST_DIR + + # Create empty files so Fluent Bit will enqueue them on start + for logfile in a b c d e ; do + touch $TEST_DIR/$logfile.log + done + + # Run Fluent Bit + $FLB_BIN -c conf/normal_rotation.conf & + FLB_PID=$! + echo "Fluent Bit started, pid=$FLB_PID" + + # Start the Logger: 5 files = 500000 log lines in total + python logger_file.py -l 100000 -s 256 -b 100 -d 0.1 \ + -f $TEST_DIR/a.log \ + -f $TEST_DIR/b.log \ + -f $TEST_DIR/c.log \ + -f $TEST_DIR/d.log \ + -f $TEST_DIR/e.log + + echo "Logger finished...wait 10 seconds" + sleep 10 + + # Count number of processed lines + write_lines=`cat $TEST_DIR/[abcdefghij].log* | wc -l` + read_lines=`cat $TEST_DIR/[abcdefghij] | wc -l` + + echo "> write lines: $write_lines" + echo "> read lines : $read_lines" + + # Check we processed same number of records + ${_ASSERT_EQUALS_} $write_lines $read_lines + + # Validate our database files has only one remaining entry per database file + for logfile in a b c d e; do + sqlite_check $TEST_DIR "$logfile.db" $FLB_PID + done + + # Stop Fluent Bit (SIGTERM) + kill -15 $FLB_PID +} + +# 2. Single Static Rotation (static process mode + rotation) +# ---------------------------------------------------------- +# Run the logger tool that creates 1 big file and let Fluent Bit process it in +# the static mode, before to promote it to 'events' and it gets rotated. +# +# Configuration file used: conf/single_static_rotation.conf + +test_single_static_rotation() { + # Write a log file of 200000 lines + + # Prepare test directory + export TEST_DIR=tmp_test + rm -rf $TEST_DIR + mkdir $TEST_DIR + + # Create empty files so Fluent Bit will enqueue them on start + touch $TEST_DIR/a.log + + # Start the Logger: 1 file with 400000 lines, we use a big size (-s) to + # avoid rotation + python logger_file.py -l 400000 -s 200000 -b 100 -d 0 \ + -f $TEST_DIR/a.log + lines=`cat $TEST_DIR/a.log | wc -l` + echo "Logger done, written lines "$lines + + # Run Fluent Bit + $FLB_BIN -c conf/single_static_rotation.conf & + FLB_PID=$! + echo "Fluent Bit started, pid=$FLB_PID" + + # Wait 3 seconds before rotation + sleep 2 + mv $TEST_DIR/a.log $TEST_DIR/a.log.1 + + lines=`cat $TEST_DIR/a | wc -l` + echo "file Rotated, mid-check: processed lines $lines" + sleep 30 + + # Count number of processed lines + write_lines=`cat $TEST_DIR/a.log.1 | wc -l` + read_lines=`cat $TEST_DIR/a | wc -l` + + echo "> write lines: $write_lines" + echo "> read lines : $read_lines" + + # Check we processed same number of records + ${_ASSERT_EQUALS_} $write_lines $read_lines + + # Validate our database files has only one remaining entry per database file + #sqlite_check $TEST_DIR "$logfile.db" $FLB_PID + + # Stop Fluent Bit (SIGTERM) + kill -15 $FLB_PID +} + +# 3. Truncate +# ----------- +# Some environments still rely on truncation mode or well known as copytruncate, +# this is the definition by logrotate(8): +# +# "Truncate the original log file to zero size in place after creating a copy, +# instead of moving the old log file and optionally creating a new one. It +# can be used when some program cannot be told to close its logfile and +# thus might continue writing (appending) to the previous log file forever. +# +# Note that there is a very small time slice between copying the file and +# truncating it, so some logging data might be lost. When this option is +# used, the create option will have no effect, as the old log file stays in +# place." +# +# This test checks that after a truncation the new lines added are properly +# processed. +# +# Configuration file used: conf/truncate_rotation.conf + +test_truncate() { + # Helper function to check monitored files + sqlite_check() + { + # Incoming parameters: + # $1: temporal directory to store data + # $2: database file name + # $3: Fluent Bit PID + # + # This function store the remaining monitored files listed in the database, + # we send the output to an .inodes for troubleshooting purposes if required + + # Get the last size of the 'a.log' file and check we have the same value + # in the database + offset=`wc -c < $TEST_DIR/a.log` + + sqlite3 $1/$2 -batch \ + ".headers off" "SELECT inode FROM in_tail_files WHERE offset=$offset" > \ + $1/$2.offset + + rows=`cat $1/$2.offset | wc -l | tr -d -C '[0-9]'` + if [ $rows != "1" ]; then + echo "> invalid database content:" + cat $1/$2.offset + echo "> open files" + ls -l /proc/$3/fd/ | grep \\.log + else + echo "> database file $1/$2 is OK" + fi + ${_ASSERT_EQUALS_} "1" $rows + } + + # Prepare test directory + export TEST_DIR=tmp_test + rm -rf $TEST_DIR + mkdir $TEST_DIR + + # Create empty files so Fluent Bit will enqueue them on start + touch $TEST_DIR/a.log + + # Start the Logger: 1 file with 200 lines, we use a big size limit (-s) to + # avoid rotation + python logger_file.py -l 200 -s 200000 -b 100 -d 0 -f $TEST_DIR/a.log + lines=`cat $TEST_DIR/a.log | wc -l` + echo "Logger done, written lines "$lines + + # Run Fluent Bit + $FLB_BIN -c conf/truncate_rotation.conf & + FLB_PID=$! + echo "Fluent Bit started, pid=$FLB_PID" + + # Wait 2 seconds before truncation + sleep 2 + pre_lines=`cat $TEST_DIR/a.log | wc -l` + truncate -s 0 $TEST_DIR/a.log + + lines=`cat $TEST_DIR/a | wc -l` + echo "file truncated, mid-check: processed lines $lines" + + # Append 100 more lines + python logger_file.py -l 100 -s 200000 -b 100 -d 0 -f $TEST_DIR/a.log + + sleep 3 + + # Count number of processed lines + write_lines=300 + read_lines=`cat $TEST_DIR/a | wc -l` + + echo "> write lines: $write_lines" + echo "> read lines : $read_lines" + + # Check we processed same number of records + ${_ASSERT_EQUALS_} $write_lines $read_lines + + sqlite_check $TEST_DIR a.db $FLB_PID + + # Stop Fluent Bit (SIGTERM) + kill -15 $FLB_PID +} + +# 4. Rotate Link +# -------------- +# This case checks that a monitored link, upon rotation, keeps the proper offset +# and database status for the real file. +# +# Example: +# +# - file with data: data.log +# - monitored link: test.log +# +# Check the behavior upon test.log -> test.log.1 behavior +# +# Configuration file used: conf/rotate_link.conf + +test_rotate_link() { + # Helper function to check monitored files + sqlite_check() + { + # Incoming parameters: + # $1: temporal directory to store data + # $2: database file name + # $3: Fluent Bit PID + # + # This function store the remaining monitored files listed in the database, + # we send the output to an .inodes for troubleshooting purposes if required + + # Get the last size of the file pointed by 'a.log.1' and check we have the + # same value in the database + offset=`wc -c < $TEST_DIR/a.log.1` + + sqlite3 $1/$2 -batch \ + ".headers off" "SELECT inode FROM in_tail_files WHERE offset=$offset \ + AND rotated=1" > $1/$2.offset + + rows=`cat $1/$2.offset | wc -l | tr -d -C '[0-9]'` + if [ $rows != "1" ]; then + echo "> invalid database content:" + cat $1/$2.offset + echo "> open files" + ls -l /proc/$3/fd/ | grep \\.log + else + echo "> offset database check $1/$2 is OK" + fi + ${_ASSERT_EQUALS_} "1" $rows + + # After rotate_wait (5 secs + watcher) we expect an empty database + sleep 6 + sqlite3 $1/$2 -batch \ + ".headers off" "SELECT inode FROM in_tail_files WHERE offset=$offset \ + AND rotated=1" > $1/$2.offset + + rows=`cat $1/$2.offset | wc -l | tr -d -C '[0-9]'` + if [ $rows != "0" ]; then + echo "> invalid database content:" + cat $1/$2.offset + echo "> open files" + ls -l /proc/$3/fd/ | grep \\.log + else + echo "> empty database check $1/$2 is OK" + fi + ${_ASSERT_EQUALS_} "0" $rows + } + + # Prepare test directory + export TEST_DIR=tmp_test + rm -rf $TEST_DIR + mkdir $TEST_DIR + + # Create empty files so Fluent Bit will enqueue them on start + touch $TEST_DIR/data.log + + # Start the Logger: 1 file with 100 lines, we use a big size limit (-s) to + # avoid rotation + python logger_file.py -l 100 -s 200000 -b 100 -d 0 -f $TEST_DIR/data.log + lines=`cat $TEST_DIR/data.log | wc -l` + ln -s data.log $TEST_DIR/a.log + echo "Logger done, written lines "$lines + + # Run Fluent Bit + $FLB_BIN -c conf/rotate_link.conf & + FLB_PID=$! + echo "Fluent Bit started, pid=$FLB_PID" + + # Wait 2 seconds and rotate file + sleep 2 + pre_lines=`cat $TEST_DIR/a.log | wc -l` + mv $TEST_DIR/a.log $TEST_DIR/a.log.1 + + lines=`cat $TEST_DIR/a | wc -l` + echo "file rotated, mid-check: processed lines $lines" + + # Append 200 more lines to the rotated link + python logger_file.py -l 200 -s 200000 -b 100 -d 0 -f $TEST_DIR/a.log.1 + + # Count number of processed lines + sleep 3 + write_lines=300 + read_lines=`cat $TEST_DIR/a | wc -l` + + echo "> write lines: $write_lines" + echo "> read lines : $read_lines" + + # Check we processed same number of records + ${_ASSERT_EQUALS_} $write_lines $read_lines + + # Check that database file have the right offset and mark the file as rotated + sqlite_check $TEST_DIR a.db $FLB_PID + + # Stop Fluent Bit (SIGTERM) + kill -15 $FLB_PID +} + +# 5. Truncate Link +# +# Test a link that gets a truncation and Fluent Bit properly use the new offset +# +# Configuration file used: conf/truncate_link.conf + +test_truncate_link() { + # Helper function to check monitored files + sqlite_check() + { + # Incoming parameters: + # $1: temporal directory to store data + # $2: database file name + # $3: Fluent Bit PID + # + # This function store the remaining monitored files listed in the database, + # we send the output to an .inodes for troubleshooting purposes if required + + # Get the last size of the 'a.log' file and check we have the same value + # in the database + offset=`wc -c < $TEST_DIR/a.log` + + sqlite3 $1/$2 -batch \ + ".headers off" "SELECT inode FROM in_tail_files WHERE offset=$offset" > \ + $1/$2.offset + + rows=`cat $1/$2.offset | wc -l | tr -d -C '[0-9]'` + if [ $rows != "1" ]; then + echo "> invalid database content:" + cat $1/$2.offset + echo "> open files" + ls -l /proc/$3/fd/ | grep \\.log + else + echo "> database file $1/$2 is OK" + fi + ${_ASSERT_EQUALS_} "1" $rows + } + + # Prepare test directory + export TEST_DIR=tmp_test + rm -rf $TEST_DIR + mkdir $TEST_DIR + + # Create empty files so Fluent Bit will enqueue them on start + touch $TEST_DIR/data.log + + # Start the Logger: 1 file with 100 lines, we use a big size limit (-s) to + # avoid rotation + python logger_file.py -l 100 -s 200000 -b 100 -d 0 -f $TEST_DIR/data.log + lines=`cat $TEST_DIR/data.log | wc -l` + ln -s data.log $TEST_DIR/a.log + echo "Logger done, written lines "$lines + + # Run Fluent Bit + $FLB_BIN -c conf/truncate_link.conf & + FLB_PID=$! + echo "Fluent Bit started, pid=$FLB_PID" + + # Wait 1 second before truncation + sleep 1 + pre_lines=`cat $TEST_DIR/a.log | wc -l` + truncate -s 0 $TEST_DIR/a.log + + sleep 2 + lines=`cat $TEST_DIR/a | wc -l` + echo "file truncated, mid-check: processed lines $lines" + + # Append 200 more lines + python logger_file.py -l 200 -s 200000 -b 100 -d 0 -f $TEST_DIR/a.log + + sleep 4 + + # Count number of processed lines + write_lines=300 + read_lines=`cat $TEST_DIR/a | wc -l` + + echo "> write lines: $write_lines" + echo "> read lines : $read_lines" + + # Check we processed same number of records + ${_ASSERT_EQUALS_} $write_lines $read_lines + + # Stop Fluent Bit (SIGTERM) + kill -15 $FLB_PID +} + +# 6. Database Resume +# +# Tests that if Fluent Bit stops, it can resume reading files from the last position +# +# Configuration file used: conf/database_resume.conf + +test_database_resume() { + # Helper function to check monitored files + sqlite_check() + { + # Incoming parameters: + # $1: temporal directory to store data + # $2: database file name + # $3: Fluent Bit PID + # + # This function store the remaining monitored files listed in the database, + # we send the output to an .inodes for troubleshooting purposes if required + + # Get the last size of the 'a.log' file and check we have the same value + # in the database + offset=`stat -c %s $TEST_DIR/a.log` + + sqlite3 $1/$2 -batch \ + ".headers off" "SELECT inode FROM in_tail_files WHERE offset=$offset" > \ + $1/$2.offset + + rows=`cat $1/$2.offset | wc -l` + if [ $rows != "1" ]; then + echo "> invalid database content:" + cat $1/$2.offset + echo "> open files" + ls -l /proc/$3/fd/ | grep \\.log + else + echo "> database file $1/$2 is OK" + fi + ${_ASSERT_EQUALS_} "1" $rows + } + + # Prepare test directory + export TEST_DIR=tmp_test + rm -rf $TEST_DIR + mkdir $TEST_DIR + + # Start the Logger: 5 files with 10000 lines each, we use a big size limit (-s) to + # avoid rotation + python logger_file.py -l 10000 -s 200000 -b 100 -d 0 \ + -f $TEST_DIR/a.log \ + -f $TEST_DIR/b.log \ + -f $TEST_DIR/c.log \ + -f $TEST_DIR/d.log \ + -f $TEST_DIR/e.log + + lines=`cat $TEST_DIR/[abcde].log | wc -l` + echo "Logger done, written lines "$lines + + # Run Fluent Bit + $FLB_BIN -c conf/database_resume.conf & + FLB_PID=$! + echo "Fluent Bit started, pid=$FLB_PID" + + # Wait 1 second before stop Fluent Bit + sleep 1 + + # Stop Fluent Bit (SIGTERM) + kill -15 $FLB_PID + + sqlite3 $TEST_DIR/logs.db "SELECT * FROM in_tail_files" + exit 0 + + sleep 2 + lines=`cat $TEST_DIR/a | wc -l` + echo "file truncated, mid-check: processed lines $lines" + + # Append 200 more lines + python logger_file.py -l 200 -s 200000 -b 100 -d 0 -f $TEST_DIR/a.log + + sleep 4 + + # Count number of processed lines + write_lines=300 + read_lines=`cat $TEST_DIR/a | wc -l` + + echo "> write lines: $write_lines" + echo "> read lines : $read_lines" + + # Check we processed same number of records + ${_ASSERT_EQUALS_} $write_lines $read_lines + + # Stop Fluent Bit (SIGTERM) + kill -15 $FLB_PID +} + + +# Launch the tests +. $FLB_RUN_TEST diff --git a/fluent-bit/tests/runtime_shell/in_tail_expect.sh b/fluent-bit/tests/runtime_shell/in_tail_expect.sh new file mode 100755 index 00000000..c847d65f --- /dev/null +++ b/fluent-bit/tests/runtime_shell/in_tail_expect.sh @@ -0,0 +1,16 @@ +#!/bin/sh + +test_in_tail_filter_expect() { + rm -rf /tmp/flb_* + + # Monitor this file + echo "{\"key\": \"val\"}" > /tmp/flb_tail_expect_1.log + + # Excluded file + echo "{\"nokey\": \"\"}" > /tmp/flb_tail_expect_2.log + + $FLB_BIN -c $FLB_RUNTIME_SHELL_CONF/in_tail_expect.conf +} + +# The following command launch the unit test +. $FLB_RUNTIME_SHELL_PATH/runtime_shell.env diff --git a/fluent-bit/tests/runtime_shell/runtime_shell.env.in b/fluent-bit/tests/runtime_shell/runtime_shell.env.in new file mode 100644 index 00000000..6a40c513 --- /dev/null +++ b/fluent-bit/tests/runtime_shell/runtime_shell.env.in @@ -0,0 +1,5 @@ +flb_runtime_shell_test_run() { + . $FLB_RUNTIME_SHELL_PATH/../lib/shunit2/shunit2 +} + +flb_runtime_shell_test_run diff --git a/fluent-bit/tests/runtime_shell/tls/certificate.pem b/fluent-bit/tests/runtime_shell/tls/certificate.pem new file mode 100644 index 00000000..b7971987 --- /dev/null +++ b/fluent-bit/tests/runtime_shell/tls/certificate.pem @@ -0,0 +1,22 @@ +-----BEGIN CERTIFICATE----- +MIIDlzCCAn+gAwIBAgIUCl24aoQh3HuyObboivsozKKOb3cwDQYJKoZIhvcNAQEL +BQAwWzELMAkGA1UEBhMCQVUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoM +GEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDEUMBIGA1UEAwwLbGVvLnZjYXAubWUw +HhcNMjIwODE3MTM0OTAyWhcNMjMwODE3MTM0OTAyWjBbMQswCQYDVQQGEwJBVTET +MBEGA1UECAwKU29tZS1TdGF0ZTEhMB8GA1UECgwYSW50ZXJuZXQgV2lkZ2l0cyBQ +dHkgTHRkMRQwEgYDVQQDDAtsZW8udmNhcC5tZTCCASIwDQYJKoZIhvcNAQEBBQAD +ggEPADCCAQoCggEBAJ2cbLtGhSeGPAXKitw+PN7U8CkaSGL7BssPFSuqtuVkJ1AH +puluHq62FHm4UgTyJB/Iundf1kBxiOehGFr9E1fGp6kUety6iBZK3mmmX0NdXY2A +ObqfvxTgPNVPEj+5Qp0CJW6fEBRhHrFgpf+OdKzoKnjSCbl1EyJ1wFZeSNDTx98I +h951xEOCobcXj1qXHDd0snRW5QNKTdSnj+3amqrNqV5DrkCyyb5Rq/VQX9LIyGV8 +tOyq50mUKeK+vOu/oNwfkZqgZ5wlY/AUyPMNvd8UKitnWeQMiMmr3ImQRZLkFnoE +2mi2Xr6ZQhmsweWbTmmUYxr+ej7L5WvBVQEkbd0CAwEAAaNTMFEwHQYDVR0OBBYE +FGliHqbvEsRN36C2w1cCq5M/gGCRMB8GA1UdIwQYMBaAFGliHqbvEsRN36C2w1cC +q5M/gGCRMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBACWCNohz +YbnnDzGXMhGlfV5PywH6+gguqTrtzJlfJB/FGHDMybBvEp9gda5ssPZfPdv6aAvo +hz2XC+uwJqYn76ZcX+LOixsqhoXJ367pT0ggMSUB/UNvGkzMFWuGerxcB5Q7xvsh +0Ui37Ojok7Ozj9Ofk/zroJB1WG3kPDtwoo+GwNmJaDtXhOutRPxCqp5Dw+dC0N0c +TlPCBRB17hx49Zjw+h9sv1VmOUtQcplNZgRL9lguv7eXB6y0sSdpDyDLlTBbRMGl +uVnQ20uKnqqALRfpXDWFL8g01xPNEIfV3KctpJfu1inLWtPuklerl0yeyUh9M1f0 +zCAi0URsGEHMFA0= +-----END CERTIFICATE----- diff --git a/fluent-bit/tests/runtime_shell/tls/private_key.pem b/fluent-bit/tests/runtime_shell/tls/private_key.pem new file mode 100644 index 00000000..b6a70539 --- /dev/null +++ b/fluent-bit/tests/runtime_shell/tls/private_key.pem @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCdnGy7RoUnhjwF +yorcPjze1PApGkhi+wbLDxUrqrblZCdQB6bpbh6uthR5uFIE8iQfyLp3X9ZAcYjn +oRha/RNXxqepFHrcuogWSt5ppl9DXV2NgDm6n78U4DzVTxI/uUKdAiVunxAUYR6x +YKX/jnSs6Cp40gm5dRMidcBWXkjQ08ffCIfedcRDgqG3F49alxw3dLJ0VuUDSk3U +p4/t2pqqzaleQ65Assm+Uav1UF/SyMhlfLTsqudJlCnivrzrv6DcH5GaoGecJWPw +FMjzDb3fFCorZ1nkDIjJq9yJkEWS5BZ6BNpotl6+mUIZrMHlm05plGMa/no+y+Vr +wVUBJG3dAgMBAAECggEANXQ/AEkLkfsZ0lD+RXIqTNzlUttiH4fJpwbHhFbSzvvn +xWHC/zpk15ZTXXDhCGJjVBBNBX2QeazH5N8jFoDslYF/jX2vqbrturnLswNFHeDF +gN8zNRNGyDrBBwtZQhl/+SYoMdtqpa7GrRv9UK4s7hOTjASYXbjSM4bCI8i4Y3JX +5iKYnejTJ4GNXm1S5g4H8QpgXxrxYU7HY1UlMBGni1BSDPb5SxANnFvgw0oypaDG +4a0KpnUmziBtZo8wIQ7+UeStS4xikIeaT2SQM/ilplPNx0HgPHbTMeTBHQylG1JU +ztEE04Y1F9UF7cqZnxqQLlKmIqsOepLeDOkRXjDtNQKBgQDQtWYbewCmGuo1+AR7 +wStOXpfhqdu6NxAwX+S06ZdAV7f4duMFb//6uJ/c997qwRlNvRvpPlJBdcT3qziS +zjvR8uk2YN5N9D+ZN0Yi11vltUYVPeGpoAk4L7+Ruj5RTVqk5iZEZ1RbAv5cRIm0 +D24H4W0Cj/aldDED2pwTb9ukEwKBgQDBUv/s7gUNM3LCkNLc1EDg2adSzoUopxHU +jIQCitXt5iFpY2Lv2tpaIz4ux4HQGwhmao2MiUYOMit0++/pFaKur66tgRQRsr7F +uS1fpjqtS51/5uFuc1xcWjVquidZXH9u9iChrYvim6SBkxzFsbu4kXNOj4I562cm +E2VWH5GETwKBgQCo5ePX4VbJFYbsXeXi8JRHO63V5Uv4Co+DVlcTQOYyH8q1vCBE +Sjrxf29/tugjOllr29o2i0StzMy1UU7bHyKx6M5qP0In+71sFJshnv6ziltI3Wc9 +ilFrstho6jt8OAle4RGe0bAmZunJaX22xbXZkshRBognpTv1Tnh4ElHBGQKBgHIW +nVohjXGg7xTLiuUvjaokSI6hugunrOoWksE9VcqziPw83uJV8Y5IRiYtLvq1OVvX +ffl1+ZXfHa5ID+kqD3uvyhIynrljFxpwkcpkuzQR77zPcDJSeis2QVfey+H8qGe/ +cLp5RJhS6d5eBxjULshZbgbqwhuURKc/wwn0T1gZAoGAXu2wqwBE8ikhpaDVFYZR +EZe0G1iq03CtabFL0oPiddytV9jJy4Mg457tkCj58uCVkxw/8qjXEnm5OoJAzSlJ +Goc4wh1N1rngYRbSGG+LxD0DeI/Hj3T5tWezMBLCE7J+hieUuVjOB9ELKip7aTj3 +KLNVudOATWYGXyvzux6ciu0= +-----END PRIVATE KEY----- |