diff options
Diffstat (limited to 'fluent-bit/tests/runtime_shell')
33 files changed, 0 insertions, 1907 deletions
diff --git a/fluent-bit/tests/runtime_shell/.gitignore b/fluent-bit/tests/runtime_shell/.gitignore deleted file mode 100644 index bf8c81f4d..000000000 --- a/fluent-bit/tests/runtime_shell/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -runtime_shell.env - diff --git a/fluent-bit/tests/runtime_shell/CMakeLists.txt b/fluent-bit/tests/runtime_shell/CMakeLists.txt deleted file mode 100644 index 79c96de5f..000000000 --- a/fluent-bit/tests/runtime_shell/CMakeLists.txt +++ /dev/null @@ -1,29 +0,0 @@ -configure_file( - "${CMAKE_CURRENT_SOURCE_DIR}/runtime_shell.env.in" - "${CMAKE_CURRENT_SOURCE_DIR}/runtime_shell.env" - ) - -set(UNIT_TESTS_SH - in_dummy_expect.sh - in_tail_expect.sh - in_http_tls_expect.sh - in_syslog_tcp_tls_expect.sh - in_syslog_tcp_plaintext_expect.sh - in_syslog_udp_plaintext_expect.sh - in_syslog_uds_dgram_plaintext_expect.sh - in_syslog_uds_stream_plaintext_expect.sh - ) - -# Prepare list of unit tests -foreach(script ${UNIT_TESTS_SH}) - add_test(NAME ${script} - COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/${script} - ) - - set_tests_properties(${script} PROPERTIES ENVIRONMENT - "FLB_ROOT=${PROJECT_SOURCE_DIR};\ -FLB_RUNTIME_SHELL_PATH=${CMAKE_CURRENT_SOURCE_DIR};\ -FLB_RUNTIME_SHELL_CONF=${CMAKE_CURRENT_SOURCE_DIR}/conf;\ -FLB_BIN=${CMAKE_BINARY_DIR}/bin/fluent-bit" - ) -endforeach() diff --git a/fluent-bit/tests/runtime_shell/common.sh b/fluent-bit/tests/runtime_shell/common.sh deleted file mode 100644 index 1f4ec663a..000000000 --- a/fluent-bit/tests/runtime_shell/common.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/sh - -# This function is meant to be used alongside with -# in_expect_base.conf or a similar setup where -# fluent-bit is set up to create a file as soon -# as it starts. -# -# I would rather use pidof but it's not viable. -# -wait_for_fluent_bit() { - result=1 - - for retry in `seq 10` - do - if test -f $1 - then - sleep 1 - result=0 - break - fi - - sleep 1 - done - - echo "$result" -} diff --git a/fluent-bit/tests/runtime_shell/conf/in_dummy_expect.conf b/fluent-bit/tests/runtime_shell/conf/in_dummy_expect.conf deleted file mode 100644 index 1b72d64e8..000000000 --- a/fluent-bit/tests/runtime_shell/conf/in_dummy_expect.conf +++ /dev/null @@ -1,25 +0,0 @@ -[SERVICE] - Flush 1 - Grace 2 - Log_Level info - -[INPUT] - Name dummy - Dummy {"r1": "someval", "s1": {"s2": null}, "r2": {"x": 0}} - Samples 1 - -[FILTER] - Name expect - Match * - Log_Level debug - # Rules - key_exists $r1 - key_not_exists $r0 - key_val_is_null $s1['s2'] - key_val_is_not_null $r2['x'] - key_val_eq $r1 someval - action exit - -[OUTPUT] - Name exit - Match * diff --git a/fluent-bit/tests/runtime_shell/conf/in_expect_base.conf b/fluent-bit/tests/runtime_shell/conf/in_expect_base.conf deleted file mode 100644 index 760b47d53..000000000 --- a/fluent-bit/tests/runtime_shell/conf/in_expect_base.conf +++ /dev/null @@ -1,28 +0,0 @@ -[SERVICE] - Flush 1 - Grace 1 - Log_Level error - Parsers_File ${FLB_ROOT}/conf/parsers.conf - -[INPUT] - name dummy - samples 1 - -[OUTPUT] - name file - match dummy.* - file ${SIGNAL_FILE_PATH} - mkdir on - -[FILTER] - Name expect - Match target_input - Log_Level debug - # Rules - key_exists $message - key_val_eq $message Hello! - action exit - -[OUTPUT] - Name exit - Match target_input diff --git a/fluent-bit/tests/runtime_shell/conf/in_http_tls_expect.conf b/fluent-bit/tests/runtime_shell/conf/in_http_tls_expect.conf deleted file mode 100644 index 2b66dff87..000000000 --- a/fluent-bit/tests/runtime_shell/conf/in_http_tls_expect.conf +++ /dev/null @@ -1,13 +0,0 @@ -@INCLUDE in_expect_base.conf - -[INPUT] - name http - tag target_input - listen ${LISTENER_HOST} - port ${LISTENER_PORT} - tls.verify no - tls on - tls.vhost ${LISTENER_VHOST} - tls.debug 4 - tls.crt_file ${FLB_RUNTIME_SHELL_PATH}/tls/certificate.pem - tls.key_file ${FLB_RUNTIME_SHELL_PATH}/tls/private_key.pem
\ No newline at end of file diff --git a/fluent-bit/tests/runtime_shell/conf/in_syslog_tcp_plaintext_expect.conf b/fluent-bit/tests/runtime_shell/conf/in_syslog_tcp_plaintext_expect.conf deleted file mode 100644 index d47f6e335..000000000 --- a/fluent-bit/tests/runtime_shell/conf/in_syslog_tcp_plaintext_expect.conf +++ /dev/null @@ -1,8 +0,0 @@ -@INCLUDE in_expect_base.conf - -[INPUT] - name syslog - tag target_input - listen ${LISTENER_HOST} - port ${LISTENER_PORT} - mode tcp
\ No newline at end of file diff --git a/fluent-bit/tests/runtime_shell/conf/in_syslog_tcp_tls_expect.conf b/fluent-bit/tests/runtime_shell/conf/in_syslog_tcp_tls_expect.conf deleted file mode 100644 index afa28e6f5..000000000 --- a/fluent-bit/tests/runtime_shell/conf/in_syslog_tcp_tls_expect.conf +++ /dev/null @@ -1,14 +0,0 @@ -@INCLUDE in_expect_base.conf - -[INPUT] - name syslog - tag target_input - listen ${LISTENER_HOST} - port ${LISTENER_PORT} - mode tcp - tls.verify no - tls on - tls.vhost ${LISTENER_VHOST} - tls.debug 4 - tls.crt_file ${FLB_RUNTIME_SHELL_PATH}/tls/certificate.pem - tls.key_file ${FLB_RUNTIME_SHELL_PATH}/tls/private_key.pem
\ No newline at end of file diff --git a/fluent-bit/tests/runtime_shell/conf/in_syslog_udp_plaintext_expect.conf b/fluent-bit/tests/runtime_shell/conf/in_syslog_udp_plaintext_expect.conf deleted file mode 100644 index ae92b39e5..000000000 --- a/fluent-bit/tests/runtime_shell/conf/in_syslog_udp_plaintext_expect.conf +++ /dev/null @@ -1,8 +0,0 @@ -@INCLUDE in_expect_base.conf - -[INPUT] - name syslog - tag target_input - listen ${LISTENER_HOST} - port ${LISTENER_PORT} - mode udp
\ No newline at end of file diff --git a/fluent-bit/tests/runtime_shell/conf/in_syslog_uds_dgram_plaintext_expect.conf b/fluent-bit/tests/runtime_shell/conf/in_syslog_uds_dgram_plaintext_expect.conf deleted file mode 100644 index 48b6c375a..000000000 --- a/fluent-bit/tests/runtime_shell/conf/in_syslog_uds_dgram_plaintext_expect.conf +++ /dev/null @@ -1,7 +0,0 @@ -@INCLUDE in_expect_base.conf - -[INPUT] - name syslog - tag target_input - path ${SOCKET_PATH} - mode unix_udp
\ No newline at end of file diff --git a/fluent-bit/tests/runtime_shell/conf/in_syslog_uds_stream_plaintext_expect.conf b/fluent-bit/tests/runtime_shell/conf/in_syslog_uds_stream_plaintext_expect.conf deleted file mode 100644 index cfebdd24d..000000000 --- a/fluent-bit/tests/runtime_shell/conf/in_syslog_uds_stream_plaintext_expect.conf +++ /dev/null @@ -1,7 +0,0 @@ -@INCLUDE in_expect_base.conf - -[INPUT] - name syslog - tag target_input - path ${SOCKET_PATH} - mode unix_tcp
\ No newline at end of file diff --git a/fluent-bit/tests/runtime_shell/conf/in_tail_expect.conf b/fluent-bit/tests/runtime_shell/conf/in_tail_expect.conf deleted file mode 100644 index 7348bacc0..000000000 --- a/fluent-bit/tests/runtime_shell/conf/in_tail_expect.conf +++ /dev/null @@ -1,38 +0,0 @@ -[SERVICE] - Flush 1 - Grace 2 - Log_Level info - Parsers_File ${FLB_ROOT}/conf/parsers.conf - -[INPUT] - name tail - path /tmp/flb_tail_expect*.log - exclude_path /tmp/flb_*2.log - read_from_head true - parser json - refresh_interval 10 - rotate_wait 1 - docker_mode false - docker_mode_flush 4 - path_key path_key - ignore_older 0 - buffer_chunk_size 32k - buffer_max_size 32k - skip_long_lines false - exit_on_eof false - db /tmp/flb_tail_expect.db - db.sync full - -[FILTER] - Name expect - Match * - Log_Level debug - # Rules - key_exists $path_key - key_val_eq $path_key /tmp/flb_tail_expect_1.log - key_not_exists $nokey - action exit - -[OUTPUT] - Name exit - Match * diff --git a/fluent-bit/tests/runtime_shell/in_dummy_expect.sh b/fluent-bit/tests/runtime_shell/in_dummy_expect.sh deleted file mode 100755 index b75477634..000000000 --- a/fluent-bit/tests/runtime_shell/in_dummy_expect.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/sh - -test_in_dummy_filter_expect() { - $FLB_BIN -c $FLB_RUNTIME_SHELL_CONF/in_dummy_expect.conf -} - -# The following command launch the unit test -. $FLB_RUNTIME_SHELL_PATH/runtime_shell.env diff --git a/fluent-bit/tests/runtime_shell/in_http_tls_expect.sh b/fluent-bit/tests/runtime_shell/in_http_tls_expect.sh deleted file mode 100755 index fb47e2ae2..000000000 --- a/fluent-bit/tests/runtime_shell/in_http_tls_expect.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/sh - -. ${FLB_RUNTIME_SHELL_PATH}/common.sh - -input_generator() { - result=$(wait_for_fluent_bit) - - if test "$result" -eq "0" - then - curl -s -k \ - -H 'content-type: application/json' \ - -d '{"message": "Hello!"}' \ - "https://${LISTENER_HOST}:${LISTENER_PORT}" - fi -} - -test_in_http_tls_filter_expect() { - export SIGNAL_FILE_PATH="/tmp/fb_signal_$$" - export LISTENER_VHOST=leo.vcap.me - export LISTENER_HOST=127.0.0.1 - export LISTENER_PORT=50000 - - input_generator & - - $FLB_BIN -c $FLB_RUNTIME_SHELL_CONF/in_http_tls_expect.conf -} - -# The following command launch the unit test -. $FLB_RUNTIME_SHELL_PATH/runtime_shell.env diff --git a/fluent-bit/tests/runtime_shell/in_syslog_tcp_plaintext_expect.sh b/fluent-bit/tests/runtime_shell/in_syslog_tcp_plaintext_expect.sh deleted file mode 100755 index 414d40698..000000000 --- a/fluent-bit/tests/runtime_shell/in_syslog_tcp_plaintext_expect.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/sh - -. ${FLB_RUNTIME_SHELL_PATH}/common.sh - -input_generator() { - result=$(wait_for_fluent_bit ${SIGNAL_FILE_PATH}) - - if test "$result" -eq "0" - then - echo '<13>1 1970-01-01T00:00:00.000000+00:00 testhost testuser - - [] Hello!' | \ - nc -w 1 $LISTENER_HOST $LISTENER_PORT - fi -} - -test_in_syslog_tcp_plaintext_filter_expect() { - export SIGNAL_FILE_PATH="/tmp/fb_signal_$$" - export LISTENER_HOST=127.0.0.1 - export LISTENER_PORT=50001 - - input_generator & - - $FLB_BIN -c $FLB_RUNTIME_SHELL_CONF/in_syslog_tcp_plaintext_expect.conf -} - -# The following command launch the unit test -. $FLB_RUNTIME_SHELL_PATH/runtime_shell.env diff --git a/fluent-bit/tests/runtime_shell/in_syslog_tcp_tls_expect.sh b/fluent-bit/tests/runtime_shell/in_syslog_tcp_tls_expect.sh deleted file mode 100755 index d76c0e3c0..000000000 --- a/fluent-bit/tests/runtime_shell/in_syslog_tcp_tls_expect.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/sh - -. ${FLB_RUNTIME_SHELL_PATH}/common.sh - -input_generator() { - result=$(wait_for_fluent_bit ${SIGNAL_FILE_PATH}) - - if test "$result" -eq "0" - then - echo '<13>1 1970-01-01T00:00:00.000000+00:00 testhost testuser - - [] Hello!' | \ - openssl s_client -connect $LISTENER_HOST:$LISTENER_PORT 2>&1 >/dev/null - fi -} - -test_in_syslog_tcp_plaintext_filter_expect() { - export SIGNAL_FILE_PATH="/tmp/fb_signal_$$" - export LISTENER_VHOST=leo.vcap.me - export LISTENER_HOST=127.0.0.1 - export LISTENER_PORT=50002 - - input_generator & - - $FLB_BIN -c $FLB_RUNTIME_SHELL_CONF/in_syslog_tcp_tls_expect.conf -} - -# The following command launch the unit test -. $FLB_RUNTIME_SHELL_PATH/runtime_shell.env diff --git a/fluent-bit/tests/runtime_shell/in_syslog_udp_plaintext_expect.sh b/fluent-bit/tests/runtime_shell/in_syslog_udp_plaintext_expect.sh deleted file mode 100755 index 23970a134..000000000 --- a/fluent-bit/tests/runtime_shell/in_syslog_udp_plaintext_expect.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/sh - -. ${FLB_RUNTIME_SHELL_PATH}/common.sh - -input_generator() { - result=$(wait_for_fluent_bit ${SIGNAL_FILE_PATH}) - - if test "$result" -eq "0" - then - echo '<13>1 1970-01-01T00:00:00.000000+00:00 testhost testuser - - [] Hello!' | \ - nc -w 1 -u $LISTENER_HOST $LISTENER_PORT - fi -} - -test_in_syslog_tcp_plaintext_filter_expect() { - export SIGNAL_FILE_PATH="/tmp/fb_signal_$$" - export LISTENER_HOST=127.0.0.1 - export LISTENER_PORT=50003 - - input_generator & - - $FLB_BIN -c $FLB_RUNTIME_SHELL_CONF/in_syslog_udp_plaintext_expect.conf -} - -# The following command launch the unit test -. $FLB_RUNTIME_SHELL_PATH/runtime_shell.env diff --git a/fluent-bit/tests/runtime_shell/in_syslog_uds_dgram_plaintext_expect.sh b/fluent-bit/tests/runtime_shell/in_syslog_uds_dgram_plaintext_expect.sh deleted file mode 100755 index 2ab7703b2..000000000 --- a/fluent-bit/tests/runtime_shell/in_syslog_uds_dgram_plaintext_expect.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/sh - -. ${FLB_RUNTIME_SHELL_PATH}/common.sh - -input_generator() { - result=$(wait_for_fluent_bit ${SIGNAL_FILE_PATH}) - - if test "$result" -eq "0" - then - echo '<13>Jan 1 00:00:00 testuser: Hello!' | nc -w 1 -U -u $SOCKET_PATH - fi -} - -test_in_syslog_uds_stream_plaintext_filter_expect() { - platform=$(uname) - - if test "$platform" != "Darwin" - then - export SIGNAL_FILE_PATH="/tmp/fb_signal_$$" - export SOCKET_PATH=/tmp/fluent_bit_syslog_uds_dgram.sock - - input_generator & - - $FLB_BIN -c $FLB_RUNTIME_SHELL_CONF/in_syslog_uds_dgram_plaintext_expect.conf - fi -} - -# The following command launch the unit test -. $FLB_RUNTIME_SHELL_PATH/runtime_shell.env diff --git a/fluent-bit/tests/runtime_shell/in_syslog_uds_stream_plaintext_expect.sh b/fluent-bit/tests/runtime_shell/in_syslog_uds_stream_plaintext_expect.sh deleted file mode 100755 index b3a38cc22..000000000 --- a/fluent-bit/tests/runtime_shell/in_syslog_uds_stream_plaintext_expect.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/sh - -. ${FLB_RUNTIME_SHELL_PATH}/common.sh - -input_generator() { - result=$(wait_for_fluent_bit ${SIGNAL_FILE_PATH}) - - if test "$result" -eq "0" - then - echo '<13>Jan 1 00:00:00 testuser: Hello!' | nc -w 1 -U $SOCKET_PATH - fi -} - -test_in_syslog_uds_stream_plaintext_filter_expect() { - export SIGNAL_FILE_PATH="/tmp/fb_signal_$$" - export SOCKET_PATH=/tmp/fluent_bit_syslog_uds_stream.sock - - input_generator & - - $FLB_BIN -c $FLB_RUNTIME_SHELL_CONF/in_syslog_uds_stream_plaintext_expect.conf -} - -# The following command launch the unit test -. $FLB_RUNTIME_SHELL_PATH/runtime_shell.env diff --git a/fluent-bit/tests/runtime_shell/in_tail/README.md b/fluent-bit/tests/runtime_shell/in_tail/README.md deleted file mode 100644 index 78c77525f..000000000 --- a/fluent-bit/tests/runtime_shell/in_tail/README.md +++ /dev/null @@ -1,126 +0,0 @@ -# Fluent Bit Tail Input Plugin Tests - -The following directory contains tests for Tail input plugin behaviors. - -## run_tests.sh - -This script provide validations for offsets, database file entries and rotation under different scenarios. The following tests are available in the script - -- test_normal_rotation -- test_single_static_rotation -- test_truncate -- test_rotate_link -- test_truncate_link - -Running the script ```test_rotation.sh``` will run every test listed above, to run a single test just append it name, e.g: - -``` -./test_rotation.sh -- test_truncate -``` - -### 1. Normal Rotation - -**Unit** - -```test_normal_rotation``` - -**Description** - -Run the logger tool that creates 5 different files, write 100000 messages to each one while rotating at 256KB. - -This test enable the database backend for Tail so it also helps to validate expected entries into the 'in_tail_files' table. - -**Configuration File** - -```conf/normal_rotation.conf``` - -### 2. Single Static Rotation - -**Unit** - -```test_single_static_rotation``` - -**Description** - -Run the logger tool that creates 1 big file and let Fluent Bit process it in the static mode, before to promote it to 'events' and it gets rotated. - -**Configuration File** - -```conf/single_static_rotation.conf``` - -### 3. Truncate - -**Unit** - -```test_truncate``` - -**Description** - - Some environments still rely on truncation mode or well known as copytruncate, - this is the definition by logrotate(8): - -> Truncate the original log file to zero size in place after creating a copy, -> instead of moving the old log file and optionally creating a new one. It -> can be used when some program cannot be told to close its logfile and -> thus might continue writing (appending) to the previous log file forever. -> -> Note that there is a very small time slice between copying the file and -> truncating it, so some logging data might be lost. When this option is -> used, the create option will have no effect, as the old log file stays in -> place. - -This test checks that after a truncation the new lines added are properly -processed. - -**Configuration File** - -```conf/truncate_rotation.conf``` - -### 4. Rotate Link - -**Unit** - -```test_rotate_link``` - -**Description** - -This test checks that a monitored link, upon rotation, keeps the proper offset and database status for the real file. - - Example: - - - file with data: data.log - - monitored link: test.log - - Check the behavior upon the following rotation: test.log -> test.log.1 - -**Configuration File** - -```conf/rotate_link.conf``` - -### 5. Truncate Link - -**Unit** - -```test_truncate_link``` - -**Description** - -Test a link that gets a truncation and Fluent Bit properly use the new offset - -**Configuration File** - -```conf/truncate_link.conf``` - -### 6. Multiline Rotation - -**Unit** - -```test_multiline_rotation``` - -**Description** - -Test a multiline rotation for issue 4190. - -**Configuration File** - -```conf/multiline_rotation.conf``` diff --git a/fluent-bit/tests/runtime_shell/in_tail/conf/multiline_rotation.conf b/fluent-bit/tests/runtime_shell/in_tail/conf/multiline_rotation.conf deleted file mode 100644 index 3695014a5..000000000 --- a/fluent-bit/tests/runtime_shell/in_tail/conf/multiline_rotation.conf +++ /dev/null @@ -1,60 +0,0 @@ -[SERVICE] - flush 1 - daemon off - log_level debug - log_file ${TEST_DIR}/out.log - -[INPUT] - name tail - tag a - path ${TEST_DIR}/a.log - db ${TEST_DIR}/a.db - db.sync full - multiline.parser cri - rotate_wait 5 - refresh_interval 2 - -[INPUT] - name tail - tag b - path ${TEST_DIR}/b.log - db ${TEST_DIR}/b.db - db.sync full - multiline.parser cri - rotate_wait 5 - refresh_interval 2 - -[INPUT] - name tail - tag c - path ${TEST_DIR}/c.log - db ${TEST_DIR}/c.db - db.sync full - multiline.parser cri - rotate_wait 5 - refresh_interval 2 - -[INPUT] - name tail - tag d - path ${TEST_DIR}/d.log - db ${TEST_DIR}/d.db - db.sync full - multiline.parser cri - rotate_wait 5 - refresh_interval 2 - -[INPUT] - name tail - tag e - path ${TEST_DIR}/e.log - db ${TEST_DIR}/e.db - db.sync full - multiline.parser cri - rotate_wait 5 - refresh_interval 2 - -[OUTPUT] - name file - match * - path ${TEST_DIR} diff --git a/fluent-bit/tests/runtime_shell/in_tail/conf/normal_rotation.conf b/fluent-bit/tests/runtime_shell/in_tail/conf/normal_rotation.conf deleted file mode 100644 index 0265df5d1..000000000 --- a/fluent-bit/tests/runtime_shell/in_tail/conf/normal_rotation.conf +++ /dev/null @@ -1,55 +0,0 @@ -[SERVICE] - flush 1 - daemon off - log_level debug - log_file ${TEST_DIR}/out.log - -[INPUT] - name tail - tag a - path ${TEST_DIR}/a.log - db ${TEST_DIR}/a.db - db.sync full - rotate_wait 5 - refresh_interval 2 - -[INPUT] - name tail - tag b - path ${TEST_DIR}/b.log - db ${TEST_DIR}/b.db - db.sync full - rotate_wait 5 - refresh_interval 2 - -[INPUT] - name tail - tag c - path ${TEST_DIR}/c.log - db ${TEST_DIR}/c.db - db.sync full - rotate_wait 5 - refresh_interval 2 - -[INPUT] - name tail - tag d - path ${TEST_DIR}/d.log - db ${TEST_DIR}/d.db - db.sync full - rotate_wait 5 - refresh_interval 2 - -[INPUT] - name tail - tag e - path ${TEST_DIR}/e.log - db ${TEST_DIR}/e.db - db.sync full - rotate_wait 5 - refresh_interval 2 - -[OUTPUT] - name file - match * - path ${TEST_DIR} diff --git a/fluent-bit/tests/runtime_shell/in_tail/conf/rotate_link.conf b/fluent-bit/tests/runtime_shell/in_tail/conf/rotate_link.conf deleted file mode 100644 index f0b017d44..000000000 --- a/fluent-bit/tests/runtime_shell/in_tail/conf/rotate_link.conf +++ /dev/null @@ -1,20 +0,0 @@ -[SERVICE] - flush 1 - daemon off - log_level debug - log_file ${TEST_DIR}/out.log - -[INPUT] - name tail - tag a - path ${TEST_DIR}/a.log - db ${TEST_DIR}/a.db - db.sync full - rotate_wait 5 - watcher_interval 1 - refresh_interval 2 - -[OUTPUT] - name file - match * - path ${TEST_DIR} diff --git a/fluent-bit/tests/runtime_shell/in_tail/conf/single_static_rotation.conf b/fluent-bit/tests/runtime_shell/in_tail/conf/single_static_rotation.conf deleted file mode 100644 index ac883c957..000000000 --- a/fluent-bit/tests/runtime_shell/in_tail/conf/single_static_rotation.conf +++ /dev/null @@ -1,19 +0,0 @@ -[SERVICE] - flush 1 - daemon off - log_level debug - log_file ${TEST_DIR}/out.log - -[INPUT] - name tail - tag a - path ${TEST_DIR}/a.log - db ${TEST_DIR}/a.db - db.sync full - rotate_wait 5 - refresh_interval 2 - -[OUTPUT] - name file - match * - path ${TEST_DIR} diff --git a/fluent-bit/tests/runtime_shell/in_tail/conf/truncate_link.conf b/fluent-bit/tests/runtime_shell/in_tail/conf/truncate_link.conf deleted file mode 100644 index 6cfa5410c..000000000 --- a/fluent-bit/tests/runtime_shell/in_tail/conf/truncate_link.conf +++ /dev/null @@ -1,19 +0,0 @@ -[SERVICE] - flush 1 - daemon off - log_level debug - log_file ${TEST_DIR}/out.log - -[INPUT] - name tail - tag a - path ${TEST_DIR}/a.log - db ${TEST_DIR}/a.db - db.sync full - rotate_wait 5 - refresh_interval 1 - -[OUTPUT] - name file - match * - path ${TEST_DIR} diff --git a/fluent-bit/tests/runtime_shell/in_tail/conf/truncate_rotation.conf b/fluent-bit/tests/runtime_shell/in_tail/conf/truncate_rotation.conf deleted file mode 100644 index ac883c957..000000000 --- a/fluent-bit/tests/runtime_shell/in_tail/conf/truncate_rotation.conf +++ /dev/null @@ -1,19 +0,0 @@ -[SERVICE] - flush 1 - daemon off - log_level debug - log_file ${TEST_DIR}/out.log - -[INPUT] - name tail - tag a - path ${TEST_DIR}/a.log - db ${TEST_DIR}/a.db - db.sync full - rotate_wait 5 - refresh_interval 2 - -[OUTPUT] - name file - match * - path ${TEST_DIR} diff --git a/fluent-bit/tests/runtime_shell/in_tail/logger_file.py b/fluent-bit/tests/runtime_shell/in_tail/logger_file.py deleted file mode 100755 index a8e2a1b39..000000000 --- a/fluent-bit/tests/runtime_shell/in_tail/logger_file.py +++ /dev/null @@ -1,72 +0,0 @@ -#!/bin/env python - -import sys -import uuid -import time -import signal -import logging -from argparse import ArgumentParser -from logging.handlers import RotatingFileHandler -from threading import Thread - -class LoggerManager: - def __init__(self, args): - # KB to bytes - self.max_bytes = (args.size * 1000) - self.backup = args.backup - self.lines = args.lines - self.delay = args.delay - self.threads = [] - - # Create a thread for every writer - for f in args.filenames: - thread = Thread(target = self.single_logger_thread, args = (f,)) - if thread is None: - print("error creating thread") - sys.exit(1) - self.threads.append(thread) - thread.start() - print("Logger thread for '" + f + "' has started") - - for th in self.threads: - th.join() - print("Logger thread finished") - - def single_logger_thread(self, name): - logger = logging.getLogger(name) - logger.setLevel(logging.DEBUG) - handler = RotatingFileHandler(name, maxBytes = self.max_bytes, - backupCount = self.backup) - logger.addHandler(handler) - rnd = uuid.uuid4() - - i = 0 - while i < self.lines: - logger.debug(rnd) - if self.delay > 0.0: - time.sleep(self.delay / 1000.0) - i = i + 1 - -def signal_handler(sig, frame): - print("stopping logger") - sys.exit(0) - -if __name__ == '__main__': - signal.signal(signal.SIGINT, signal_handler) - - # Define arguments - parser = ArgumentParser() - parser.add_argument("-b", "--backup", dest="backup", default=50, type=int) - parser.add_argument("-d", "--delay", dest="delay", default=0.1, type=float, - help="milliseconds delay between line writes") - parser.add_argument("-l", "--lines", dest="lines", default=1000, type=int) - parser.add_argument("-f", "--file", dest="filenames", action='append', required=True, - help="write logs to FILE", metavar="FILE") - parser.add_argument("-s", "--size", dest="size", type=int, - help="maximum log file size in KB before rotation", - default=256) - # Read arguments - args = parser.parse_args() - - # Start the Logger - lm = LoggerManager(args) diff --git a/fluent-bit/tests/runtime_shell/in_tail/run_tests.sh b/fluent-bit/tests/runtime_shell/in_tail/run_tests.sh deleted file mode 100755 index 09bb9c438..000000000 --- a/fluent-bit/tests/runtime_shell/in_tail/run_tests.sh +++ /dev/null @@ -1,530 +0,0 @@ -#!/bin/sh - -# Environment variables -FLB_BIN=`realpath ../../../build/bin/fluent-bit` -FLB_RUNTIME_SHELL_PATH=`realpath $(pwd)/../` -FLB_RUN_TEST=`realpath $FLB_RUNTIME_SHELL_PATH/../lib/shunit2/shunit2` - -# Colorize shunit2 -bold=$(tput bold) -normal=$(tput sgr0) -SHUNIT_TEST_PREFIX="$bold==========> UNIT TEST: $normal" - -# 1. Normal Rotation -# ------------------ -# Run the logger tool that creates 5 different files, write 100000 messages to each one -# while rotating at 256KB. -# -# This test enable the database backend for Tail so it also helps to validate expected -# entries into the 'in_tail_files' table. -# -# Configuration file used: conf/normal_rotation.conf - -test_normal_rotation() { - # Helper function to check monitored files - sqlite_check() - { - # Incoming parameters: - # $1: temporal directory to store data - # $2: database file name - # $3: Fluent Bit PID - # - # This function store the remaining monitored files listed in the database, - # we send the output to an .inodes for troubleshooting purposes if required - sqlite3 $1/$2 -batch \ - ".headers off" ".width 20" "SELECT inode FROM in_tail_files" > \ - $1/$2.inodes - - rows=`cat $1/$2.inodes | wc -l | tr -d -C '[0-9]'` - if [ $rows != "1" ]; then - echo "> database file $1/$2 contains $rows rows, inodes:" - cat $1/$2.inodes - echo "> open files" - ls -l /proc/$3/fd/ | grep \\.log - else - echo "> database file $1/$2 is OK" - fi - ${_ASSERT_EQUALS_} "1" $rows - } - - # Prepare test directory - export TEST_DIR=tmp_test - rm -rf $TEST_DIR - mkdir $TEST_DIR - - # Create empty files so Fluent Bit will enqueue them on start - for logfile in a b c d e ; do - touch $TEST_DIR/$logfile.log - done - - # Run Fluent Bit - $FLB_BIN -c conf/normal_rotation.conf & - FLB_PID=$! - echo "Fluent Bit started, pid=$FLB_PID" - - # Start the Logger: 5 files = 500000 log lines in total - python logger_file.py -l 100000 -s 256 -b 100 -d 0.1 \ - -f $TEST_DIR/a.log \ - -f $TEST_DIR/b.log \ - -f $TEST_DIR/c.log \ - -f $TEST_DIR/d.log \ - -f $TEST_DIR/e.log - - echo "Logger finished...wait 10 seconds" - sleep 10 - - # Count number of processed lines - write_lines=`cat $TEST_DIR/[abcdefghij].log* | wc -l` - read_lines=`cat $TEST_DIR/[abcdefghij] | wc -l` - - echo "> write lines: $write_lines" - echo "> read lines : $read_lines" - - # Check we processed same number of records - ${_ASSERT_EQUALS_} $write_lines $read_lines - - # Validate our database files has only one remaining entry per database file - for logfile in a b c d e; do - sqlite_check $TEST_DIR "$logfile.db" $FLB_PID - done - - # Stop Fluent Bit (SIGTERM) - kill -15 $FLB_PID -} - -# 2. Single Static Rotation (static process mode + rotation) -# ---------------------------------------------------------- -# Run the logger tool that creates 1 big file and let Fluent Bit process it in -# the static mode, before to promote it to 'events' and it gets rotated. -# -# Configuration file used: conf/single_static_rotation.conf - -test_single_static_rotation() { - # Write a log file of 200000 lines - - # Prepare test directory - export TEST_DIR=tmp_test - rm -rf $TEST_DIR - mkdir $TEST_DIR - - # Create empty files so Fluent Bit will enqueue them on start - touch $TEST_DIR/a.log - - # Start the Logger: 1 file with 400000 lines, we use a big size (-s) to - # avoid rotation - python logger_file.py -l 400000 -s 200000 -b 100 -d 0 \ - -f $TEST_DIR/a.log - lines=`cat $TEST_DIR/a.log | wc -l` - echo "Logger done, written lines "$lines - - # Run Fluent Bit - $FLB_BIN -c conf/single_static_rotation.conf & - FLB_PID=$! - echo "Fluent Bit started, pid=$FLB_PID" - - # Wait 3 seconds before rotation - sleep 2 - mv $TEST_DIR/a.log $TEST_DIR/a.log.1 - - lines=`cat $TEST_DIR/a | wc -l` - echo "file Rotated, mid-check: processed lines $lines" - sleep 30 - - # Count number of processed lines - write_lines=`cat $TEST_DIR/a.log.1 | wc -l` - read_lines=`cat $TEST_DIR/a | wc -l` - - echo "> write lines: $write_lines" - echo "> read lines : $read_lines" - - # Check we processed same number of records - ${_ASSERT_EQUALS_} $write_lines $read_lines - - # Validate our database files has only one remaining entry per database file - #sqlite_check $TEST_DIR "$logfile.db" $FLB_PID - - # Stop Fluent Bit (SIGTERM) - kill -15 $FLB_PID -} - -# 3. Truncate -# ----------- -# Some environments still rely on truncation mode or well known as copytruncate, -# this is the definition by logrotate(8): -# -# "Truncate the original log file to zero size in place after creating a copy, -# instead of moving the old log file and optionally creating a new one. It -# can be used when some program cannot be told to close its logfile and -# thus might continue writing (appending) to the previous log file forever. -# -# Note that there is a very small time slice between copying the file and -# truncating it, so some logging data might be lost. When this option is -# used, the create option will have no effect, as the old log file stays in -# place." -# -# This test checks that after a truncation the new lines added are properly -# processed. -# -# Configuration file used: conf/truncate_rotation.conf - -test_truncate() { - # Helper function to check monitored files - sqlite_check() - { - # Incoming parameters: - # $1: temporal directory to store data - # $2: database file name - # $3: Fluent Bit PID - # - # This function store the remaining monitored files listed in the database, - # we send the output to an .inodes for troubleshooting purposes if required - - # Get the last size of the 'a.log' file and check we have the same value - # in the database - offset=`wc -c < $TEST_DIR/a.log` - - sqlite3 $1/$2 -batch \ - ".headers off" "SELECT inode FROM in_tail_files WHERE offset=$offset" > \ - $1/$2.offset - - rows=`cat $1/$2.offset | wc -l | tr -d -C '[0-9]'` - if [ $rows != "1" ]; then - echo "> invalid database content:" - cat $1/$2.offset - echo "> open files" - ls -l /proc/$3/fd/ | grep \\.log - else - echo "> database file $1/$2 is OK" - fi - ${_ASSERT_EQUALS_} "1" $rows - } - - # Prepare test directory - export TEST_DIR=tmp_test - rm -rf $TEST_DIR - mkdir $TEST_DIR - - # Create empty files so Fluent Bit will enqueue them on start - touch $TEST_DIR/a.log - - # Start the Logger: 1 file with 200 lines, we use a big size limit (-s) to - # avoid rotation - python logger_file.py -l 200 -s 200000 -b 100 -d 0 -f $TEST_DIR/a.log - lines=`cat $TEST_DIR/a.log | wc -l` - echo "Logger done, written lines "$lines - - # Run Fluent Bit - $FLB_BIN -c conf/truncate_rotation.conf & - FLB_PID=$! - echo "Fluent Bit started, pid=$FLB_PID" - - # Wait 2 seconds before truncation - sleep 2 - pre_lines=`cat $TEST_DIR/a.log | wc -l` - truncate -s 0 $TEST_DIR/a.log - - lines=`cat $TEST_DIR/a | wc -l` - echo "file truncated, mid-check: processed lines $lines" - - # Append 100 more lines - python logger_file.py -l 100 -s 200000 -b 100 -d 0 -f $TEST_DIR/a.log - - sleep 3 - - # Count number of processed lines - write_lines=300 - read_lines=`cat $TEST_DIR/a | wc -l` - - echo "> write lines: $write_lines" - echo "> read lines : $read_lines" - - # Check we processed same number of records - ${_ASSERT_EQUALS_} $write_lines $read_lines - - sqlite_check $TEST_DIR a.db $FLB_PID - - # Stop Fluent Bit (SIGTERM) - kill -15 $FLB_PID -} - -# 4. Rotate Link -# -------------- -# This case checks that a monitored link, upon rotation, keeps the proper offset -# and database status for the real file. -# -# Example: -# -# - file with data: data.log -# - monitored link: test.log -# -# Check the behavior upon test.log -> test.log.1 behavior -# -# Configuration file used: conf/rotate_link.conf - -test_rotate_link() { - # Helper function to check monitored files - sqlite_check() - { - # Incoming parameters: - # $1: temporal directory to store data - # $2: database file name - # $3: Fluent Bit PID - # - # This function store the remaining monitored files listed in the database, - # we send the output to an .inodes for troubleshooting purposes if required - - # Get the last size of the file pointed by 'a.log.1' and check we have the - # same value in the database - offset=`wc -c < $TEST_DIR/a.log.1` - - sqlite3 $1/$2 -batch \ - ".headers off" "SELECT inode FROM in_tail_files WHERE offset=$offset \ - AND rotated=1" > $1/$2.offset - - rows=`cat $1/$2.offset | wc -l | tr -d -C '[0-9]'` - if [ $rows != "1" ]; then - echo "> invalid database content:" - cat $1/$2.offset - echo "> open files" - ls -l /proc/$3/fd/ | grep \\.log - else - echo "> offset database check $1/$2 is OK" - fi - ${_ASSERT_EQUALS_} "1" $rows - - # After rotate_wait (5 secs + watcher) we expect an empty database - sleep 6 - sqlite3 $1/$2 -batch \ - ".headers off" "SELECT inode FROM in_tail_files WHERE offset=$offset \ - AND rotated=1" > $1/$2.offset - - rows=`cat $1/$2.offset | wc -l | tr -d -C '[0-9]'` - if [ $rows != "0" ]; then - echo "> invalid database content:" - cat $1/$2.offset - echo "> open files" - ls -l /proc/$3/fd/ | grep \\.log - else - echo "> empty database check $1/$2 is OK" - fi - ${_ASSERT_EQUALS_} "0" $rows - } - - # Prepare test directory - export TEST_DIR=tmp_test - rm -rf $TEST_DIR - mkdir $TEST_DIR - - # Create empty files so Fluent Bit will enqueue them on start - touch $TEST_DIR/data.log - - # Start the Logger: 1 file with 100 lines, we use a big size limit (-s) to - # avoid rotation - python logger_file.py -l 100 -s 200000 -b 100 -d 0 -f $TEST_DIR/data.log - lines=`cat $TEST_DIR/data.log | wc -l` - ln -s data.log $TEST_DIR/a.log - echo "Logger done, written lines "$lines - - # Run Fluent Bit - $FLB_BIN -c conf/rotate_link.conf & - FLB_PID=$! - echo "Fluent Bit started, pid=$FLB_PID" - - # Wait 2 seconds and rotate file - sleep 2 - pre_lines=`cat $TEST_DIR/a.log | wc -l` - mv $TEST_DIR/a.log $TEST_DIR/a.log.1 - - lines=`cat $TEST_DIR/a | wc -l` - echo "file rotated, mid-check: processed lines $lines" - - # Append 200 more lines to the rotated link - python logger_file.py -l 200 -s 200000 -b 100 -d 0 -f $TEST_DIR/a.log.1 - - # Count number of processed lines - sleep 3 - write_lines=300 - read_lines=`cat $TEST_DIR/a | wc -l` - - echo "> write lines: $write_lines" - echo "> read lines : $read_lines" - - # Check we processed same number of records - ${_ASSERT_EQUALS_} $write_lines $read_lines - - # Check that database file have the right offset and mark the file as rotated - sqlite_check $TEST_DIR a.db $FLB_PID - - # Stop Fluent Bit (SIGTERM) - kill -15 $FLB_PID -} - -# 5. Truncate Link -# -# Test a link that gets a truncation and Fluent Bit properly use the new offset -# -# Configuration file used: conf/truncate_link.conf - -test_truncate_link() { - # Helper function to check monitored files - sqlite_check() - { - # Incoming parameters: - # $1: temporal directory to store data - # $2: database file name - # $3: Fluent Bit PID - # - # This function store the remaining monitored files listed in the database, - # we send the output to an .inodes for troubleshooting purposes if required - - # Get the last size of the 'a.log' file and check we have the same value - # in the database - offset=`wc -c < $TEST_DIR/a.log` - - sqlite3 $1/$2 -batch \ - ".headers off" "SELECT inode FROM in_tail_files WHERE offset=$offset" > \ - $1/$2.offset - - rows=`cat $1/$2.offset | wc -l | tr -d -C '[0-9]'` - if [ $rows != "1" ]; then - echo "> invalid database content:" - cat $1/$2.offset - echo "> open files" - ls -l /proc/$3/fd/ | grep \\.log - else - echo "> database file $1/$2 is OK" - fi - ${_ASSERT_EQUALS_} "1" $rows - } - - # Prepare test directory - export TEST_DIR=tmp_test - rm -rf $TEST_DIR - mkdir $TEST_DIR - - # Create empty files so Fluent Bit will enqueue them on start - touch $TEST_DIR/data.log - - # Start the Logger: 1 file with 100 lines, we use a big size limit (-s) to - # avoid rotation - python logger_file.py -l 100 -s 200000 -b 100 -d 0 -f $TEST_DIR/data.log - lines=`cat $TEST_DIR/data.log | wc -l` - ln -s data.log $TEST_DIR/a.log - echo "Logger done, written lines "$lines - - # Run Fluent Bit - $FLB_BIN -c conf/truncate_link.conf & - FLB_PID=$! - echo "Fluent Bit started, pid=$FLB_PID" - - # Wait 1 second before truncation - sleep 1 - pre_lines=`cat $TEST_DIR/a.log | wc -l` - truncate -s 0 $TEST_DIR/a.log - - sleep 2 - lines=`cat $TEST_DIR/a | wc -l` - echo "file truncated, mid-check: processed lines $lines" - - # Append 200 more lines - python logger_file.py -l 200 -s 200000 -b 100 -d 0 -f $TEST_DIR/a.log - - sleep 4 - - # Count number of processed lines - write_lines=300 - read_lines=`cat $TEST_DIR/a | wc -l` - - echo "> write lines: $write_lines" - echo "> read lines : $read_lines" - - # Check we processed same number of records - ${_ASSERT_EQUALS_} $write_lines $read_lines - - # Stop Fluent Bit (SIGTERM) - kill -15 $FLB_PID -} - -# 6. Multiline + rotation -# ------------------ -# Run the logger tool that creates 5 different files, write 100000 messages to each one -# while rotating at 256KB. -# -# This test for issue 4190 -# -# Configuration file used: conf/multiline_rotation.conf - -test_multiline_rotation() { - # Helper function to check monitored files - sqlite_check() - { - # Incoming parameters: - # $1: temporal directory to store data - # $2: database file name - # $3: Fluent Bit PID - # - # This function store the remaining monitored files listed in the database, - # we send the output to an .inodes for troubleshooting purposes if required - sqlite3 $1/$2 -batch \ - ".headers off" ".width 20" "SELECT inode FROM in_tail_files" > \ - $1/$2.inodes - - rows=`cat $1/$2.inodes | wc -l | tr -d -C '[0-9]'` - if [ $rows != "1" ]; then - echo "> database file $1/$2 contains $rows rows, inodes:" - cat $1/$2.inodes - echo "> open files" - ls -l /proc/$3/fd/ | grep \\.log - else - echo "> database file $1/$2 is OK" - fi - ${_ASSERT_EQUALS_} "1" $rows - } - - # Prepare test directory - export TEST_DIR=tmp_test - rm -rf $TEST_DIR - mkdir $TEST_DIR - - # Create empty files so Fluent Bit will enqueue them on start - for logfile in a b c d e ; do - touch $TEST_DIR/$logfile.log - done - - # Run Fluent Bit - $FLB_BIN -c conf/multiline_rotation.conf & - FLB_PID=$! - echo "Fluent Bit started, pid=$FLB_PID" - - # Start the Logger: 5 files = 500000 log lines in total - python logger_file.py -l 100000 -s 256 -b 100 -d 0.1 \ - -f $TEST_DIR/a.log \ - -f $TEST_DIR/b.log \ - -f $TEST_DIR/c.log \ - -f $TEST_DIR/d.log \ - -f $TEST_DIR/e.log - - echo "Logger finished...wait 10 seconds" - sleep 10 - - # Count number of processed lines - write_lines=`cat $TEST_DIR/[abcdefghij].log* | wc -l` - read_lines=`cat $TEST_DIR/[abcdefghij] | wc -l` - - echo "> write lines: $write_lines" - echo "> read lines : $read_lines" - - # Check we processed same number of records - ${_ASSERT_EQUALS_} $write_lines $read_lines - - # Validate our database files has only one remaining entry per database file - for logfile in a b c d e; do - sqlite_check $TEST_DIR "$logfile.db" $FLB_PID - done - - # Stop Fluent Bit (SIGTERM) - kill -15 $FLB_PID -} - -# Launch the tests -. $FLB_RUN_TEST diff --git a/fluent-bit/tests/runtime_shell/in_tail/test_rotation.sh b/fluent-bit/tests/runtime_shell/in_tail/test_rotation.sh deleted file mode 100644 index 889fa7b06..000000000 --- a/fluent-bit/tests/runtime_shell/in_tail/test_rotation.sh +++ /dev/null @@ -1,542 +0,0 @@ -#!/bin/sh - -# Environment variables -FLB_BIN=`realpath ../../../build/bin/fluent-bit` -FLB_RUNTIME_SHELL_PATH=`realpath $(pwd)/../` -FLB_RUN_TEST=`realpath $FLB_RUNTIME_SHELL_PATH/../lib/shunit2/shunit2` - -# Colorize shunit2 -bold=$(tput bold) -normal=$(tput sgr0) -SHUNIT_TEST_PREFIX="$bold==========> UNIT TEST: $normal" - -# 1. Normal Rotation -# ------------------ -# Run the logger tool that creates 5 different files, write 100000 messages to each one -# while rotating at 256KB. -# -# This test enable the database backend for Tail so it also helps to validate expected -# entries into the 'in_tail_files' table. -# -# Configuration file used: conf/normal_rotation.conf - -test_normal_rotation() { - # Helper function to check monitored files - sqlite_check() - { - # Incoming parameters: - # $1: temporal directory to store data - # $2: database file name - # $3: Fluent Bit PID - # - # This function store the remaining monitored files listed in the database, - # we send the output to an .inodes for troubleshooting purposes if required - sqlite3 $1/$2 -batch \ - ".headers off" ".width 20" "SELECT inode FROM in_tail_files" > \ - $1/$2.inodes - - rows=`cat $1/$2.inodes | wc -l | tr -d -C '[0-9]'` - if [ $rows != "1" ]; then - echo "> database file $1/$2 contains $rows rows, inodes:" - cat $1/$2.inodes - echo "> open files" - ls -l /proc/$3/fd/ | grep \\.log - else - echo "> database file $1/$2 is OK" - fi - ${_ASSERT_EQUALS_} "1" $rows - } - - # Prepare test directory - export TEST_DIR=tmp_test - rm -rf $TEST_DIR - mkdir $TEST_DIR - - # Create empty files so Fluent Bit will enqueue them on start - for logfile in a b c d e ; do - touch $TEST_DIR/$logfile.log - done - - # Run Fluent Bit - $FLB_BIN -c conf/normal_rotation.conf & - FLB_PID=$! - echo "Fluent Bit started, pid=$FLB_PID" - - # Start the Logger: 5 files = 500000 log lines in total - python logger_file.py -l 100000 -s 256 -b 100 -d 0.1 \ - -f $TEST_DIR/a.log \ - -f $TEST_DIR/b.log \ - -f $TEST_DIR/c.log \ - -f $TEST_DIR/d.log \ - -f $TEST_DIR/e.log - - echo "Logger finished...wait 10 seconds" - sleep 10 - - # Count number of processed lines - write_lines=`cat $TEST_DIR/[abcdefghij].log* | wc -l` - read_lines=`cat $TEST_DIR/[abcdefghij] | wc -l` - - echo "> write lines: $write_lines" - echo "> read lines : $read_lines" - - # Check we processed same number of records - ${_ASSERT_EQUALS_} $write_lines $read_lines - - # Validate our database files has only one remaining entry per database file - for logfile in a b c d e; do - sqlite_check $TEST_DIR "$logfile.db" $FLB_PID - done - - # Stop Fluent Bit (SIGTERM) - kill -15 $FLB_PID -} - -# 2. Single Static Rotation (static process mode + rotation) -# ---------------------------------------------------------- -# Run the logger tool that creates 1 big file and let Fluent Bit process it in -# the static mode, before to promote it to 'events' and it gets rotated. -# -# Configuration file used: conf/single_static_rotation.conf - -test_single_static_rotation() { - # Write a log file of 200000 lines - - # Prepare test directory - export TEST_DIR=tmp_test - rm -rf $TEST_DIR - mkdir $TEST_DIR - - # Create empty files so Fluent Bit will enqueue them on start - touch $TEST_DIR/a.log - - # Start the Logger: 1 file with 400000 lines, we use a big size (-s) to - # avoid rotation - python logger_file.py -l 400000 -s 200000 -b 100 -d 0 \ - -f $TEST_DIR/a.log - lines=`cat $TEST_DIR/a.log | wc -l` - echo "Logger done, written lines "$lines - - # Run Fluent Bit - $FLB_BIN -c conf/single_static_rotation.conf & - FLB_PID=$! - echo "Fluent Bit started, pid=$FLB_PID" - - # Wait 3 seconds before rotation - sleep 2 - mv $TEST_DIR/a.log $TEST_DIR/a.log.1 - - lines=`cat $TEST_DIR/a | wc -l` - echo "file Rotated, mid-check: processed lines $lines" - sleep 30 - - # Count number of processed lines - write_lines=`cat $TEST_DIR/a.log.1 | wc -l` - read_lines=`cat $TEST_DIR/a | wc -l` - - echo "> write lines: $write_lines" - echo "> read lines : $read_lines" - - # Check we processed same number of records - ${_ASSERT_EQUALS_} $write_lines $read_lines - - # Validate our database files has only one remaining entry per database file - #sqlite_check $TEST_DIR "$logfile.db" $FLB_PID - - # Stop Fluent Bit (SIGTERM) - kill -15 $FLB_PID -} - -# 3. Truncate -# ----------- -# Some environments still rely on truncation mode or well known as copytruncate, -# this is the definition by logrotate(8): -# -# "Truncate the original log file to zero size in place after creating a copy, -# instead of moving the old log file and optionally creating a new one. It -# can be used when some program cannot be told to close its logfile and -# thus might continue writing (appending) to the previous log file forever. -# -# Note that there is a very small time slice between copying the file and -# truncating it, so some logging data might be lost. When this option is -# used, the create option will have no effect, as the old log file stays in -# place." -# -# This test checks that after a truncation the new lines added are properly -# processed. -# -# Configuration file used: conf/truncate_rotation.conf - -test_truncate() { - # Helper function to check monitored files - sqlite_check() - { - # Incoming parameters: - # $1: temporal directory to store data - # $2: database file name - # $3: Fluent Bit PID - # - # This function store the remaining monitored files listed in the database, - # we send the output to an .inodes for troubleshooting purposes if required - - # Get the last size of the 'a.log' file and check we have the same value - # in the database - offset=`wc -c < $TEST_DIR/a.log` - - sqlite3 $1/$2 -batch \ - ".headers off" "SELECT inode FROM in_tail_files WHERE offset=$offset" > \ - $1/$2.offset - - rows=`cat $1/$2.offset | wc -l | tr -d -C '[0-9]'` - if [ $rows != "1" ]; then - echo "> invalid database content:" - cat $1/$2.offset - echo "> open files" - ls -l /proc/$3/fd/ | grep \\.log - else - echo "> database file $1/$2 is OK" - fi - ${_ASSERT_EQUALS_} "1" $rows - } - - # Prepare test directory - export TEST_DIR=tmp_test - rm -rf $TEST_DIR - mkdir $TEST_DIR - - # Create empty files so Fluent Bit will enqueue them on start - touch $TEST_DIR/a.log - - # Start the Logger: 1 file with 200 lines, we use a big size limit (-s) to - # avoid rotation - python logger_file.py -l 200 -s 200000 -b 100 -d 0 -f $TEST_DIR/a.log - lines=`cat $TEST_DIR/a.log | wc -l` - echo "Logger done, written lines "$lines - - # Run Fluent Bit - $FLB_BIN -c conf/truncate_rotation.conf & - FLB_PID=$! - echo "Fluent Bit started, pid=$FLB_PID" - - # Wait 2 seconds before truncation - sleep 2 - pre_lines=`cat $TEST_DIR/a.log | wc -l` - truncate -s 0 $TEST_DIR/a.log - - lines=`cat $TEST_DIR/a | wc -l` - echo "file truncated, mid-check: processed lines $lines" - - # Append 100 more lines - python logger_file.py -l 100 -s 200000 -b 100 -d 0 -f $TEST_DIR/a.log - - sleep 3 - - # Count number of processed lines - write_lines=300 - read_lines=`cat $TEST_DIR/a | wc -l` - - echo "> write lines: $write_lines" - echo "> read lines : $read_lines" - - # Check we processed same number of records - ${_ASSERT_EQUALS_} $write_lines $read_lines - - sqlite_check $TEST_DIR a.db $FLB_PID - - # Stop Fluent Bit (SIGTERM) - kill -15 $FLB_PID -} - -# 4. Rotate Link -# -------------- -# This case checks that a monitored link, upon rotation, keeps the proper offset -# and database status for the real file. -# -# Example: -# -# - file with data: data.log -# - monitored link: test.log -# -# Check the behavior upon test.log -> test.log.1 behavior -# -# Configuration file used: conf/rotate_link.conf - -test_rotate_link() { - # Helper function to check monitored files - sqlite_check() - { - # Incoming parameters: - # $1: temporal directory to store data - # $2: database file name - # $3: Fluent Bit PID - # - # This function store the remaining monitored files listed in the database, - # we send the output to an .inodes for troubleshooting purposes if required - - # Get the last size of the file pointed by 'a.log.1' and check we have the - # same value in the database - offset=`wc -c < $TEST_DIR/a.log.1` - - sqlite3 $1/$2 -batch \ - ".headers off" "SELECT inode FROM in_tail_files WHERE offset=$offset \ - AND rotated=1" > $1/$2.offset - - rows=`cat $1/$2.offset | wc -l | tr -d -C '[0-9]'` - if [ $rows != "1" ]; then - echo "> invalid database content:" - cat $1/$2.offset - echo "> open files" - ls -l /proc/$3/fd/ | grep \\.log - else - echo "> offset database check $1/$2 is OK" - fi - ${_ASSERT_EQUALS_} "1" $rows - - # After rotate_wait (5 secs + watcher) we expect an empty database - sleep 6 - sqlite3 $1/$2 -batch \ - ".headers off" "SELECT inode FROM in_tail_files WHERE offset=$offset \ - AND rotated=1" > $1/$2.offset - - rows=`cat $1/$2.offset | wc -l | tr -d -C '[0-9]'` - if [ $rows != "0" ]; then - echo "> invalid database content:" - cat $1/$2.offset - echo "> open files" - ls -l /proc/$3/fd/ | grep \\.log - else - echo "> empty database check $1/$2 is OK" - fi - ${_ASSERT_EQUALS_} "0" $rows - } - - # Prepare test directory - export TEST_DIR=tmp_test - rm -rf $TEST_DIR - mkdir $TEST_DIR - - # Create empty files so Fluent Bit will enqueue them on start - touch $TEST_DIR/data.log - - # Start the Logger: 1 file with 100 lines, we use a big size limit (-s) to - # avoid rotation - python logger_file.py -l 100 -s 200000 -b 100 -d 0 -f $TEST_DIR/data.log - lines=`cat $TEST_DIR/data.log | wc -l` - ln -s data.log $TEST_DIR/a.log - echo "Logger done, written lines "$lines - - # Run Fluent Bit - $FLB_BIN -c conf/rotate_link.conf & - FLB_PID=$! - echo "Fluent Bit started, pid=$FLB_PID" - - # Wait 2 seconds and rotate file - sleep 2 - pre_lines=`cat $TEST_DIR/a.log | wc -l` - mv $TEST_DIR/a.log $TEST_DIR/a.log.1 - - lines=`cat $TEST_DIR/a | wc -l` - echo "file rotated, mid-check: processed lines $lines" - - # Append 200 more lines to the rotated link - python logger_file.py -l 200 -s 200000 -b 100 -d 0 -f $TEST_DIR/a.log.1 - - # Count number of processed lines - sleep 3 - write_lines=300 - read_lines=`cat $TEST_DIR/a | wc -l` - - echo "> write lines: $write_lines" - echo "> read lines : $read_lines" - - # Check we processed same number of records - ${_ASSERT_EQUALS_} $write_lines $read_lines - - # Check that database file have the right offset and mark the file as rotated - sqlite_check $TEST_DIR a.db $FLB_PID - - # Stop Fluent Bit (SIGTERM) - kill -15 $FLB_PID -} - -# 5. Truncate Link -# -# Test a link that gets a truncation and Fluent Bit properly use the new offset -# -# Configuration file used: conf/truncate_link.conf - -test_truncate_link() { - # Helper function to check monitored files - sqlite_check() - { - # Incoming parameters: - # $1: temporal directory to store data - # $2: database file name - # $3: Fluent Bit PID - # - # This function store the remaining monitored files listed in the database, - # we send the output to an .inodes for troubleshooting purposes if required - - # Get the last size of the 'a.log' file and check we have the same value - # in the database - offset=`wc -c < $TEST_DIR/a.log` - - sqlite3 $1/$2 -batch \ - ".headers off" "SELECT inode FROM in_tail_files WHERE offset=$offset" > \ - $1/$2.offset - - rows=`cat $1/$2.offset | wc -l | tr -d -C '[0-9]'` - if [ $rows != "1" ]; then - echo "> invalid database content:" - cat $1/$2.offset - echo "> open files" - ls -l /proc/$3/fd/ | grep \\.log - else - echo "> database file $1/$2 is OK" - fi - ${_ASSERT_EQUALS_} "1" $rows - } - - # Prepare test directory - export TEST_DIR=tmp_test - rm -rf $TEST_DIR - mkdir $TEST_DIR - - # Create empty files so Fluent Bit will enqueue them on start - touch $TEST_DIR/data.log - - # Start the Logger: 1 file with 100 lines, we use a big size limit (-s) to - # avoid rotation - python logger_file.py -l 100 -s 200000 -b 100 -d 0 -f $TEST_DIR/data.log - lines=`cat $TEST_DIR/data.log | wc -l` - ln -s data.log $TEST_DIR/a.log - echo "Logger done, written lines "$lines - - # Run Fluent Bit - $FLB_BIN -c conf/truncate_link.conf & - FLB_PID=$! - echo "Fluent Bit started, pid=$FLB_PID" - - # Wait 1 second before truncation - sleep 1 - pre_lines=`cat $TEST_DIR/a.log | wc -l` - truncate -s 0 $TEST_DIR/a.log - - sleep 2 - lines=`cat $TEST_DIR/a | wc -l` - echo "file truncated, mid-check: processed lines $lines" - - # Append 200 more lines - python logger_file.py -l 200 -s 200000 -b 100 -d 0 -f $TEST_DIR/a.log - - sleep 4 - - # Count number of processed lines - write_lines=300 - read_lines=`cat $TEST_DIR/a | wc -l` - - echo "> write lines: $write_lines" - echo "> read lines : $read_lines" - - # Check we processed same number of records - ${_ASSERT_EQUALS_} $write_lines $read_lines - - # Stop Fluent Bit (SIGTERM) - kill -15 $FLB_PID -} - -# 6. Database Resume -# -# Tests that if Fluent Bit stops, it can resume reading files from the last position -# -# Configuration file used: conf/database_resume.conf - -test_database_resume() { - # Helper function to check monitored files - sqlite_check() - { - # Incoming parameters: - # $1: temporal directory to store data - # $2: database file name - # $3: Fluent Bit PID - # - # This function store the remaining monitored files listed in the database, - # we send the output to an .inodes for troubleshooting purposes if required - - # Get the last size of the 'a.log' file and check we have the same value - # in the database - offset=`stat -c %s $TEST_DIR/a.log` - - sqlite3 $1/$2 -batch \ - ".headers off" "SELECT inode FROM in_tail_files WHERE offset=$offset" > \ - $1/$2.offset - - rows=`cat $1/$2.offset | wc -l` - if [ $rows != "1" ]; then - echo "> invalid database content:" - cat $1/$2.offset - echo "> open files" - ls -l /proc/$3/fd/ | grep \\.log - else - echo "> database file $1/$2 is OK" - fi - ${_ASSERT_EQUALS_} "1" $rows - } - - # Prepare test directory - export TEST_DIR=tmp_test - rm -rf $TEST_DIR - mkdir $TEST_DIR - - # Start the Logger: 5 files with 10000 lines each, we use a big size limit (-s) to - # avoid rotation - python logger_file.py -l 10000 -s 200000 -b 100 -d 0 \ - -f $TEST_DIR/a.log \ - -f $TEST_DIR/b.log \ - -f $TEST_DIR/c.log \ - -f $TEST_DIR/d.log \ - -f $TEST_DIR/e.log - - lines=`cat $TEST_DIR/[abcde].log | wc -l` - echo "Logger done, written lines "$lines - - # Run Fluent Bit - $FLB_BIN -c conf/database_resume.conf & - FLB_PID=$! - echo "Fluent Bit started, pid=$FLB_PID" - - # Wait 1 second before stop Fluent Bit - sleep 1 - - # Stop Fluent Bit (SIGTERM) - kill -15 $FLB_PID - - sqlite3 $TEST_DIR/logs.db "SELECT * FROM in_tail_files" - exit 0 - - sleep 2 - lines=`cat $TEST_DIR/a | wc -l` - echo "file truncated, mid-check: processed lines $lines" - - # Append 200 more lines - python logger_file.py -l 200 -s 200000 -b 100 -d 0 -f $TEST_DIR/a.log - - sleep 4 - - # Count number of processed lines - write_lines=300 - read_lines=`cat $TEST_DIR/a | wc -l` - - echo "> write lines: $write_lines" - echo "> read lines : $read_lines" - - # Check we processed same number of records - ${_ASSERT_EQUALS_} $write_lines $read_lines - - # Stop Fluent Bit (SIGTERM) - kill -15 $FLB_PID -} - - -# Launch the tests -. $FLB_RUN_TEST diff --git a/fluent-bit/tests/runtime_shell/in_tail_expect.sh b/fluent-bit/tests/runtime_shell/in_tail_expect.sh deleted file mode 100755 index c847d65f4..000000000 --- a/fluent-bit/tests/runtime_shell/in_tail_expect.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/sh - -test_in_tail_filter_expect() { - rm -rf /tmp/flb_* - - # Monitor this file - echo "{\"key\": \"val\"}" > /tmp/flb_tail_expect_1.log - - # Excluded file - echo "{\"nokey\": \"\"}" > /tmp/flb_tail_expect_2.log - - $FLB_BIN -c $FLB_RUNTIME_SHELL_CONF/in_tail_expect.conf -} - -# The following command launch the unit test -. $FLB_RUNTIME_SHELL_PATH/runtime_shell.env diff --git a/fluent-bit/tests/runtime_shell/runtime_shell.env.in b/fluent-bit/tests/runtime_shell/runtime_shell.env.in deleted file mode 100644 index 6a40c513f..000000000 --- a/fluent-bit/tests/runtime_shell/runtime_shell.env.in +++ /dev/null @@ -1,5 +0,0 @@ -flb_runtime_shell_test_run() { - . $FLB_RUNTIME_SHELL_PATH/../lib/shunit2/shunit2 -} - -flb_runtime_shell_test_run diff --git a/fluent-bit/tests/runtime_shell/tls/certificate.pem b/fluent-bit/tests/runtime_shell/tls/certificate.pem deleted file mode 100644 index b79719873..000000000 --- a/fluent-bit/tests/runtime_shell/tls/certificate.pem +++ /dev/null @@ -1,22 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIDlzCCAn+gAwIBAgIUCl24aoQh3HuyObboivsozKKOb3cwDQYJKoZIhvcNAQEL -BQAwWzELMAkGA1UEBhMCQVUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoM -GEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDEUMBIGA1UEAwwLbGVvLnZjYXAubWUw -HhcNMjIwODE3MTM0OTAyWhcNMjMwODE3MTM0OTAyWjBbMQswCQYDVQQGEwJBVTET -MBEGA1UECAwKU29tZS1TdGF0ZTEhMB8GA1UECgwYSW50ZXJuZXQgV2lkZ2l0cyBQ -dHkgTHRkMRQwEgYDVQQDDAtsZW8udmNhcC5tZTCCASIwDQYJKoZIhvcNAQEBBQAD -ggEPADCCAQoCggEBAJ2cbLtGhSeGPAXKitw+PN7U8CkaSGL7BssPFSuqtuVkJ1AH -puluHq62FHm4UgTyJB/Iundf1kBxiOehGFr9E1fGp6kUety6iBZK3mmmX0NdXY2A -ObqfvxTgPNVPEj+5Qp0CJW6fEBRhHrFgpf+OdKzoKnjSCbl1EyJ1wFZeSNDTx98I -h951xEOCobcXj1qXHDd0snRW5QNKTdSnj+3amqrNqV5DrkCyyb5Rq/VQX9LIyGV8 -tOyq50mUKeK+vOu/oNwfkZqgZ5wlY/AUyPMNvd8UKitnWeQMiMmr3ImQRZLkFnoE -2mi2Xr6ZQhmsweWbTmmUYxr+ej7L5WvBVQEkbd0CAwEAAaNTMFEwHQYDVR0OBBYE -FGliHqbvEsRN36C2w1cCq5M/gGCRMB8GA1UdIwQYMBaAFGliHqbvEsRN36C2w1cC -q5M/gGCRMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBACWCNohz -YbnnDzGXMhGlfV5PywH6+gguqTrtzJlfJB/FGHDMybBvEp9gda5ssPZfPdv6aAvo -hz2XC+uwJqYn76ZcX+LOixsqhoXJ367pT0ggMSUB/UNvGkzMFWuGerxcB5Q7xvsh -0Ui37Ojok7Ozj9Ofk/zroJB1WG3kPDtwoo+GwNmJaDtXhOutRPxCqp5Dw+dC0N0c -TlPCBRB17hx49Zjw+h9sv1VmOUtQcplNZgRL9lguv7eXB6y0sSdpDyDLlTBbRMGl -uVnQ20uKnqqALRfpXDWFL8g01xPNEIfV3KctpJfu1inLWtPuklerl0yeyUh9M1f0 -zCAi0URsGEHMFA0= ------END CERTIFICATE----- diff --git a/fluent-bit/tests/runtime_shell/tls/private_key.pem b/fluent-bit/tests/runtime_shell/tls/private_key.pem deleted file mode 100644 index b6a705395..000000000 --- a/fluent-bit/tests/runtime_shell/tls/private_key.pem +++ /dev/null @@ -1,28 +0,0 @@ ------BEGIN PRIVATE KEY----- -MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCdnGy7RoUnhjwF -yorcPjze1PApGkhi+wbLDxUrqrblZCdQB6bpbh6uthR5uFIE8iQfyLp3X9ZAcYjn -oRha/RNXxqepFHrcuogWSt5ppl9DXV2NgDm6n78U4DzVTxI/uUKdAiVunxAUYR6x -YKX/jnSs6Cp40gm5dRMidcBWXkjQ08ffCIfedcRDgqG3F49alxw3dLJ0VuUDSk3U -p4/t2pqqzaleQ65Assm+Uav1UF/SyMhlfLTsqudJlCnivrzrv6DcH5GaoGecJWPw -FMjzDb3fFCorZ1nkDIjJq9yJkEWS5BZ6BNpotl6+mUIZrMHlm05plGMa/no+y+Vr -wVUBJG3dAgMBAAECggEANXQ/AEkLkfsZ0lD+RXIqTNzlUttiH4fJpwbHhFbSzvvn -xWHC/zpk15ZTXXDhCGJjVBBNBX2QeazH5N8jFoDslYF/jX2vqbrturnLswNFHeDF -gN8zNRNGyDrBBwtZQhl/+SYoMdtqpa7GrRv9UK4s7hOTjASYXbjSM4bCI8i4Y3JX -5iKYnejTJ4GNXm1S5g4H8QpgXxrxYU7HY1UlMBGni1BSDPb5SxANnFvgw0oypaDG -4a0KpnUmziBtZo8wIQ7+UeStS4xikIeaT2SQM/ilplPNx0HgPHbTMeTBHQylG1JU -ztEE04Y1F9UF7cqZnxqQLlKmIqsOepLeDOkRXjDtNQKBgQDQtWYbewCmGuo1+AR7 -wStOXpfhqdu6NxAwX+S06ZdAV7f4duMFb//6uJ/c997qwRlNvRvpPlJBdcT3qziS -zjvR8uk2YN5N9D+ZN0Yi11vltUYVPeGpoAk4L7+Ruj5RTVqk5iZEZ1RbAv5cRIm0 -D24H4W0Cj/aldDED2pwTb9ukEwKBgQDBUv/s7gUNM3LCkNLc1EDg2adSzoUopxHU -jIQCitXt5iFpY2Lv2tpaIz4ux4HQGwhmao2MiUYOMit0++/pFaKur66tgRQRsr7F -uS1fpjqtS51/5uFuc1xcWjVquidZXH9u9iChrYvim6SBkxzFsbu4kXNOj4I562cm -E2VWH5GETwKBgQCo5ePX4VbJFYbsXeXi8JRHO63V5Uv4Co+DVlcTQOYyH8q1vCBE -Sjrxf29/tugjOllr29o2i0StzMy1UU7bHyKx6M5qP0In+71sFJshnv6ziltI3Wc9 -ilFrstho6jt8OAle4RGe0bAmZunJaX22xbXZkshRBognpTv1Tnh4ElHBGQKBgHIW -nVohjXGg7xTLiuUvjaokSI6hugunrOoWksE9VcqziPw83uJV8Y5IRiYtLvq1OVvX -ffl1+ZXfHa5ID+kqD3uvyhIynrljFxpwkcpkuzQR77zPcDJSeis2QVfey+H8qGe/ -cLp5RJhS6d5eBxjULshZbgbqwhuURKc/wwn0T1gZAoGAXu2wqwBE8ikhpaDVFYZR -EZe0G1iq03CtabFL0oPiddytV9jJy4Mg457tkCj58uCVkxw/8qjXEnm5OoJAzSlJ -Goc4wh1N1rngYRbSGG+LxD0DeI/Hj3T5tWezMBLCE7J+hieUuVjOB9ELKip7aTj3 -KLNVudOATWYGXyvzux6ciu0= ------END PRIVATE KEY----- |