diff options
Diffstat (limited to 'src/fluent-bit/tests/runtime')
323 files changed, 42916 insertions, 0 deletions
diff --git a/src/fluent-bit/tests/runtime/CMakeLists.txt b/src/fluent-bit/tests/runtime/CMakeLists.txt new file mode 100644 index 000000000..d6d3e7101 --- /dev/null +++ b/src/fluent-bit/tests/runtime/CMakeLists.txt @@ -0,0 +1,173 @@ +find_package(Threads REQUIRED) + +include_directories(acutest/) + +define_property(GLOBAL PROPERTY CHECK_PROGRAMS + BRIEF_DOCS "Runtime Test Programs" + FULL_DOCS "Runtime Test Programs") +set_property(GLOBAL PROPERTY CHECK_PROGRAMS "") + +# Macro to set definitions +macro(FLB_RT_TEST BUILT src) + if (${BUILT}) + list(APPEND CHECK_PROGRAMS + ${src} + ) + endif() +endmacro() + +# Macro to set definitions +macro(FLB_RT_CORE_TEST BUILT src) + list(APPEND CHECK_PROGRAMS + ${src} + ) +endmacro() + +# Core +FLB_RT_CORE_TEST(FLB_COROUTINE_TIMEOUT "core-timeout.c") + +FLB_RT_TEST(FLB_CHUNK_TRACE "core_chunk_trace.c") + +# Input Plugins +FLB_RT_TEST(FLB_IN_EVENT_TEST "in_event_test.c") + +if(FLB_OUT_LIB) + # These plugins works only on Linux + if(${CMAKE_SYSTEM_NAME} MATCHES "Linux") + FLB_RT_TEST(FLB_IN_CPU "in_cpu.c") + FLB_RT_TEST(FLB_IN_DISK "in_disk.c") + FLB_RT_TEST(FLB_IN_MEM "in_mem.c") + FLB_RT_TEST(FLB_IN_PROC "in_proc.c") + FLB_RT_TEST(FLB_IN_NETIF "in_netif.c") + FLB_RT_TEST(FLB_IN_PODMAN_METRICS "in_podman_metrics.c") + endif() + FLB_RT_TEST(FLB_IN_HEAD "in_head.c") + FLB_RT_TEST(FLB_IN_DUMMY "in_dummy.c") + FLB_RT_TEST(FLB_IN_HTTP "in_http.c") + FLB_RT_TEST(FLB_IN_ELASTICSEARCH "in_elasticsearch.c") + FLB_RT_TEST(FLB_IN_MQTT "in_mqtt.c") + FLB_RT_TEST(FLB_IN_OPENTELEMETRY "in_opentelemetry.c") + FLB_RT_TEST(FLB_IN_RANDOM "in_random.c") + FLB_RT_TEST(FLB_IN_STATSD "in_statsd.c") + FLB_RT_TEST(FLB_IN_SPLUNK "in_splunk.c") + FLB_RT_TEST(FLB_IN_SYSLOG "in_syslog.c") + FLB_RT_TEST(FLB_IN_TAIL "in_tail.c") + FLB_RT_TEST(FLB_IN_UDP "in_udp.c") + FLB_RT_TEST(FLB_IN_TCP "in_tcp.c") + FLB_RT_TEST(FLB_IN_FORWARD "in_forward.c") + FLB_RT_TEST(FLB_IN_FLUENTBIT_METRICS "in_fluentbit_metrics.c") +endif() + +# Filter Plugins +if(FLB_IN_LIB AND FLB_OUT_LIB) + FLB_RT_TEST(FLB_FILTER_AWS "filter_aws.c") + FLB_RT_TEST(FLB_FILTER_CHECKLIST "filter_checklist.c") + FLB_RT_TEST(FLB_FILTER_EXPECT "filter_expect.c") + FLB_RT_TEST(FLB_FILTER_STDOUT "filter_stdout.c") + FLB_RT_TEST(FLB_FILTER_GREP "filter_grep.c") + FLB_RT_TEST(FLB_FILTER_THROTTLE "filter_throttle.c") + FLB_RT_TEST(FLB_FILTER_THROTTLE_SIZE "filter_throttle_size.c") + FLB_RT_TEST(FLB_FILTER_NEST "filter_nest.c") + FLB_RT_TEST(FLB_FILTER_REWRITE_TAG "filter_rewrite_tag.c") + FLB_RT_TEST(FLB_FILTER_KUBERNETES "filter_kubernetes.c") + FLB_RT_TEST(FLB_FILTER_PARSER "filter_parser.c") + FLB_RT_TEST(FLB_FILTER_MODIFY "filter_modify.c") + FLB_RT_TEST(FLB_FILTER_LUA "filter_lua.c") + FLB_RT_TEST(FLB_FILTER_TYPE_CONVERTER "filter_type_converter.c") + FLB_RT_TEST(FLB_FILTER_RECORD_MODIFIER "filter_record_modifier.c") + FLB_RT_TEST(FLB_FILTER_MULTILINE "filter_multiline.c") + if (FLB_FILTER_WASM) + FLB_RT_TEST(FLB_FILTER_WASM "filter_wasm.c") + endif () + FLB_RT_TEST(FLB_FILTER_ECS "filter_ecs.c") + FLB_RT_TEST(FLB_FILTER_LOG_TO_METRICS "filter_log_to_metrics.c") +endif() + + +# Output Plugins +if(FLB_IN_LIB) + FLB_RT_TEST(FLB_OUT_LIB "core_engine.c") + FLB_RT_TEST(FLB_OUT_LIB "core_log.c") + FLB_RT_TEST(FLB_OUT_LIB "config_map_opts.c") + FLB_RT_TEST(FLB_OUT_COUNTER "out_counter.c") + FLB_RT_TEST(FLB_OUT_DATADOG "out_datadog.c") + FLB_RT_TEST(FLB_OUT_SKYWALKING "out_skywalking.c") + FLB_RT_TEST(FLB_OUT_ES "out_elasticsearch.c") + FLB_RT_TEST(FLB_OUT_OPENSEARCH "out_opensearch.c") + FLB_RT_TEST(FLB_OUT_EXIT "out_exit.c") + FLB_RT_TEST(FLB_OUT_FLOWCOUNTER "out_flowcounter.c") + FLB_RT_TEST(FLB_OUT_FORWARD "out_forward.c") + FLB_RT_TEST(FLB_OUT_HTTP "out_http.c") + FLB_RT_TEST(FLB_OUT_LIB "out_lib.c") + FLB_RT_TEST(FLB_OUT_LOKI "out_loki.c") + FLB_RT_TEST(FLB_OUT_NULL "out_null.c") + FLB_RT_TEST(FLB_OUT_PLOT "out_plot.c") + FLB_RT_TEST(FLB_OUT_RETRY "out_retry.c") + FLB_RT_TEST(FLB_OUT_SPLUNK "out_splunk.c") + FLB_RT_TEST(FLB_OUT_STDOUT "out_stdout.c") + FLB_RT_TEST(FLB_OUT_SYSLOG "out_syslog.c") + FLB_RT_TEST(FLB_OUT_TCP "out_tcp.c") + + if (FLB_RECORD_ACCESSOR) + FLB_RT_TEST(FLB_OUT_STACKDRIVER "out_stackdriver.c") + endif() + + FLB_RT_TEST(FLB_OUT_CLOUDWATCH_LOGS "out_cloudwatch.c") + FLB_RT_TEST(FLB_OUT_KINESIS_FIREHOSE "out_firehose.c") + FLB_RT_TEST(FLB_OUT_KINESIS_STREAMS "out_kinesis.c") + # These plugins work only on Linux + if(NOT FLB_SYSTEM_WINDOWS) + FLB_RT_TEST(FLB_OUT_FILE "out_file.c") + endif() + FLB_RT_TEST(FLB_OUT_S3 "out_s3.c") + FLB_RT_TEST(FLB_OUT_TD "out_td.c") + +endif() + +if (FLB_CUSTOM_CALYPTIA) + FLB_RT_TEST(FLB_CUSTOM_CALYPTIA "custom_calyptia_test.c") +endif() + + +# HTTP Client Debug (requires -DFLB_HTTP_CLIENT_DEBUG=On) +if(FLB_HTTP_CLIENT_DEBUG) + FLB_RT_TEST(FLB_OUT_TD "http_callbacks.c") +endif() + +set(SYSTEMD_LIB, "") +if(FLB_HAVE_SYSTEMD) + set(SYSTEMD_LIB, "systemd") +endif() + +set(FLB_TESTS_DATA_PATH ${CMAKE_CURRENT_SOURCE_DIR}) +configure_file( + "${CMAKE_CURRENT_SOURCE_DIR}/flb_tests_runtime.h.in" + "${CMAKE_CURRENT_SOURCE_DIR}/flb_tests_runtime.h" + ) + +foreach(source_file ${CHECK_PROGRAMS}) + get_filename_component(o_source_file_we ${source_file} NAME_WE) + set(source_file_we flb-rt-${o_source_file_we}) + if(FLB_WITHOUT_${source_file_we}) + message("Skipping test ${source_file_we}") + else() + add_executable( + ${source_file_we} + ${source_file} + ) + add_sanitizers(${source_file_we}) + target_link_libraries(${source_file_we} + fluent-bit-static + ${CMAKE_THREAD_LIBS_INIT} + ${SYSTEMD_LIB} + ) + if(FLB_AVRO_ENCODER) + target_link_libraries(${source_file_we} avro-static jansson) + endif() + add_test(NAME ${source_file_we} + COMMAND ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/${source_file_we} + WORKING_DIRECTORY ${CMAKE_HOME_DIRECTORY}/build) + set_tests_properties(${source_file_we} PROPERTIES LABELS "runtime") + set_property(TARGET ${source_file_we} APPEND_STRING PROPERTY COMPILE_FLAGS "-D${o_source_file_we}") + endif() +endforeach() diff --git a/src/fluent-bit/tests/runtime/config_map_opts.c b/src/fluent-bit/tests/runtime/config_map_opts.c new file mode 100644 index 000000000..57b396bb4 --- /dev/null +++ b/src/fluent-bit/tests/runtime/config_map_opts.c @@ -0,0 +1,35 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ +#include <fluent-bit.h> +#include "flb_tests_runtime.h" + +/* Test functions */ +void flb_test_config_map_opts(void); + +/* Test list */ +TEST_LIST = { + {"config_map_opts", flb_test_config_map_opts }, + {NULL, NULL} +}; + +void flb_test_config_map_opts(void) +{ + flb_ctx_t *ctx = NULL; + int in_ffd, r; + + flb_init_env(); + + ctx = flb_create(); + in_ffd = flb_input(ctx, (char *) "tail", NULL); + r = flb_input_property_check(ctx, in_ffd, "invalid_option", "invalid value"); + TEST_CHECK(r != 0); + + in_ffd = flb_filter(ctx, (char *) "kubernetes", NULL); + r = flb_filter_property_check(ctx, in_ffd, "invalid_option", "invalid value"); + TEST_CHECK(r != 0); + + in_ffd = flb_output(ctx, (char *) "stdout", NULL); + r = flb_output_property_check(ctx, in_ffd, "invalid_option", "invalid value"); + TEST_CHECK(r != 0); + + flb_destroy(ctx); +} diff --git a/src/fluent-bit/tests/runtime/core-timeout.c b/src/fluent-bit/tests/runtime/core-timeout.c new file mode 100644 index 000000000..10146d3cd --- /dev/null +++ b/src/fluent-bit/tests/runtime/core-timeout.c @@ -0,0 +1,94 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <fluent-bit.h> +#include "flb_tests_runtime.h" + +#define ELAPSED_TIME_LIMIT 9 + +void flb_test_timeout_coroutine_recovery() +{ + int output_instance_id; + int input_instance_id; + time_t elapsed_time; + time_t start_time; + time_t stop_time; + flb_ctx_t *ctx; + int64_t ret; + + ctx = flb_create(); + + TEST_CHECK(flb_service_set(ctx, "Flush", "0.5", + "Grace", "10", + NULL) == 0); + + ret = flb_service_set(ctx, + "Log_Level", "info", + NULL); + + TEST_CHECK_(ret == 0, "setting service options"); + + input_instance_id = flb_input(ctx, (char *) "dummy", NULL); + TEST_CHECK(input_instance_id >= 0); + + TEST_CHECK(flb_input_set(ctx, input_instance_id, + "samples", "1", + "rate" , "1", + NULL) == 0); + + output_instance_id = flb_output(ctx, (char *) "tcp", NULL); + TEST_CHECK(output_instance_id >= 0); + TEST_CHECK(flb_output_set(ctx, output_instance_id, + "match", "*", + "retry_limit", "no_retries", + "host", "35.243.247.233", + "port", "54321", + "net.keepalive", "off", + "net.connect_timeout", "5s", + NULL) == 0); + + /* Start test */ + /* Start the engine */ + ret = flb_start(ctx); + TEST_CHECK_(ret == 0, "starting engine"); + + sleep(10); + + start_time = time(NULL); + + ret = flb_stop(ctx); + TEST_CHECK_(ret == 0, "stopping engine"); + + stop_time = time(NULL); + + if (ctx) { + flb_destroy(ctx); + } + + elapsed_time = stop_time - start_time; + + TEST_CHECK_(ELAPSED_TIME_LIMIT >= elapsed_time, + "for hung coroutines"); +} + +/* Test list */ +TEST_LIST = { + {"timeout_coroutine_recovery", flb_test_timeout_coroutine_recovery}, + {NULL, NULL} +}; diff --git a/src/fluent-bit/tests/runtime/core_chunk_trace.c b/src/fluent-bit/tests/runtime/core_chunk_trace.c new file mode 100644 index 000000000..6fb00a9b4 --- /dev/null +++ b/src/fluent-bit/tests/runtime/core_chunk_trace.c @@ -0,0 +1,153 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2016 Treasure Data Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <fluent-bit.h> +#include <fluent-bit/flb_time.h> +#include <fluent-bit/flb_chunk_trace.h> +#include <fluent-bit/flb_router.h> +#include <pthread.h> +#include <stdlib.h> +#include <unistd.h> +#include "flb_tests_runtime.h" + + +struct callback_record { + void *data; + size_t size; +}; + +struct callback_records { + int num_records; + struct callback_record *records; +}; + +int callback_add_record(void* data, size_t size, void* cb_data) +{ + struct callback_records *ctx = (struct callback_records *)cb_data; + + if (size > 0) { + flb_info("[test] flush record"); + /* We should check ctx->num_records has a valid value. */ + if (ctx->num_records < 0) { + return -1; + } + if (ctx->records == NULL) { + ctx->records = (struct callback_record *) + flb_calloc(1, sizeof(struct callback_record)); + } else { + ctx->records = (struct callback_record *) + flb_realloc(ctx->records, + (ctx->num_records+1)*sizeof(struct callback_record)); + } + if (ctx->records == NULL) { + return -1; + } + ctx->records[ctx->num_records].size = size; + ctx->records[ctx->num_records].data = data; + ctx->num_records++; + } + return 0; +} + +void do_test_records_trace(void (*records_cb)(struct callback_records *)) +{ + flb_ctx_t *ctx = NULL; + struct flb_input_instance *input; + struct flb_output_instance *output; + int i; + struct flb_lib_out_cb cb; + struct callback_records *records; + + records = flb_calloc(1, sizeof(struct callback_records)); + records->num_records = 0; + records->records = NULL; + cb.cb = callback_add_record; + cb.data = (void *)records; + + ctx = flb_create(); + + input = flb_input_new(ctx->config, "dummy", NULL, FLB_TRUE); + TEST_CHECK(input != NULL); + + output = flb_output_new(ctx->config, (char *) "stdout", NULL, FLB_TRUE); + TEST_CHECK(output != NULL); + + TEST_CHECK(flb_service_set(ctx, "Flush", "0.5", + "Grace", "1", + "Enable_Chunk_Trace", "On", + NULL) == 0); + + + flb_router_connect_direct(input, output); + + TEST_CHECK(flb_chunk_trace_context_new(input, "lib", "test.", (void *)&cb, NULL) != NULL); + + /* Start test */ + TEST_CHECK(flb_start(ctx) == 0); + + /* 4 sec passed. It must have flushed */ + sleep(5); + + records_cb(records); + + flb_stop(ctx); + + for (i = 0; i < records->num_records; i++) { + flb_lib_free(records->records[i].data); + } + flb_free(records->records); + flb_free(records); + + flb_destroy(ctx); +} + +void flb_test_dummy_records_trace_simple(struct callback_records *records) +{ + int i; + msgpack_unpacked result; + msgpack_object *obj; + size_t off = 0; + struct flb_time ftm; + + TEST_CHECK(records->num_records > 0); + for (i = 0; i < records->num_records; i++) { + msgpack_unpacked_init(&result); + + while (msgpack_unpack_next(&result, records->records[i].data, + records->records[i].size, &off) == MSGPACK_UNPACK_SUCCESS) { + flb_time_pop_from_msgpack(&ftm, &result, &obj); + //TEST_CHECK(ftm.tm.tv_sec == 1234); + //TEST_CHECK(ftm.tm.tv_nsec == 1234); + } + msgpack_unpacked_destroy(&result); + } +} + +void flb_test_trace() +{ + do_test_records_trace(flb_test_dummy_records_trace_simple); +} + +/* Test list */ +TEST_LIST = { +#ifdef FLB_HAVE_CHUNK_TRACE + {"trace", flb_test_trace}, +#endif + {NULL, NULL} +}; diff --git a/src/fluent-bit/tests/runtime/core_engine.c b/src/fluent-bit/tests/runtime/core_engine.c new file mode 100644 index 000000000..a7f136a32 --- /dev/null +++ b/src/fluent-bit/tests/runtime/core_engine.c @@ -0,0 +1,167 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2019-2020 The Fluent Bit Authors + * Copyright (C) 2015-2018 Treasure Data Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <fluent-bit.h> +#include <stdlib.h> +#include <unistd.h> +#include <string.h> + +#include "flb_tests_runtime.h" + +/* Test data*/ + +/* Test functions*/ +void flb_test_engine_wildcard(void); + +/* Test list */ +TEST_LIST = { + {"wildcard", flb_test_engine_wildcard }, + {NULL, NULL} +}; + +#define MAX_WAIT_TIME 1500 +int64_t result_time; +static inline int64_t set_result(int64_t v) +{ + int64_t old = __sync_lock_test_and_set(&result_time, v); + return old; +} + +static inline int64_t get_result(void) +{ + int64_t old = __sync_fetch_and_add(&result_time, 0); + + return old; +} + +static inline int64_t time_in_ms() +{ + int ms; + struct timespec s; + TEST_CHECK(clock_gettime(CLOCK_MONOTONIC, &s) == 0); + ms = s.tv_nsec / 1.0e6; + if (ms >= 1000) { + ms = 0; + } + return 1000 * s.tv_sec + ms; +} + +int callback_test(void* data, size_t size, void* cb_data) +{ + if (size > 0) { + flb_lib_free(data); + set_result(time_in_ms()); /* success */ + } + return 0; +} + +int check_routing(const char* tag, + const char* match, + const char* match_regex, + bool expect) +{ + int in_ffd; + int out_ffd; + int64_t ret; + int64_t start; + flb_ctx_t *ctx = NULL; + char *str = (char*)"[1, {\"key\":\"value\"}]"; + + struct flb_lib_out_cb cb; + cb.cb = callback_test; + cb.data = NULL; + + /* initialize */ + ctx = flb_create(); + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx, in_ffd, "tag", tag, NULL); + + out_ffd = flb_output(ctx, (char *) "lib", &cb); + TEST_CHECK(out_ffd >= 0); + if (match) { + flb_output_set(ctx, out_ffd, "match", match, NULL); + } + if (match_regex) { + flb_output_set(ctx, out_ffd, "match_regex", match_regex, NULL); + } + + flb_service_set(ctx, "Flush", "1", "Grace", "1", "Daemon", "false", NULL); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* start test */ + flb_lib_push(ctx, in_ffd, str, strlen(str)); + set_result(0); + start = time_in_ms(); + while ( (ret = get_result()) == 0 && (time_in_ms() - start < MAX_WAIT_TIME)) + usleep(10); + + if (expect ? ret == 0 : ret > 0) { + flb_error("Mismatch: tag:%s, match:%s, match_regex:%s, expect:%s\n", + tag, + match ? match : "null", + match_regex ? match_regex : "null", + expect ? "true" : "false"); + } + TEST_CHECK(expect ? ret > 0 : ret == 0); + + /* finalize */ + flb_stop(ctx); + flb_destroy(ctx); + + return 0; +} + +void flb_test_engine_wildcard(void) +{ + struct test_wildcard_fmt { + const char* tag; + const char* match; + const char* match_regex; + bool expect; + }; + int i = 0; + + struct test_wildcard_fmt checklist[] = + { + {"cpu.rpi","cpu.rpi", NULL, true }, + {"cpu.rpi","cpu.ard", NULL, false }, + {"cpu.rpi","cpu.*", NULL, true }, + {"cpu.rpi","*", NULL, true }, + {"cpu.rpi","*.*", NULL, true }, + {"cpu.rpi","*.rpi", NULL, true }, + {"cpu.rpi","mem.*", NULL, false }, + {"cpu.rpi","*u.r*", NULL, true }, + {"cpu.rpi",NULL, "[a-z]*", true }, + {"cpu.rpi",NULL, "[A-Z]*", false }, + {NULL, NULL, NULL, 0} + }; + + while(checklist[i].tag != NULL){ + check_routing(checklist[i].tag, + checklist[i].match, + checklist[i].match_regex, + checklist[i].expect); + i++; + } +} diff --git a/src/fluent-bit/tests/runtime/core_log.c b/src/fluent-bit/tests/runtime/core_log.c new file mode 100644 index 000000000..8d466b705 --- /dev/null +++ b/src/fluent-bit/tests/runtime/core_log.c @@ -0,0 +1,308 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +#include <fluent-bit.h> +#include <fluent-bit/flb_log.h> + +#include "flb_tests_runtime.h" + + +static void check_result(char *level, int ret, int expect_truncated) +{ + if (expect_truncated == FLB_TRUE) { + if (!TEST_CHECK(ret > 0)) { + TEST_MSG("log is not truncated.level=%s ret=%d",level, ret); + /* + printf("ret=%d\n", ret); + */ + } + } + else { + if (!TEST_CHECK(ret == 0)) { + TEST_MSG("log is truncated.level=%s ret=%d",level, ret); + /* + printf("ret=%d\n", ret); + */ + } + } + +} +static void check_if_truncated(char* data, int expect_truncated) +{ + int ret; + ret = flb_error_is_truncated("%s", data); + check_result("error", ret, expect_truncated); + + ret = flb_warn_is_truncated("%s", data); + check_result("warn", ret, expect_truncated); + + ret = flb_info_is_truncated("%s", data); + check_result("info", ret, expect_truncated); + + ret = flb_debug_is_truncated("%s", data); + check_result("debug", ret, expect_truncated); +} + +static int cb_not_truncated_log(void *record, size_t size, void *data) +{ + check_if_truncated((char*)data, FLB_FALSE); + + flb_free(record); + return 0; +} + + +static int cb_truncated_log(void *record, size_t size, void *data) +{ + check_if_truncated((char*)data, FLB_TRUE); + + flb_free(record); + return 0; +} + +static int cb_resize(void *record, size_t size, void *data) +{ + int ret; + char *log = (char*)data; + + ret = flb_error_is_truncated("%s", log); + if (!TEST_CHECK(ret > 0)) { + TEST_MSG("log is not truncated.ret=%d", ret); + } + + ret = flb_error_is_truncated("%.*s", (int)(strlen(log) - ret), log); + if (!TEST_CHECK(ret == 0)) { + TEST_MSG("log is truncated.ret=%d", ret); + } + + flb_free(record); + return 0; +} + +static int cb_log_level(void *record, size_t size, void *data) +{ + int ret; + char *log = (char*)data; + + ret = flb_error_is_truncated("%s", log); + if (!TEST_CHECK(ret > 0)) { + TEST_MSG("log is not truncated.ret=%d", ret); + } + + /* log_level is error. The function will return 0 */ + ret = flb_info_is_truncated("%s", log); + if (!TEST_CHECK(ret == 0)) { + TEST_MSG("log is truncated.ret=%d", ret); + } + + flb_free(record); + return 0; +} + +void test_not_truncated_log() +{ + int ret; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + int i; + char *msg = "[1, {\"msg\":\"body\"}]"; + char log[128] = {0}; + struct flb_lib_out_cb cb_data; + + cb_data.cb = cb_not_truncated_log; + cb_data.data = &log[0]; + + for (i=0; i<sizeof(log)-1; i++) { + log[i] = 'a'; + } + + ctx = flb_create(); + TEST_CHECK(ctx != NULL); + + flb_service_set(ctx, + "Flush", "0.200000000", + "Grace", "1", + "Log_Level", "debug", + NULL); + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + + out_ffd = flb_output(ctx, (char *) "lib", (void*)&cb_data); + TEST_CHECK(out_ffd >= 0); + ret = flb_output_set(ctx, out_ffd, + "match", "*", + NULL); + TEST_CHECK(ret == 0); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + ret = flb_lib_push(ctx, in_ffd, msg, strlen(msg)); + TEST_CHECK(ret >= 0); + + sleep(1); + + flb_stop(ctx); + flb_destroy(ctx); + +} + +void test_truncated_log() +{ + int ret; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + int i; + char *msg = "[1, {\"msg\":\"body\"}]"; + char log[4096 * 5] = {0}; + struct flb_lib_out_cb cb_data; + + cb_data.cb = cb_truncated_log; + cb_data.data = &log[0]; + + for (i=0; i<sizeof(log)-1; i++) { + log[i] = 'a'; + } + + ctx = flb_create(); + TEST_CHECK(ctx != NULL); + + flb_service_set(ctx, + "Flush", "0.200000000", + "Grace", "1", + "Log_Level", "debug", + NULL); + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + + out_ffd = flb_output(ctx, (char *) "lib", (void*)&cb_data); + TEST_CHECK(out_ffd >= 0); + ret = flb_output_set(ctx, out_ffd, + "match", "*", + NULL); + TEST_CHECK(ret == 0); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + ret = flb_lib_push(ctx, in_ffd, msg, strlen(msg)); + TEST_CHECK(ret >= 0); + + sleep(1); + + flb_stop(ctx); + flb_destroy(ctx); + +} + +void test_resize() +{ + int ret; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + int i; + char *msg = "[1, {\"msg\":\"body\"}]"; + char log[4096 * 5] = {0}; + struct flb_lib_out_cb cb_data; + + cb_data.cb = cb_resize; + cb_data.data = &log[0]; + + for (i=0; i<sizeof(log)-1; i++) { + log[i] = 'a'; + } + + ctx = flb_create(); + TEST_CHECK(ctx != NULL); + + flb_service_set(ctx, + "Flush", "0.200000000", + "Grace", "1", + "Log_Level", "error", + NULL); + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + + out_ffd = flb_output(ctx, (char *) "lib", (void*)&cb_data); + TEST_CHECK(out_ffd >= 0); + ret = flb_output_set(ctx, out_ffd, + "match", "*", + NULL); + TEST_CHECK(ret == 0); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + ret = flb_lib_push(ctx, in_ffd, msg, strlen(msg)); + TEST_CHECK(ret >= 0); + + sleep(1); + + flb_stop(ctx); + flb_destroy(ctx); + +} + +void test_log_level() +{ + int ret; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + int i; + char *msg = "[1, {\"msg\":\"body\"}]"; + char log[4096 * 5] = {0}; + struct flb_lib_out_cb cb_data; + + cb_data.cb = cb_log_level; + cb_data.data = &log[0]; + + for (i=0; i<sizeof(log)-1; i++) { + log[i] = 'a'; + } + + ctx = flb_create(); + TEST_CHECK(ctx != NULL); + + flb_service_set(ctx, + "Flush", "0.200000000", + "Grace", "1", + "Log_Level", "error", + NULL); + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + + out_ffd = flb_output(ctx, (char *) "lib", (void*)&cb_data); + TEST_CHECK(out_ffd >= 0); + ret = flb_output_set(ctx, out_ffd, + "match", "*", + NULL); + TEST_CHECK(ret == 0); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + ret = flb_lib_push(ctx, in_ffd, msg, strlen(msg)); + TEST_CHECK(ret >= 0); + + sleep(1); + + flb_stop(ctx); + flb_destroy(ctx); + +} + +/* Test list */ +TEST_LIST = { + {"not_truncated_log", test_not_truncated_log }, + {"truncated_log", test_truncated_log }, + {"resize", test_resize }, + {"log_level", test_log_level }, + {NULL, NULL} +}; diff --git a/src/fluent-bit/tests/runtime/custom_calyptia_test.c b/src/fluent-bit/tests/runtime/custom_calyptia_test.c new file mode 100644 index 000000000..5ad30e7a0 --- /dev/null +++ b/src/fluent-bit/tests/runtime/custom_calyptia_test.c @@ -0,0 +1,59 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +#include <fluent-bit.h> +#include <fluent-bit/flb_custom.h> +#include "flb_tests_runtime.h" + +flb_sds_t custom_calyptia_pipeline_config_get(struct flb_config *ctx); + +void flb_custom_calyptia_pipeline_config_get_test() +{ + const char *cfg_str = "[INPUT]\n name dummy.0\n[INPUT]\n name fluentbit_metrics.1\n tag _calyptia_cloud\n scrape_on_start true\n scrape_interval 30\n\n\n[OUTPUT]\n name stdout.0\n match *\n retry_limit 1\n\n"; + flb_ctx_t *ctx; + int in_ffd_dummy; + int in_ffd_metrics; + int out_ffd; + struct flb_custom_instance *calyptia; + flb_sds_t cfg; + + ctx = flb_create(); + flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + + in_ffd_dummy = flb_input(ctx, (char *) "dummy", NULL); + TEST_CHECK(in_ffd_dummy >= 0); + + in_ffd_metrics = flb_input(ctx, (char *) "fluentbit_metrics", NULL); + TEST_CHECK(in_ffd_metrics >= 0); + flb_input_set(ctx, in_ffd_metrics, + "tag", "_calyptia_cloud", + "scrape_on_start", "true", + "scrape_interval", "30", + NULL); + + out_ffd = flb_output(ctx, (char *) "stdout", NULL); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, "match", "*", NULL); + + calyptia = flb_custom_new(ctx->config, (char *)"calyptia", NULL); + TEST_CHECK(calyptia != NULL); + flb_custom_set_property(calyptia, "api_key", "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"); + flb_custom_set_property(calyptia, "log_level", "debug"); + flb_custom_set_property(calyptia, "log_level", "7DDD2941-3ED6-4B8C-9F84-DD04C4A018A4"); + flb_custom_set_property(calyptia, "add_label", "pipeline_id 7DDD2941-3ED6-4B8C-9F84-DD04C4A018A4"); + flb_custom_set_property(calyptia, "calyptia_host", "cloud-api.calyptia.com"); + flb_custom_set_property(calyptia, "calyptia_port", "443"); + + cfg = custom_calyptia_pipeline_config_get(ctx->config); + TEST_CHECK(strcmp(cfg, cfg_str) == 0); + + // fix a thread local storage bug on macos + flb_output_prepare(); + flb_sds_destroy(cfg); + flb_destroy(ctx); +} + +/* Test list */ +TEST_LIST = { + {"get_config_test", flb_custom_calyptia_pipeline_config_get_test}, + {NULL, NULL} +}; diff --git a/src/fluent-bit/tests/runtime/data/common/json_invalid.h b/src/fluent-bit/tests/runtime/data/common/json_invalid.h new file mode 100755 index 000000000..87172d157 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/common/json_invalid.h @@ -0,0 +1,6 @@ +#define JSON_INVALID "[" \ + "1448403340," \ + "{" \ + "{{{{{{{{" "\"END_KEY\": \"JSON_END\"" \ + "}]" + diff --git a/src/fluent-bit/tests/runtime/data/common/json_long.h b/src/fluent-bit/tests/runtime/data/common/json_long.h new file mode 100755 index 000000000..e579b679e --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/common/json_long.h @@ -0,0 +1,1006 @@ +#define JSON_LONG "[" \ + "1448403340," \ + "{" \ + "\"key_0\": \"val_0\"," \ + "\"key_1\": \"val_1\"," \ + "\"key_2\": \"val_2\"," \ + "\"key_3\": \"val_3\"," \ + "\"key_4\": \"val_4\"," \ + "\"key_5\": \"val_5\"," \ + "\"key_6\": \"val_6\"," \ + "\"key_7\": \"val_7\"," \ + "\"key_8\": \"val_8\"," \ + "\"key_9\": \"val_9\"," \ + "\"key_10\": \"val_10\"," \ + "\"key_11\": \"val_11\"," \ + "\"key_12\": \"val_12\"," \ + "\"key_13\": \"val_13\"," \ + "\"key_14\": \"val_14\"," \ + "\"key_15\": \"val_15\"," \ + "\"key_16\": \"val_16\"," \ + "\"key_17\": \"val_17\"," \ + "\"key_18\": \"val_18\"," \ + "\"key_19\": \"val_19\"," \ + "\"key_20\": \"val_20\"," \ + "\"key_21\": \"val_21\"," \ + "\"key_22\": \"val_22\"," \ + "\"key_23\": \"val_23\"," \ + "\"key_24\": \"val_24\"," \ + "\"key_25\": \"val_25\"," \ + "\"key_26\": \"val_26\"," \ + "\"key_27\": \"val_27\"," \ + "\"key_28\": \"val_28\"," \ + "\"key_29\": \"val_29\"," \ + "\"key_30\": \"val_30\"," \ + "\"key_31\": \"val_31\"," \ + "\"key_32\": \"val_32\"," \ + "\"key_33\": \"val_33\"," \ + "\"key_34\": \"val_34\"," \ + "\"key_35\": \"val_35\"," \ + "\"key_36\": \"val_36\"," \ + "\"key_37\": \"val_37\"," \ + "\"key_38\": \"val_38\"," \ + "\"key_39\": \"val_39\"," \ + "\"key_40\": \"val_40\"," \ + "\"key_41\": \"val_41\"," \ + "\"key_42\": \"val_42\"," \ + "\"key_43\": \"val_43\"," \ + "\"key_44\": \"val_44\"," \ + "\"key_45\": \"val_45\"," \ + "\"key_46\": \"val_46\"," \ + "\"key_47\": \"val_47\"," \ + "\"key_48\": \"val_48\"," \ + "\"key_49\": \"val_49\"," \ + "\"key_50\": \"val_50\"," \ + "\"key_51\": \"val_51\"," \ + "\"key_52\": \"val_52\"," \ + "\"key_53\": \"val_53\"," \ + "\"key_54\": \"val_54\"," \ + "\"key_55\": \"val_55\"," \ + "\"key_56\": \"val_56\"," \ + "\"key_57\": \"val_57\"," \ + "\"key_58\": \"val_58\"," \ + "\"key_59\": \"val_59\"," \ + "\"key_60\": \"val_60\"," \ + "\"key_61\": \"val_61\"," \ + "\"key_62\": \"val_62\"," \ + "\"key_63\": \"val_63\"," \ + "\"key_64\": \"val_64\"," \ + "\"key_65\": \"val_65\"," \ + "\"key_66\": \"val_66\"," \ + "\"key_67\": \"val_67\"," \ + "\"key_68\": \"val_68\"," \ + "\"key_69\": \"val_69\"," \ + "\"key_70\": \"val_70\"," \ + "\"key_71\": \"val_71\"," \ + "\"key_72\": \"val_72\"," \ + "\"key_73\": \"val_73\"," \ + "\"key_74\": \"val_74\"," \ + "\"key_75\": \"val_75\"," \ + "\"key_76\": \"val_76\"," \ + "\"key_77\": \"val_77\"," \ + "\"key_78\": \"val_78\"," \ + "\"key_79\": \"val_79\"," \ + "\"key_80\": \"val_80\"," \ + "\"key_81\": \"val_81\"," \ + "\"key_82\": \"val_82\"," \ + "\"key_83\": \"val_83\"," \ + "\"key_84\": \"val_84\"," \ + "\"key_85\": \"val_85\"," \ + "\"key_86\": \"val_86\"," \ + "\"key_87\": \"val_87\"," \ + "\"key_88\": \"val_88\"," \ + "\"key_89\": \"val_89\"," \ + "\"key_90\": \"val_90\"," \ + "\"key_91\": \"val_91\"," \ + "\"key_92\": \"val_92\"," \ + "\"key_93\": \"val_93\"," \ + "\"key_94\": \"val_94\"," \ + "\"key_95\": \"val_95\"," \ + "\"key_96\": \"val_96\"," \ + "\"key_97\": \"val_97\"," \ + "\"key_98\": \"val_98\"," \ + "\"key_99\": \"val_99\"," \ + "\"key_100\": \"val_100\"," \ + "\"key_101\": \"val_101\"," \ + "\"key_102\": \"val_102\"," \ + "\"key_103\": \"val_103\"," \ + "\"key_104\": \"val_104\"," \ + "\"key_105\": \"val_105\"," \ + "\"key_106\": \"val_106\"," \ + "\"key_107\": \"val_107\"," \ + "\"key_108\": \"val_108\"," \ + "\"key_109\": \"val_109\"," \ + "\"key_110\": \"val_110\"," \ + "\"key_111\": \"val_111\"," \ + "\"key_112\": \"val_112\"," \ + "\"key_113\": \"val_113\"," \ + "\"key_114\": \"val_114\"," \ + "\"key_115\": \"val_115\"," \ + "\"key_116\": \"val_116\"," \ + "\"key_117\": \"val_117\"," \ + "\"key_118\": \"val_118\"," \ + "\"key_119\": \"val_119\"," \ + "\"key_120\": \"val_120\"," \ + "\"key_121\": \"val_121\"," \ + "\"key_122\": \"val_122\"," \ + "\"key_123\": \"val_123\"," \ + "\"key_124\": \"val_124\"," \ + "\"key_125\": \"val_125\"," \ + "\"key_126\": \"val_126\"," \ + "\"key_127\": \"val_127\"," \ + "\"key_128\": \"val_128\"," \ + "\"key_129\": \"val_129\"," \ + "\"key_130\": \"val_130\"," \ + "\"key_131\": \"val_131\"," \ + "\"key_132\": \"val_132\"," \ + "\"key_133\": \"val_133\"," \ + "\"key_134\": \"val_134\"," \ + "\"key_135\": \"val_135\"," \ + "\"key_136\": \"val_136\"," \ + "\"key_137\": \"val_137\"," \ + "\"key_138\": \"val_138\"," \ + "\"key_139\": \"val_139\"," \ + "\"key_140\": \"val_140\"," \ + "\"key_141\": \"val_141\"," \ + "\"key_142\": \"val_142\"," \ + "\"key_143\": \"val_143\"," \ + "\"key_144\": \"val_144\"," \ + "\"key_145\": \"val_145\"," \ + "\"key_146\": \"val_146\"," \ + "\"key_147\": \"val_147\"," \ + "\"key_148\": \"val_148\"," \ + "\"key_149\": \"val_149\"," \ + "\"key_150\": \"val_150\"," \ + "\"key_151\": \"val_151\"," \ + "\"key_152\": \"val_152\"," \ + "\"key_153\": \"val_153\"," \ + "\"key_154\": \"val_154\"," \ + "\"key_155\": \"val_155\"," \ + "\"key_156\": \"val_156\"," \ + "\"key_157\": \"val_157\"," \ + "\"key_158\": \"val_158\"," \ + "\"key_159\": \"val_159\"," \ + "\"key_160\": \"val_160\"," \ + "\"key_161\": \"val_161\"," \ + "\"key_162\": \"val_162\"," \ + "\"key_163\": \"val_163\"," \ + "\"key_164\": \"val_164\"," \ + "\"key_165\": \"val_165\"," \ + "\"key_166\": \"val_166\"," \ + "\"key_167\": \"val_167\"," \ + "\"key_168\": \"val_168\"," \ + "\"key_169\": \"val_169\"," \ + "\"key_170\": \"val_170\"," \ + "\"key_171\": \"val_171\"," \ + "\"key_172\": \"val_172\"," \ + "\"key_173\": \"val_173\"," \ + "\"key_174\": \"val_174\"," \ + "\"key_175\": \"val_175\"," \ + "\"key_176\": \"val_176\"," \ + "\"key_177\": \"val_177\"," \ + "\"key_178\": \"val_178\"," \ + "\"key_179\": \"val_179\"," \ + "\"key_180\": \"val_180\"," \ + "\"key_181\": \"val_181\"," \ + "\"key_182\": \"val_182\"," \ + "\"key_183\": \"val_183\"," \ + "\"key_184\": \"val_184\"," \ + "\"key_185\": \"val_185\"," \ + "\"key_186\": \"val_186\"," \ + "\"key_187\": \"val_187\"," \ + "\"key_188\": \"val_188\"," \ + "\"key_189\": \"val_189\"," \ + "\"key_190\": \"val_190\"," \ + "\"key_191\": \"val_191\"," \ + "\"key_192\": \"val_192\"," \ + "\"key_193\": \"val_193\"," \ + "\"key_194\": \"val_194\"," \ + "\"key_195\": \"val_195\"," \ + "\"key_196\": \"val_196\"," \ + "\"key_197\": \"val_197\"," \ + "\"key_198\": \"val_198\"," \ + "\"key_199\": \"val_199\"," \ + "\"key_200\": \"val_200\"," \ + "\"key_201\": \"val_201\"," \ + "\"key_202\": \"val_202\"," \ + "\"key_203\": \"val_203\"," \ + "\"key_204\": \"val_204\"," \ + "\"key_205\": \"val_205\"," \ + "\"key_206\": \"val_206\"," \ + "\"key_207\": \"val_207\"," \ + "\"key_208\": \"val_208\"," \ + "\"key_209\": \"val_209\"," \ + "\"key_210\": \"val_210\"," \ + "\"key_211\": \"val_211\"," \ + "\"key_212\": \"val_212\"," \ + "\"key_213\": \"val_213\"," \ + "\"key_214\": \"val_214\"," \ + "\"key_215\": \"val_215\"," \ + "\"key_216\": \"val_216\"," \ + "\"key_217\": \"val_217\"," \ + "\"key_218\": \"val_218\"," \ + "\"key_219\": \"val_219\"," \ + "\"key_220\": \"val_220\"," \ + "\"key_221\": \"val_221\"," \ + "\"key_222\": \"val_222\"," \ + "\"key_223\": \"val_223\"," \ + "\"key_224\": \"val_224\"," \ + "\"key_225\": \"val_225\"," \ + "\"key_226\": \"val_226\"," \ + "\"key_227\": \"val_227\"," \ + "\"key_228\": \"val_228\"," \ + "\"key_229\": \"val_229\"," \ + "\"key_230\": \"val_230\"," \ + "\"key_231\": \"val_231\"," \ + "\"key_232\": \"val_232\"," \ + "\"key_233\": \"val_233\"," \ + "\"key_234\": \"val_234\"," \ + "\"key_235\": \"val_235\"," \ + "\"key_236\": \"val_236\"," \ + "\"key_237\": \"val_237\"," \ + "\"key_238\": \"val_238\"," \ + "\"key_239\": \"val_239\"," \ + "\"key_240\": \"val_240\"," \ + "\"key_241\": \"val_241\"," \ + "\"key_242\": \"val_242\"," \ + "\"key_243\": \"val_243\"," \ + "\"key_244\": \"val_244\"," \ + "\"key_245\": \"val_245\"," \ + "\"key_246\": \"val_246\"," \ + "\"key_247\": \"val_247\"," \ + "\"key_248\": \"val_248\"," \ + "\"key_249\": \"val_249\"," \ + "\"key_250\": \"val_250\"," \ + "\"key_251\": \"val_251\"," \ + "\"key_252\": \"val_252\"," \ + "\"key_253\": \"val_253\"," \ + "\"key_254\": \"val_254\"," \ + "\"key_255\": \"val_255\"," \ + "\"key_256\": \"val_256\"," \ + "\"key_257\": \"val_257\"," \ + "\"key_258\": \"val_258\"," \ + "\"key_259\": \"val_259\"," \ + "\"key_260\": \"val_260\"," \ + "\"key_261\": \"val_261\"," \ + "\"key_262\": \"val_262\"," \ + "\"key_263\": \"val_263\"," \ + "\"key_264\": \"val_264\"," \ + "\"key_265\": \"val_265\"," \ + "\"key_266\": \"val_266\"," \ + "\"key_267\": \"val_267\"," \ + "\"key_268\": \"val_268\"," \ + "\"key_269\": \"val_269\"," \ + "\"key_270\": \"val_270\"," \ + "\"key_271\": \"val_271\"," \ + "\"key_272\": \"val_272\"," \ + "\"key_273\": \"val_273\"," \ + "\"key_274\": \"val_274\"," \ + "\"key_275\": \"val_275\"," \ + "\"key_276\": \"val_276\"," \ + "\"key_277\": \"val_277\"," \ + "\"key_278\": \"val_278\"," \ + "\"key_279\": \"val_279\"," \ + "\"key_280\": \"val_280\"," \ + "\"key_281\": \"val_281\"," \ + "\"key_282\": \"val_282\"," \ + "\"key_283\": \"val_283\"," \ + "\"key_284\": \"val_284\"," \ + "\"key_285\": \"val_285\"," \ + "\"key_286\": \"val_286\"," \ + "\"key_287\": \"val_287\"," \ + "\"key_288\": \"val_288\"," \ + "\"key_289\": \"val_289\"," \ + "\"key_290\": \"val_290\"," \ + "\"key_291\": \"val_291\"," \ + "\"key_292\": \"val_292\"," \ + "\"key_293\": \"val_293\"," \ + "\"key_294\": \"val_294\"," \ + "\"key_295\": \"val_295\"," \ + "\"key_296\": \"val_296\"," \ + "\"key_297\": \"val_297\"," \ + "\"key_298\": \"val_298\"," \ + "\"key_299\": \"val_299\"," \ + "\"key_300\": \"val_300\"," \ + "\"key_301\": \"val_301\"," \ + "\"key_302\": \"val_302\"," \ + "\"key_303\": \"val_303\"," \ + "\"key_304\": \"val_304\"," \ + "\"key_305\": \"val_305\"," \ + "\"key_306\": \"val_306\"," \ + "\"key_307\": \"val_307\"," \ + "\"key_308\": \"val_308\"," \ + "\"key_309\": \"val_309\"," \ + "\"key_310\": \"val_310\"," \ + "\"key_311\": \"val_311\"," \ + "\"key_312\": \"val_312\"," \ + "\"key_313\": \"val_313\"," \ + "\"key_314\": \"val_314\"," \ + "\"key_315\": \"val_315\"," \ + "\"key_316\": \"val_316\"," \ + "\"key_317\": \"val_317\"," \ + "\"key_318\": \"val_318\"," \ + "\"key_319\": \"val_319\"," \ + "\"key_320\": \"val_320\"," \ + "\"key_321\": \"val_321\"," \ + "\"key_322\": \"val_322\"," \ + "\"key_323\": \"val_323\"," \ + "\"key_324\": \"val_324\"," \ + "\"key_325\": \"val_325\"," \ + "\"key_326\": \"val_326\"," \ + "\"key_327\": \"val_327\"," \ + "\"key_328\": \"val_328\"," \ + "\"key_329\": \"val_329\"," \ + "\"key_330\": \"val_330\"," \ + "\"key_331\": \"val_331\"," \ + "\"key_332\": \"val_332\"," \ + "\"key_333\": \"val_333\"," \ + "\"key_334\": \"val_334\"," \ + "\"key_335\": \"val_335\"," \ + "\"key_336\": \"val_336\"," \ + "\"key_337\": \"val_337\"," \ + "\"key_338\": \"val_338\"," \ + "\"key_339\": \"val_339\"," \ + "\"key_340\": \"val_340\"," \ + "\"key_341\": \"val_341\"," \ + "\"key_342\": \"val_342\"," \ + "\"key_343\": \"val_343\"," \ + "\"key_344\": \"val_344\"," \ + "\"key_345\": \"val_345\"," \ + "\"key_346\": \"val_346\"," \ + "\"key_347\": \"val_347\"," \ + "\"key_348\": \"val_348\"," \ + "\"key_349\": \"val_349\"," \ + "\"key_350\": \"val_350\"," \ + "\"key_351\": \"val_351\"," \ + "\"key_352\": \"val_352\"," \ + "\"key_353\": \"val_353\"," \ + "\"key_354\": \"val_354\"," \ + "\"key_355\": \"val_355\"," \ + "\"key_356\": \"val_356\"," \ + "\"key_357\": \"val_357\"," \ + "\"key_358\": \"val_358\"," \ + "\"key_359\": \"val_359\"," \ + "\"key_360\": \"val_360\"," \ + "\"key_361\": \"val_361\"," \ + "\"key_362\": \"val_362\"," \ + "\"key_363\": \"val_363\"," \ + "\"key_364\": \"val_364\"," \ + "\"key_365\": \"val_365\"," \ + "\"key_366\": \"val_366\"," \ + "\"key_367\": \"val_367\"," \ + "\"key_368\": \"val_368\"," \ + "\"key_369\": \"val_369\"," \ + "\"key_370\": \"val_370\"," \ + "\"key_371\": \"val_371\"," \ + "\"key_372\": \"val_372\"," \ + "\"key_373\": \"val_373\"," \ + "\"key_374\": \"val_374\"," \ + "\"key_375\": \"val_375\"," \ + "\"key_376\": \"val_376\"," \ + "\"key_377\": \"val_377\"," \ + "\"key_378\": \"val_378\"," \ + "\"key_379\": \"val_379\"," \ + "\"key_380\": \"val_380\"," \ + "\"key_381\": \"val_381\"," \ + "\"key_382\": \"val_382\"," \ + "\"key_383\": \"val_383\"," \ + "\"key_384\": \"val_384\"," \ + "\"key_385\": \"val_385\"," \ + "\"key_386\": \"val_386\"," \ + "\"key_387\": \"val_387\"," \ + "\"key_388\": \"val_388\"," \ + "\"key_389\": \"val_389\"," \ + "\"key_390\": \"val_390\"," \ + "\"key_391\": \"val_391\"," \ + "\"key_392\": \"val_392\"," \ + "\"key_393\": \"val_393\"," \ + "\"key_394\": \"val_394\"," \ + "\"key_395\": \"val_395\"," \ + "\"key_396\": \"val_396\"," \ + "\"key_397\": \"val_397\"," \ + "\"key_398\": \"val_398\"," \ + "\"key_399\": \"val_399\"," \ + "\"key_400\": \"val_400\"," \ + "\"key_401\": \"val_401\"," \ + "\"key_402\": \"val_402\"," \ + "\"key_403\": \"val_403\"," \ + "\"key_404\": \"val_404\"," \ + "\"key_405\": \"val_405\"," \ + "\"key_406\": \"val_406\"," \ + "\"key_407\": \"val_407\"," \ + "\"key_408\": \"val_408\"," \ + "\"key_409\": \"val_409\"," \ + "\"key_410\": \"val_410\"," \ + "\"key_411\": \"val_411\"," \ + "\"key_412\": \"val_412\"," \ + "\"key_413\": \"val_413\"," \ + "\"key_414\": \"val_414\"," \ + "\"key_415\": \"val_415\"," \ + "\"key_416\": \"val_416\"," \ + "\"key_417\": \"val_417\"," \ + "\"key_418\": \"val_418\"," \ + "\"key_419\": \"val_419\"," \ + "\"key_420\": \"val_420\"," \ + "\"key_421\": \"val_421\"," \ + "\"key_422\": \"val_422\"," \ + "\"key_423\": \"val_423\"," \ + "\"key_424\": \"val_424\"," \ + "\"key_425\": \"val_425\"," \ + "\"key_426\": \"val_426\"," \ + "\"key_427\": \"val_427\"," \ + "\"key_428\": \"val_428\"," \ + "\"key_429\": \"val_429\"," \ + "\"key_430\": \"val_430\"," \ + "\"key_431\": \"val_431\"," \ + "\"key_432\": \"val_432\"," \ + "\"key_433\": \"val_433\"," \ + "\"key_434\": \"val_434\"," \ + "\"key_435\": \"val_435\"," \ + "\"key_436\": \"val_436\"," \ + "\"key_437\": \"val_437\"," \ + "\"key_438\": \"val_438\"," \ + "\"key_439\": \"val_439\"," \ + "\"key_440\": \"val_440\"," \ + "\"key_441\": \"val_441\"," \ + "\"key_442\": \"val_442\"," \ + "\"key_443\": \"val_443\"," \ + "\"key_444\": \"val_444\"," \ + "\"key_445\": \"val_445\"," \ + "\"key_446\": \"val_446\"," \ + "\"key_447\": \"val_447\"," \ + "\"key_448\": \"val_448\"," \ + "\"key_449\": \"val_449\"," \ + "\"key_450\": \"val_450\"," \ + "\"key_451\": \"val_451\"," \ + "\"key_452\": \"val_452\"," \ + "\"key_453\": \"val_453\"," \ + "\"key_454\": \"val_454\"," \ + "\"key_455\": \"val_455\"," \ + "\"key_456\": \"val_456\"," \ + "\"key_457\": \"val_457\"," \ + "\"key_458\": \"val_458\"," \ + "\"key_459\": \"val_459\"," \ + "\"key_460\": \"val_460\"," \ + "\"key_461\": \"val_461\"," \ + "\"key_462\": \"val_462\"," \ + "\"key_463\": \"val_463\"," \ + "\"key_464\": \"val_464\"," \ + "\"key_465\": \"val_465\"," \ + "\"key_466\": \"val_466\"," \ + "\"key_467\": \"val_467\"," \ + "\"key_468\": \"val_468\"," \ + "\"key_469\": \"val_469\"," \ + "\"key_470\": \"val_470\"," \ + "\"key_471\": \"val_471\"," \ + "\"key_472\": \"val_472\"," \ + "\"key_473\": \"val_473\"," \ + "\"key_474\": \"val_474\"," \ + "\"key_475\": \"val_475\"," \ + "\"key_476\": \"val_476\"," \ + "\"key_477\": \"val_477\"," \ + "\"key_478\": \"val_478\"," \ + "\"key_479\": \"val_479\"," \ + "\"key_480\": \"val_480\"," \ + "\"key_481\": \"val_481\"," \ + "\"key_482\": \"val_482\"," \ + "\"key_483\": \"val_483\"," \ + "\"key_484\": \"val_484\"," \ + "\"key_485\": \"val_485\"," \ + "\"key_486\": \"val_486\"," \ + "\"key_487\": \"val_487\"," \ + "\"key_488\": \"val_488\"," \ + "\"key_489\": \"val_489\"," \ + "\"key_490\": \"val_490\"," \ + "\"key_491\": \"val_491\"," \ + "\"key_492\": \"val_492\"," \ + "\"key_493\": \"val_493\"," \ + "\"key_494\": \"val_494\"," \ + "\"key_495\": \"val_495\"," \ + "\"key_496\": \"val_496\"," \ + "\"key_497\": \"val_497\"," \ + "\"key_498\": \"val_498\"," \ + "\"key_499\": \"val_499\"," \ + "\"key_500\": \"val_500\"," \ + "\"key_501\": \"val_501\"," \ + "\"key_502\": \"val_502\"," \ + "\"key_503\": \"val_503\"," \ + "\"key_504\": \"val_504\"," \ + "\"key_505\": \"val_505\"," \ + "\"key_506\": \"val_506\"," \ + "\"key_507\": \"val_507\"," \ + "\"key_508\": \"val_508\"," \ + "\"key_509\": \"val_509\"," \ + "\"key_510\": \"val_510\"," \ + "\"key_511\": \"val_511\"," \ + "\"key_512\": \"val_512\"," \ + "\"key_513\": \"val_513\"," \ + "\"key_514\": \"val_514\"," \ + "\"key_515\": \"val_515\"," \ + "\"key_516\": \"val_516\"," \ + "\"key_517\": \"val_517\"," \ + "\"key_518\": \"val_518\"," \ + "\"key_519\": \"val_519\"," \ + "\"key_520\": \"val_520\"," \ + "\"key_521\": \"val_521\"," \ + "\"key_522\": \"val_522\"," \ + "\"key_523\": \"val_523\"," \ + "\"key_524\": \"val_524\"," \ + "\"key_525\": \"val_525\"," \ + "\"key_526\": \"val_526\"," \ + "\"key_527\": \"val_527\"," \ + "\"key_528\": \"val_528\"," \ + "\"key_529\": \"val_529\"," \ + "\"key_530\": \"val_530\"," \ + "\"key_531\": \"val_531\"," \ + "\"key_532\": \"val_532\"," \ + "\"key_533\": \"val_533\"," \ + "\"key_534\": \"val_534\"," \ + "\"key_535\": \"val_535\"," \ + "\"key_536\": \"val_536\"," \ + "\"key_537\": \"val_537\"," \ + "\"key_538\": \"val_538\"," \ + "\"key_539\": \"val_539\"," \ + "\"key_540\": \"val_540\"," \ + "\"key_541\": \"val_541\"," \ + "\"key_542\": \"val_542\"," \ + "\"key_543\": \"val_543\"," \ + "\"key_544\": \"val_544\"," \ + "\"key_545\": \"val_545\"," \ + "\"key_546\": \"val_546\"," \ + "\"key_547\": \"val_547\"," \ + "\"key_548\": \"val_548\"," \ + "\"key_549\": \"val_549\"," \ + "\"key_550\": \"val_550\"," \ + "\"key_551\": \"val_551\"," \ + "\"key_552\": \"val_552\"," \ + "\"key_553\": \"val_553\"," \ + "\"key_554\": \"val_554\"," \ + "\"key_555\": \"val_555\"," \ + "\"key_556\": \"val_556\"," \ + "\"key_557\": \"val_557\"," \ + "\"key_558\": \"val_558\"," \ + "\"key_559\": \"val_559\"," \ + "\"key_560\": \"val_560\"," \ + "\"key_561\": \"val_561\"," \ + "\"key_562\": \"val_562\"," \ + "\"key_563\": \"val_563\"," \ + "\"key_564\": \"val_564\"," \ + "\"key_565\": \"val_565\"," \ + "\"key_566\": \"val_566\"," \ + "\"key_567\": \"val_567\"," \ + "\"key_568\": \"val_568\"," \ + "\"key_569\": \"val_569\"," \ + "\"key_570\": \"val_570\"," \ + "\"key_571\": \"val_571\"," \ + "\"key_572\": \"val_572\"," \ + "\"key_573\": \"val_573\"," \ + "\"key_574\": \"val_574\"," \ + "\"key_575\": \"val_575\"," \ + "\"key_576\": \"val_576\"," \ + "\"key_577\": \"val_577\"," \ + "\"key_578\": \"val_578\"," \ + "\"key_579\": \"val_579\"," \ + "\"key_580\": \"val_580\"," \ + "\"key_581\": \"val_581\"," \ + "\"key_582\": \"val_582\"," \ + "\"key_583\": \"val_583\"," \ + "\"key_584\": \"val_584\"," \ + "\"key_585\": \"val_585\"," \ + "\"key_586\": \"val_586\"," \ + "\"key_587\": \"val_587\"," \ + "\"key_588\": \"val_588\"," \ + "\"key_589\": \"val_589\"," \ + "\"key_590\": \"val_590\"," \ + "\"key_591\": \"val_591\"," \ + "\"key_592\": \"val_592\"," \ + "\"key_593\": \"val_593\"," \ + "\"key_594\": \"val_594\"," \ + "\"key_595\": \"val_595\"," \ + "\"key_596\": \"val_596\"," \ + "\"key_597\": \"val_597\"," \ + "\"key_598\": \"val_598\"," \ + "\"key_599\": \"val_599\"," \ + "\"key_600\": \"val_600\"," \ + "\"key_601\": \"val_601\"," \ + "\"key_602\": \"val_602\"," \ + "\"key_603\": \"val_603\"," \ + "\"key_604\": \"val_604\"," \ + "\"key_605\": \"val_605\"," \ + "\"key_606\": \"val_606\"," \ + "\"key_607\": \"val_607\"," \ + "\"key_608\": \"val_608\"," \ + "\"key_609\": \"val_609\"," \ + "\"key_610\": \"val_610\"," \ + "\"key_611\": \"val_611\"," \ + "\"key_612\": \"val_612\"," \ + "\"key_613\": \"val_613\"," \ + "\"key_614\": \"val_614\"," \ + "\"key_615\": \"val_615\"," \ + "\"key_616\": \"val_616\"," \ + "\"key_617\": \"val_617\"," \ + "\"key_618\": \"val_618\"," \ + "\"key_619\": \"val_619\"," \ + "\"key_620\": \"val_620\"," \ + "\"key_621\": \"val_621\"," \ + "\"key_622\": \"val_622\"," \ + "\"key_623\": \"val_623\"," \ + "\"key_624\": \"val_624\"," \ + "\"key_625\": \"val_625\"," \ + "\"key_626\": \"val_626\"," \ + "\"key_627\": \"val_627\"," \ + "\"key_628\": \"val_628\"," \ + "\"key_629\": \"val_629\"," \ + "\"key_630\": \"val_630\"," \ + "\"key_631\": \"val_631\"," \ + "\"key_632\": \"val_632\"," \ + "\"key_633\": \"val_633\"," \ + "\"key_634\": \"val_634\"," \ + "\"key_635\": \"val_635\"," \ + "\"key_636\": \"val_636\"," \ + "\"key_637\": \"val_637\"," \ + "\"key_638\": \"val_638\"," \ + "\"key_639\": \"val_639\"," \ + "\"key_640\": \"val_640\"," \ + "\"key_641\": \"val_641\"," \ + "\"key_642\": \"val_642\"," \ + "\"key_643\": \"val_643\"," \ + "\"key_644\": \"val_644\"," \ + "\"key_645\": \"val_645\"," \ + "\"key_646\": \"val_646\"," \ + "\"key_647\": \"val_647\"," \ + "\"key_648\": \"val_648\"," \ + "\"key_649\": \"val_649\"," \ + "\"key_650\": \"val_650\"," \ + "\"key_651\": \"val_651\"," \ + "\"key_652\": \"val_652\"," \ + "\"key_653\": \"val_653\"," \ + "\"key_654\": \"val_654\"," \ + "\"key_655\": \"val_655\"," \ + "\"key_656\": \"val_656\"," \ + "\"key_657\": \"val_657\"," \ + "\"key_658\": \"val_658\"," \ + "\"key_659\": \"val_659\"," \ + "\"key_660\": \"val_660\"," \ + "\"key_661\": \"val_661\"," \ + "\"key_662\": \"val_662\"," \ + "\"key_663\": \"val_663\"," \ + "\"key_664\": \"val_664\"," \ + "\"key_665\": \"val_665\"," \ + "\"key_666\": \"val_666\"," \ + "\"key_667\": \"val_667\"," \ + "\"key_668\": \"val_668\"," \ + "\"key_669\": \"val_669\"," \ + "\"key_670\": \"val_670\"," \ + "\"key_671\": \"val_671\"," \ + "\"key_672\": \"val_672\"," \ + "\"key_673\": \"val_673\"," \ + "\"key_674\": \"val_674\"," \ + "\"key_675\": \"val_675\"," \ + "\"key_676\": \"val_676\"," \ + "\"key_677\": \"val_677\"," \ + "\"key_678\": \"val_678\"," \ + "\"key_679\": \"val_679\"," \ + "\"key_680\": \"val_680\"," \ + "\"key_681\": \"val_681\"," \ + "\"key_682\": \"val_682\"," \ + "\"key_683\": \"val_683\"," \ + "\"key_684\": \"val_684\"," \ + "\"key_685\": \"val_685\"," \ + "\"key_686\": \"val_686\"," \ + "\"key_687\": \"val_687\"," \ + "\"key_688\": \"val_688\"," \ + "\"key_689\": \"val_689\"," \ + "\"key_690\": \"val_690\"," \ + "\"key_691\": \"val_691\"," \ + "\"key_692\": \"val_692\"," \ + "\"key_693\": \"val_693\"," \ + "\"key_694\": \"val_694\"," \ + "\"key_695\": \"val_695\"," \ + "\"key_696\": \"val_696\"," \ + "\"key_697\": \"val_697\"," \ + "\"key_698\": \"val_698\"," \ + "\"key_699\": \"val_699\"," \ + "\"key_700\": \"val_700\"," \ + "\"key_701\": \"val_701\"," \ + "\"key_702\": \"val_702\"," \ + "\"key_703\": \"val_703\"," \ + "\"key_704\": \"val_704\"," \ + "\"key_705\": \"val_705\"," \ + "\"key_706\": \"val_706\"," \ + "\"key_707\": \"val_707\"," \ + "\"key_708\": \"val_708\"," \ + "\"key_709\": \"val_709\"," \ + "\"key_710\": \"val_710\"," \ + "\"key_711\": \"val_711\"," \ + "\"key_712\": \"val_712\"," \ + "\"key_713\": \"val_713\"," \ + "\"key_714\": \"val_714\"," \ + "\"key_715\": \"val_715\"," \ + "\"key_716\": \"val_716\"," \ + "\"key_717\": \"val_717\"," \ + "\"key_718\": \"val_718\"," \ + "\"key_719\": \"val_719\"," \ + "\"key_720\": \"val_720\"," \ + "\"key_721\": \"val_721\"," \ + "\"key_722\": \"val_722\"," \ + "\"key_723\": \"val_723\"," \ + "\"key_724\": \"val_724\"," \ + "\"key_725\": \"val_725\"," \ + "\"key_726\": \"val_726\"," \ + "\"key_727\": \"val_727\"," \ + "\"key_728\": \"val_728\"," \ + "\"key_729\": \"val_729\"," \ + "\"key_730\": \"val_730\"," \ + "\"key_731\": \"val_731\"," \ + "\"key_732\": \"val_732\"," \ + "\"key_733\": \"val_733\"," \ + "\"key_734\": \"val_734\"," \ + "\"key_735\": \"val_735\"," \ + "\"key_736\": \"val_736\"," \ + "\"key_737\": \"val_737\"," \ + "\"key_738\": \"val_738\"," \ + "\"key_739\": \"val_739\"," \ + "\"key_740\": \"val_740\"," \ + "\"key_741\": \"val_741\"," \ + "\"key_742\": \"val_742\"," \ + "\"key_743\": \"val_743\"," \ + "\"key_744\": \"val_744\"," \ + "\"key_745\": \"val_745\"," \ + "\"key_746\": \"val_746\"," \ + "\"key_747\": \"val_747\"," \ + "\"key_748\": \"val_748\"," \ + "\"key_749\": \"val_749\"," \ + "\"key_750\": \"val_750\"," \ + "\"key_751\": \"val_751\"," \ + "\"key_752\": \"val_752\"," \ + "\"key_753\": \"val_753\"," \ + "\"key_754\": \"val_754\"," \ + "\"key_755\": \"val_755\"," \ + "\"key_756\": \"val_756\"," \ + "\"key_757\": \"val_757\"," \ + "\"key_758\": \"val_758\"," \ + "\"key_759\": \"val_759\"," \ + "\"key_760\": \"val_760\"," \ + "\"key_761\": \"val_761\"," \ + "\"key_762\": \"val_762\"," \ + "\"key_763\": \"val_763\"," \ + "\"key_764\": \"val_764\"," \ + "\"key_765\": \"val_765\"," \ + "\"key_766\": \"val_766\"," \ + "\"key_767\": \"val_767\"," \ + "\"key_768\": \"val_768\"," \ + "\"key_769\": \"val_769\"," \ + "\"key_770\": \"val_770\"," \ + "\"key_771\": \"val_771\"," \ + "\"key_772\": \"val_772\"," \ + "\"key_773\": \"val_773\"," \ + "\"key_774\": \"val_774\"," \ + "\"key_775\": \"val_775\"," \ + "\"key_776\": \"val_776\"," \ + "\"key_777\": \"val_777\"," \ + "\"key_778\": \"val_778\"," \ + "\"key_779\": \"val_779\"," \ + "\"key_780\": \"val_780\"," \ + "\"key_781\": \"val_781\"," \ + "\"key_782\": \"val_782\"," \ + "\"key_783\": \"val_783\"," \ + "\"key_784\": \"val_784\"," \ + "\"key_785\": \"val_785\"," \ + "\"key_786\": \"val_786\"," \ + "\"key_787\": \"val_787\"," \ + "\"key_788\": \"val_788\"," \ + "\"key_789\": \"val_789\"," \ + "\"key_790\": \"val_790\"," \ + "\"key_791\": \"val_791\"," \ + "\"key_792\": \"val_792\"," \ + "\"key_793\": \"val_793\"," \ + "\"key_794\": \"val_794\"," \ + "\"key_795\": \"val_795\"," \ + "\"key_796\": \"val_796\"," \ + "\"key_797\": \"val_797\"," \ + "\"key_798\": \"val_798\"," \ + "\"key_799\": \"val_799\"," \ + "\"key_800\": \"val_800\"," \ + "\"key_801\": \"val_801\"," \ + "\"key_802\": \"val_802\"," \ + "\"key_803\": \"val_803\"," \ + "\"key_804\": \"val_804\"," \ + "\"key_805\": \"val_805\"," \ + "\"key_806\": \"val_806\"," \ + "\"key_807\": \"val_807\"," \ + "\"key_808\": \"val_808\"," \ + "\"key_809\": \"val_809\"," \ + "\"key_810\": \"val_810\"," \ + "\"key_811\": \"val_811\"," \ + "\"key_812\": \"val_812\"," \ + "\"key_813\": \"val_813\"," \ + "\"key_814\": \"val_814\"," \ + "\"key_815\": \"val_815\"," \ + "\"key_816\": \"val_816\"," \ + "\"key_817\": \"val_817\"," \ + "\"key_818\": \"val_818\"," \ + "\"key_819\": \"val_819\"," \ + "\"key_820\": \"val_820\"," \ + "\"key_821\": \"val_821\"," \ + "\"key_822\": \"val_822\"," \ + "\"key_823\": \"val_823\"," \ + "\"key_824\": \"val_824\"," \ + "\"key_825\": \"val_825\"," \ + "\"key_826\": \"val_826\"," \ + "\"key_827\": \"val_827\"," \ + "\"key_828\": \"val_828\"," \ + "\"key_829\": \"val_829\"," \ + "\"key_830\": \"val_830\"," \ + "\"key_831\": \"val_831\"," \ + "\"key_832\": \"val_832\"," \ + "\"key_833\": \"val_833\"," \ + "\"key_834\": \"val_834\"," \ + "\"key_835\": \"val_835\"," \ + "\"key_836\": \"val_836\"," \ + "\"key_837\": \"val_837\"," \ + "\"key_838\": \"val_838\"," \ + "\"key_839\": \"val_839\"," \ + "\"key_840\": \"val_840\"," \ + "\"key_841\": \"val_841\"," \ + "\"key_842\": \"val_842\"," \ + "\"key_843\": \"val_843\"," \ + "\"key_844\": \"val_844\"," \ + "\"key_845\": \"val_845\"," \ + "\"key_846\": \"val_846\"," \ + "\"key_847\": \"val_847\"," \ + "\"key_848\": \"val_848\"," \ + "\"key_849\": \"val_849\"," \ + "\"key_850\": \"val_850\"," \ + "\"key_851\": \"val_851\"," \ + "\"key_852\": \"val_852\"," \ + "\"key_853\": \"val_853\"," \ + "\"key_854\": \"val_854\"," \ + "\"key_855\": \"val_855\"," \ + "\"key_856\": \"val_856\"," \ + "\"key_857\": \"val_857\"," \ + "\"key_858\": \"val_858\"," \ + "\"key_859\": \"val_859\"," \ + "\"key_860\": \"val_860\"," \ + "\"key_861\": \"val_861\"," \ + "\"key_862\": \"val_862\"," \ + "\"key_863\": \"val_863\"," \ + "\"key_864\": \"val_864\"," \ + "\"key_865\": \"val_865\"," \ + "\"key_866\": \"val_866\"," \ + "\"key_867\": \"val_867\"," \ + "\"key_868\": \"val_868\"," \ + "\"key_869\": \"val_869\"," \ + "\"key_870\": \"val_870\"," \ + "\"key_871\": \"val_871\"," \ + "\"key_872\": \"val_872\"," \ + "\"key_873\": \"val_873\"," \ + "\"key_874\": \"val_874\"," \ + "\"key_875\": \"val_875\"," \ + "\"key_876\": \"val_876\"," \ + "\"key_877\": \"val_877\"," \ + "\"key_878\": \"val_878\"," \ + "\"key_879\": \"val_879\"," \ + "\"key_880\": \"val_880\"," \ + "\"key_881\": \"val_881\"," \ + "\"key_882\": \"val_882\"," \ + "\"key_883\": \"val_883\"," \ + "\"key_884\": \"val_884\"," \ + "\"key_885\": \"val_885\"," \ + "\"key_886\": \"val_886\"," \ + "\"key_887\": \"val_887\"," \ + "\"key_888\": \"val_888\"," \ + "\"key_889\": \"val_889\"," \ + "\"key_890\": \"val_890\"," \ + "\"key_891\": \"val_891\"," \ + "\"key_892\": \"val_892\"," \ + "\"key_893\": \"val_893\"," \ + "\"key_894\": \"val_894\"," \ + "\"key_895\": \"val_895\"," \ + "\"key_896\": \"val_896\"," \ + "\"key_897\": \"val_897\"," \ + "\"key_898\": \"val_898\"," \ + "\"key_899\": \"val_899\"," \ + "\"key_900\": \"val_900\"," \ + "\"key_901\": \"val_901\"," \ + "\"key_902\": \"val_902\"," \ + "\"key_903\": \"val_903\"," \ + "\"key_904\": \"val_904\"," \ + "\"key_905\": \"val_905\"," \ + "\"key_906\": \"val_906\"," \ + "\"key_907\": \"val_907\"," \ + "\"key_908\": \"val_908\"," \ + "\"key_909\": \"val_909\"," \ + "\"key_910\": \"val_910\"," \ + "\"key_911\": \"val_911\"," \ + "\"key_912\": \"val_912\"," \ + "\"key_913\": \"val_913\"," \ + "\"key_914\": \"val_914\"," \ + "\"key_915\": \"val_915\"," \ + "\"key_916\": \"val_916\"," \ + "\"key_917\": \"val_917\"," \ + "\"key_918\": \"val_918\"," \ + "\"key_919\": \"val_919\"," \ + "\"key_920\": \"val_920\"," \ + "\"key_921\": \"val_921\"," \ + "\"key_922\": \"val_922\"," \ + "\"key_923\": \"val_923\"," \ + "\"key_924\": \"val_924\"," \ + "\"key_925\": \"val_925\"," \ + "\"key_926\": \"val_926\"," \ + "\"key_927\": \"val_927\"," \ + "\"key_928\": \"val_928\"," \ + "\"key_929\": \"val_929\"," \ + "\"key_930\": \"val_930\"," \ + "\"key_931\": \"val_931\"," \ + "\"key_932\": \"val_932\"," \ + "\"key_933\": \"val_933\"," \ + "\"key_934\": \"val_934\"," \ + "\"key_935\": \"val_935\"," \ + "\"key_936\": \"val_936\"," \ + "\"key_937\": \"val_937\"," \ + "\"key_938\": \"val_938\"," \ + "\"key_939\": \"val_939\"," \ + "\"key_940\": \"val_940\"," \ + "\"key_941\": \"val_941\"," \ + "\"key_942\": \"val_942\"," \ + "\"key_943\": \"val_943\"," \ + "\"key_944\": \"val_944\"," \ + "\"key_945\": \"val_945\"," \ + "\"key_946\": \"val_946\"," \ + "\"key_947\": \"val_947\"," \ + "\"key_948\": \"val_948\"," \ + "\"key_949\": \"val_949\"," \ + "\"key_950\": \"val_950\"," \ + "\"key_951\": \"val_951\"," \ + "\"key_952\": \"val_952\"," \ + "\"key_953\": \"val_953\"," \ + "\"key_954\": \"val_954\"," \ + "\"key_955\": \"val_955\"," \ + "\"key_956\": \"val_956\"," \ + "\"key_957\": \"val_957\"," \ + "\"key_958\": \"val_958\"," \ + "\"key_959\": \"val_959\"," \ + "\"key_960\": \"val_960\"," \ + "\"key_961\": \"val_961\"," \ + "\"key_962\": \"val_962\"," \ + "\"key_963\": \"val_963\"," \ + "\"key_964\": \"val_964\"," \ + "\"key_965\": \"val_965\"," \ + "\"key_966\": \"val_966\"," \ + "\"key_967\": \"val_967\"," \ + "\"key_968\": \"val_968\"," \ + "\"key_969\": \"val_969\"," \ + "\"key_970\": \"val_970\"," \ + "\"key_971\": \"val_971\"," \ + "\"key_972\": \"val_972\"," \ + "\"key_973\": \"val_973\"," \ + "\"key_974\": \"val_974\"," \ + "\"key_975\": \"val_975\"," \ + "\"key_976\": \"val_976\"," \ + "\"key_977\": \"val_977\"," \ + "\"key_978\": \"val_978\"," \ + "\"key_979\": \"val_979\"," \ + "\"key_980\": \"val_980\"," \ + "\"key_981\": \"val_981\"," \ + "\"key_982\": \"val_982\"," \ + "\"key_983\": \"val_983\"," \ + "\"key_984\": \"val_984\"," \ + "\"key_985\": \"val_985\"," \ + "\"key_986\": \"val_986\"," \ + "\"key_987\": \"val_987\"," \ + "\"key_988\": \"val_988\"," \ + "\"key_989\": \"val_989\"," \ + "\"key_990\": \"val_990\"," \ + "\"key_991\": \"val_991\"," \ + "\"key_992\": \"val_992\"," \ + "\"key_993\": \"val_993\"," \ + "\"key_994\": \"val_994\"," \ + "\"key_995\": \"val_995\"," \ + "\"key_996\": \"val_996\"," \ + "\"key_997\": \"val_997\"," \ + "\"key_998\": \"val_998\"," \ + "\"key_999\": \"val_999\"," \ + "\"END_KEY\": \"JSON_END\"" \ + "}]" + diff --git a/src/fluent-bit/tests/runtime/data/common/json_small.h b/src/fluent-bit/tests/runtime/data/common/json_small.h new file mode 100755 index 000000000..b4af842f1 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/common/json_small.h @@ -0,0 +1,256 @@ +#define JSON_SMALL "[" \ + "1448403340," \ + "{" \ + "\"key_0\": false," \ + "\"key_1\": false," \ + "\"key_2\": false," \ + "\"key_3\": false," \ + "\"key_4\": false," \ + "\"key_5\": false," \ + "\"key_6\": false," \ + "\"key_7\": false," \ + "\"key_8\": false," \ + "\"key_9\": false," \ + "\"key_10\": false," \ + "\"key_11\": false," \ + "\"key_12\": false," \ + "\"key_13\": false," \ + "\"key_14\": false," \ + "\"key_15\": false," \ + "\"key_16\": false," \ + "\"key_17\": false," \ + "\"key_18\": false," \ + "\"key_19\": false," \ + "\"key_20\": false," \ + "\"key_21\": false," \ + "\"key_22\": false," \ + "\"key_23\": false," \ + "\"key_24\": false," \ + "\"key_25\": false," \ + "\"key_26\": false," \ + "\"key_27\": false," \ + "\"key_28\": false," \ + "\"key_29\": false," \ + "\"key_30\": false," \ + "\"key_31\": false," \ + "\"key_32\": false," \ + "\"key_33\": false," \ + "\"key_34\": false," \ + "\"key_35\": false," \ + "\"key_36\": false," \ + "\"key_37\": false," \ + "\"key_38\": false," \ + "\"key_39\": false," \ + "\"key_40\": false," \ + "\"key_41\": false," \ + "\"key_42\": false," \ + "\"key_43\": false," \ + "\"key_44\": false," \ + "\"key_45\": false," \ + "\"key_46\": false," \ + "\"key_47\": false," \ + "\"key_48\": false," \ + "\"key_49\": false," \ + "\"key_50\": false," \ + "\"key_51\": false," \ + "\"key_52\": false," \ + "\"key_53\": false," \ + "\"key_54\": false," \ + "\"key_55\": false," \ + "\"key_56\": false," \ + "\"key_57\": false," \ + "\"key_58\": false," \ + "\"key_59\": false," \ + "\"key_60\": false," \ + "\"key_61\": false," \ + "\"key_62\": false," \ + "\"key_63\": false," \ + "\"key_64\": false," \ + "\"key_65\": false," \ + "\"key_66\": false," \ + "\"key_67\": false," \ + "\"key_68\": false," \ + "\"key_69\": false," \ + "\"key_70\": false," \ + "\"key_71\": false," \ + "\"key_72\": false," \ + "\"key_73\": false," \ + "\"key_74\": false," \ + "\"key_75\": false," \ + "\"key_76\": false," \ + "\"key_77\": false," \ + "\"key_78\": false," \ + "\"key_79\": false," \ + "\"key_80\": false," \ + "\"key_81\": false," \ + "\"key_82\": false," \ + "\"key_83\": false," \ + "\"key_84\": false," \ + "\"key_85\": false," \ + "\"key_86\": false," \ + "\"key_87\": false," \ + "\"key_88\": false," \ + "\"key_89\": false," \ + "\"key_90\": false," \ + "\"key_91\": false," \ + "\"key_92\": false," \ + "\"key_93\": false," \ + "\"key_94\": false," \ + "\"key_95\": false," \ + "\"key_96\": false," \ + "\"key_97\": false," \ + "\"key_98\": false," \ + "\"key_99\": false," \ + "\"key_100\": false," \ + "\"key_101\": false," \ + "\"key_102\": false," \ + "\"key_103\": false," \ + "\"key_104\": false," \ + "\"key_105\": false," \ + "\"key_106\": false," \ + "\"key_107\": false," \ + "\"key_108\": false," \ + "\"key_109\": false," \ + "\"key_110\": false," \ + "\"key_111\": false," \ + "\"key_112\": false," \ + "\"key_113\": false," \ + "\"key_114\": false," \ + "\"key_115\": false," \ + "\"key_116\": false," \ + "\"key_117\": false," \ + "\"key_118\": false," \ + "\"key_119\": false," \ + "\"key_120\": false," \ + "\"key_121\": false," \ + "\"key_122\": false," \ + "\"key_123\": false," \ + "\"key_124\": false," \ + "\"key_125\": false," \ + "\"key_126\": false," \ + "\"key_127\": false," \ + "\"key_128\": false," \ + "\"key_129\": false," \ + "\"key_130\": false," \ + "\"key_131\": false," \ + "\"key_132\": false," \ + "\"key_133\": false," \ + "\"key_134\": false," \ + "\"key_135\": false," \ + "\"key_136\": false," \ + "\"key_137\": false," \ + "\"key_138\": false," \ + "\"key_139\": false," \ + "\"key_140\": false," \ + "\"key_141\": false," \ + "\"key_142\": false," \ + "\"key_143\": false," \ + "\"key_144\": false," \ + "\"key_145\": false," \ + "\"key_146\": false," \ + "\"key_147\": false," \ + "\"key_148\": false," \ + "\"key_149\": false," \ + "\"key_150\": false," \ + "\"key_151\": false," \ + "\"key_152\": false," \ + "\"key_153\": false," \ + "\"key_154\": false," \ + "\"key_155\": false," \ + "\"key_156\": false," \ + "\"key_157\": false," \ + "\"key_158\": false," \ + "\"key_159\": false," \ + "\"key_160\": false," \ + "\"key_161\": false," \ + "\"key_162\": false," \ + "\"key_163\": false," \ + "\"key_164\": false," \ + "\"key_165\": false," \ + "\"key_166\": false," \ + "\"key_167\": false," \ + "\"key_168\": false," \ + "\"key_169\": false," \ + "\"key_170\": false," \ + "\"key_171\": false," \ + "\"key_172\": false," \ + "\"key_173\": false," \ + "\"key_174\": false," \ + "\"key_175\": false," \ + "\"key_176\": false," \ + "\"key_177\": false," \ + "\"key_178\": false," \ + "\"key_179\": false," \ + "\"key_180\": false," \ + "\"key_181\": false," \ + "\"key_182\": false," \ + "\"key_183\": false," \ + "\"key_184\": false," \ + "\"key_185\": false," \ + "\"key_186\": false," \ + "\"key_187\": false," \ + "\"key_188\": false," \ + "\"key_189\": false," \ + "\"key_190\": false," \ + "\"key_191\": false," \ + "\"key_192\": false," \ + "\"key_193\": false," \ + "\"key_194\": false," \ + "\"key_195\": false," \ + "\"key_196\": false," \ + "\"key_197\": false," \ + "\"key_198\": false," \ + "\"key_199\": false," \ + "\"key_200\": false," \ + "\"key_201\": false," \ + "\"key_202\": false," \ + "\"key_203\": false," \ + "\"key_204\": false," \ + "\"key_205\": false," \ + "\"key_206\": false," \ + "\"key_207\": false," \ + "\"key_208\": false," \ + "\"key_209\": false," \ + "\"key_210\": false," \ + "\"key_211\": false," \ + "\"key_212\": false," \ + "\"key_213\": false," \ + "\"key_214\": false," \ + "\"key_215\": false," \ + "\"key_216\": false," \ + "\"key_217\": false," \ + "\"key_218\": false," \ + "\"key_219\": false," \ + "\"key_220\": false," \ + "\"key_221\": false," \ + "\"key_222\": false," \ + "\"key_223\": false," \ + "\"key_224\": false," \ + "\"key_225\": false," \ + "\"key_226\": false," \ + "\"key_227\": false," \ + "\"key_228\": false," \ + "\"key_229\": false," \ + "\"key_230\": false," \ + "\"key_231\": false," \ + "\"key_232\": false," \ + "\"key_233\": false," \ + "\"key_234\": false," \ + "\"key_235\": false," \ + "\"key_236\": false," \ + "\"key_237\": false," \ + "\"key_238\": false," \ + "\"key_239\": false," \ + "\"key_240\": false," \ + "\"key_241\": false," \ + "\"key_242\": false," \ + "\"key_243\": false," \ + "\"key_244\": false," \ + "\"key_245\": false," \ + "\"key_246\": false," \ + "\"key_247\": false," \ + "\"key_248\": false," \ + "\"key_249\": false," \ + "\"END_KEY\": \"JSON_END\"" \ + "}]" + diff --git a/src/fluent-bit/tests/runtime/data/common/parsers.conf b/src/fluent-bit/tests/runtime/data/common/parsers.conf new file mode 120000 index 000000000..944111e31 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/common/parsers.conf @@ -0,0 +1 @@ +../../../../conf/parsers.conf
\ No newline at end of file diff --git a/src/fluent-bit/tests/runtime/data/datadog/json.h b/src/fluent-bit/tests/runtime/data/datadog/json.h new file mode 100644 index 000000000..146dd5324 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/datadog/json.h @@ -0,0 +1,12 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +#define JSON "[" \ + "1448403340," \ + "{" \ + "\"key_0\": false," \ + "\"key_1\": true," \ + "\"key_2\": \"some string\"," \ + "\"key_3\": 0.12345678," \ + "\"key.4\": 5000," \ + "\"END_KEY\": \"JSON_END\"" \ + "}]" diff --git a/src/fluent-bit/tests/runtime/data/es/json_es.h b/src/fluent-bit/tests/runtime/data/es/json_es.h new file mode 100755 index 000000000..40f8ab1ca --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/es/json_es.h @@ -0,0 +1,17 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +#define JSON_ES "[" \ + "1448403340," \ + "{" \ + "\"key_0\": false," \ + "\"key_1\": true," \ + "\"key_2\": \"some string\"," \ + "\"key_3\": 0.12345678," \ + "\"key.4\": 5000," \ + "\"END_KEY\": \"JSON_END\"" \ + "}]" + + +#define JSON_DOTS \ + "[1448403340," \ + "{\".le.vel\":\"error\", \".fo.o\":[{\".o.k\": [{\".b.ar\": \"baz\"}]}]}]" diff --git a/src/fluent-bit/tests/runtime/data/in_elasticsearch/json_bulk.h b/src/fluent-bit/tests/runtime/data/in_elasticsearch/json_bulk.h new file mode 100644 index 000000000..ca8f751c0 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/in_elasticsearch/json_bulk.h @@ -0,0 +1,25 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +#define NDJSON_BULK "{ \"index\" : { \"_index\" : \"test\", \"_id\" : \"1\" } }\n" \ + "{ \"field1\" : \"value1\" }\n" \ + "{ \"delete\" : { \"_index\" : \"test\", \"_id\" : \"2\" } }\n" \ + "{ \"create\" : { \"_index\" : \"test\", \"_id\" : \"3\" } }\n" \ + "{ \"field1\" : \"value3\", \"field2\" : \"value4\" }\n" \ + "{ \"update\" : {\"_id\" : \"1\", \"_index\" : \"test\"} }\n" \ + "{ \"doc\" : {\"field2\" : \"value2\"} }\n" \ + "{ \"index\" : { \"_index\" : \"test\", \"_id\" : \"10\" } }\n" \ + "{ \"field1\" : \"value1\", \"a\": \"line\", \"that\" : \"is\", \"long\": \"line\", \"contained\": \"request\" }\n" \ + "{ \"delete\" : { \"_index\" : \"test\", \"_id\" : \"20\" } }\n" \ + "{ \"create\" : { \"_index\" : \"test\", \"_id\" : \"30\" } }\n" \ + "{ \"field10\" : \"value30\", \"field20\" : \"value40\", \"message\": \"ok\" }\n" \ + "{ \"update\" : {\"_id\" : \"10\", \"_index\" : \"test\"} }\n" \ + "{ \"doc\" : {\"field20\" : \"value20\"} }\n" \ + "{ \"index\" : { \"_index\" : \"test\", \"_id\" : \"11\" } }\n" \ + "{ \"field11\" : \"value11\", \"nested\": {\"message\":\"ok\"} }\n" \ + "{ \"delete\" : { \"_index\" : \"test\", \"_id\" : \"21\" } }\n" \ + "{ \"create\" : { \"_index\" : \"test\", \"_id\" : \"31\" } }\n" \ + "{ \"field11\" : \"value31\", \"field21\" : \"value41\", \"nested\": { \"multiply\": {\"message\": \"ok\"}} }\n" \ + "{ \"update\" : {\"_id\" : \"11\", \"_index\" : \"test\"} }\n" \ + "{ \"doc\" : {\"field21\" : \"value21\"} }\n" \ + "{ \"index\" : { \"_index\" : \"test\", \"_id\" : \"41\" } }\n" \ + "{ \"field41\" : \"value41\", \"nested\": {\"message\": \"ok\"} }\n" diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/log/annotations-exclude/annotations-exclude_default_text.log b/src/fluent-bit/tests/runtime/data/kubernetes/log/annotations-exclude/annotations-exclude_default_text.log new file mode 100644 index 000000000..d5cd3af55 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/log/annotations-exclude/annotations-exclude_default_text.log @@ -0,0 +1,2 @@ +{"log":"Simple text\n","stream":"stdout","time":"2019-04-01T17:58:33.598656444Z"} +{"log":"Simple text\n","stream":"stderr","time":"2019-04-01T17:58:33.698656444Z"} diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/log/annotations-exclude/annotations-exclude_invalid_text.log b/src/fluent-bit/tests/runtime/data/kubernetes/log/annotations-exclude/annotations-exclude_invalid_text.log new file mode 100644 index 000000000..d5cd3af55 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/log/annotations-exclude/annotations-exclude_invalid_text.log @@ -0,0 +1,2 @@ +{"log":"Simple text\n","stream":"stdout","time":"2019-04-01T17:58:33.598656444Z"} +{"log":"Simple text\n","stream":"stderr","time":"2019-04-01T17:58:33.698656444Z"} diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/log/annotations-exclude/annotations-exclude_multiple-1_container-1.log b/src/fluent-bit/tests/runtime/data/kubernetes/log/annotations-exclude/annotations-exclude_multiple-1_container-1.log new file mode 100644 index 000000000..d5cd3af55 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/log/annotations-exclude/annotations-exclude_multiple-1_container-1.log @@ -0,0 +1,2 @@ +{"log":"Simple text\n","stream":"stdout","time":"2019-04-01T17:58:33.598656444Z"} +{"log":"Simple text\n","stream":"stderr","time":"2019-04-01T17:58:33.698656444Z"} diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/log/annotations-exclude/annotations-exclude_multiple-1_container-2.log b/src/fluent-bit/tests/runtime/data/kubernetes/log/annotations-exclude/annotations-exclude_multiple-1_container-2.log new file mode 100644 index 000000000..d5cd3af55 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/log/annotations-exclude/annotations-exclude_multiple-1_container-2.log @@ -0,0 +1,2 @@ +{"log":"Simple text\n","stream":"stdout","time":"2019-04-01T17:58:33.598656444Z"} +{"log":"Simple text\n","stream":"stderr","time":"2019-04-01T17:58:33.698656444Z"} diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/log/annotations-exclude/annotations-exclude_multiple-1_container-3.log b/src/fluent-bit/tests/runtime/data/kubernetes/log/annotations-exclude/annotations-exclude_multiple-1_container-3.log new file mode 100644 index 000000000..d5cd3af55 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/log/annotations-exclude/annotations-exclude_multiple-1_container-3.log @@ -0,0 +1,2 @@ +{"log":"Simple text\n","stream":"stdout","time":"2019-04-01T17:58:33.598656444Z"} +{"log":"Simple text\n","stream":"stderr","time":"2019-04-01T17:58:33.698656444Z"} diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/log/annotations-exclude/annotations-exclude_multiple-1_container-4.log b/src/fluent-bit/tests/runtime/data/kubernetes/log/annotations-exclude/annotations-exclude_multiple-1_container-4.log new file mode 100644 index 000000000..d5cd3af55 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/log/annotations-exclude/annotations-exclude_multiple-1_container-4.log @@ -0,0 +1,2 @@ +{"log":"Simple text\n","stream":"stdout","time":"2019-04-01T17:58:33.598656444Z"} +{"log":"Simple text\n","stream":"stderr","time":"2019-04-01T17:58:33.698656444Z"} diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/log/annotations-exclude/annotations-exclude_multiple-2_container-1.log b/src/fluent-bit/tests/runtime/data/kubernetes/log/annotations-exclude/annotations-exclude_multiple-2_container-1.log new file mode 100644 index 000000000..d5cd3af55 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/log/annotations-exclude/annotations-exclude_multiple-2_container-1.log @@ -0,0 +1,2 @@ +{"log":"Simple text\n","stream":"stdout","time":"2019-04-01T17:58:33.598656444Z"} +{"log":"Simple text\n","stream":"stderr","time":"2019-04-01T17:58:33.698656444Z"} diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/log/annotations-exclude/annotations-exclude_multiple-2_container-2.log b/src/fluent-bit/tests/runtime/data/kubernetes/log/annotations-exclude/annotations-exclude_multiple-2_container-2.log new file mode 100644 index 000000000..d5cd3af55 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/log/annotations-exclude/annotations-exclude_multiple-2_container-2.log @@ -0,0 +1,2 @@ +{"log":"Simple text\n","stream":"stdout","time":"2019-04-01T17:58:33.598656444Z"} +{"log":"Simple text\n","stream":"stderr","time":"2019-04-01T17:58:33.698656444Z"} diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/log/annotations-exclude/annotations-exclude_multiple-2_container-3.log b/src/fluent-bit/tests/runtime/data/kubernetes/log/annotations-exclude/annotations-exclude_multiple-2_container-3.log new file mode 100644 index 000000000..d5cd3af55 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/log/annotations-exclude/annotations-exclude_multiple-2_container-3.log @@ -0,0 +1,2 @@ +{"log":"Simple text\n","stream":"stdout","time":"2019-04-01T17:58:33.598656444Z"} +{"log":"Simple text\n","stream":"stderr","time":"2019-04-01T17:58:33.698656444Z"} diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/log/annotations-exclude/annotations-exclude_multiple-2_container-4.log b/src/fluent-bit/tests/runtime/data/kubernetes/log/annotations-exclude/annotations-exclude_multiple-2_container-4.log new file mode 100644 index 000000000..d5cd3af55 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/log/annotations-exclude/annotations-exclude_multiple-2_container-4.log @@ -0,0 +1,2 @@ +{"log":"Simple text\n","stream":"stdout","time":"2019-04-01T17:58:33.598656444Z"} +{"log":"Simple text\n","stream":"stderr","time":"2019-04-01T17:58:33.698656444Z"} diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/log/annotations-exclude/annotations-exclude_multiple-3_container-1.log b/src/fluent-bit/tests/runtime/data/kubernetes/log/annotations-exclude/annotations-exclude_multiple-3_container-1.log new file mode 100644 index 000000000..d5cd3af55 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/log/annotations-exclude/annotations-exclude_multiple-3_container-1.log @@ -0,0 +1,2 @@ +{"log":"Simple text\n","stream":"stdout","time":"2019-04-01T17:58:33.598656444Z"} +{"log":"Simple text\n","stream":"stderr","time":"2019-04-01T17:58:33.698656444Z"} diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/log/annotations-exclude/annotations-exclude_multiple-3_container-2.log b/src/fluent-bit/tests/runtime/data/kubernetes/log/annotations-exclude/annotations-exclude_multiple-3_container-2.log new file mode 100644 index 000000000..d5cd3af55 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/log/annotations-exclude/annotations-exclude_multiple-3_container-2.log @@ -0,0 +1,2 @@ +{"log":"Simple text\n","stream":"stdout","time":"2019-04-01T17:58:33.598656444Z"} +{"log":"Simple text\n","stream":"stderr","time":"2019-04-01T17:58:33.698656444Z"} diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/log/annotations-exclude/annotations-exclude_multiple-3_container-3.log b/src/fluent-bit/tests/runtime/data/kubernetes/log/annotations-exclude/annotations-exclude_multiple-3_container-3.log new file mode 100644 index 000000000..d5cd3af55 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/log/annotations-exclude/annotations-exclude_multiple-3_container-3.log @@ -0,0 +1,2 @@ +{"log":"Simple text\n","stream":"stdout","time":"2019-04-01T17:58:33.598656444Z"} +{"log":"Simple text\n","stream":"stderr","time":"2019-04-01T17:58:33.698656444Z"} diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/log/annotations-exclude/annotations-exclude_multiple-3_container-4.log b/src/fluent-bit/tests/runtime/data/kubernetes/log/annotations-exclude/annotations-exclude_multiple-3_container-4.log new file mode 100644 index 000000000..d5cd3af55 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/log/annotations-exclude/annotations-exclude_multiple-3_container-4.log @@ -0,0 +1,2 @@ +{"log":"Simple text\n","stream":"stdout","time":"2019-04-01T17:58:33.598656444Z"} +{"log":"Simple text\n","stream":"stderr","time":"2019-04-01T17:58:33.698656444Z"} diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/log/annotations-exclude/annotations-exclude_multiple-4_container-1.log b/src/fluent-bit/tests/runtime/data/kubernetes/log/annotations-exclude/annotations-exclude_multiple-4_container-1.log new file mode 100644 index 000000000..d5cd3af55 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/log/annotations-exclude/annotations-exclude_multiple-4_container-1.log @@ -0,0 +1,2 @@ +{"log":"Simple text\n","stream":"stdout","time":"2019-04-01T17:58:33.598656444Z"} +{"log":"Simple text\n","stream":"stderr","time":"2019-04-01T17:58:33.698656444Z"} diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/log/annotations-exclude/annotations-exclude_multiple-4_container-2.log b/src/fluent-bit/tests/runtime/data/kubernetes/log/annotations-exclude/annotations-exclude_multiple-4_container-2.log new file mode 100644 index 000000000..d5cd3af55 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/log/annotations-exclude/annotations-exclude_multiple-4_container-2.log @@ -0,0 +1,2 @@ +{"log":"Simple text\n","stream":"stdout","time":"2019-04-01T17:58:33.598656444Z"} +{"log":"Simple text\n","stream":"stderr","time":"2019-04-01T17:58:33.698656444Z"} diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/log/annotations-exclude/annotations-exclude_multiple-4_container-3.log b/src/fluent-bit/tests/runtime/data/kubernetes/log/annotations-exclude/annotations-exclude_multiple-4_container-3.log new file mode 100644 index 000000000..d5cd3af55 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/log/annotations-exclude/annotations-exclude_multiple-4_container-3.log @@ -0,0 +1,2 @@ +{"log":"Simple text\n","stream":"stdout","time":"2019-04-01T17:58:33.598656444Z"} +{"log":"Simple text\n","stream":"stderr","time":"2019-04-01T17:58:33.698656444Z"} diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/log/annotations-exclude/annotations-exclude_multiple-4_container-4.log b/src/fluent-bit/tests/runtime/data/kubernetes/log/annotations-exclude/annotations-exclude_multiple-4_container-4.log new file mode 100644 index 000000000..d5cd3af55 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/log/annotations-exclude/annotations-exclude_multiple-4_container-4.log @@ -0,0 +1,2 @@ +{"log":"Simple text\n","stream":"stdout","time":"2019-04-01T17:58:33.598656444Z"} +{"log":"Simple text\n","stream":"stderr","time":"2019-04-01T17:58:33.698656444Z"} diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/log/annotations-exclude/annotations-exclude_stderr_text.log b/src/fluent-bit/tests/runtime/data/kubernetes/log/annotations-exclude/annotations-exclude_stderr_text.log new file mode 100644 index 000000000..a857f5ab3 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/log/annotations-exclude/annotations-exclude_stderr_text.log @@ -0,0 +1,2 @@ +{"log":"Simple text\n","stream":"stderr","time":"2019-04-01T17:58:33.698656444Z"} +{"log":"Simple text\n","stream":"stdout","time":"2019-04-01T17:58:33.598656444Z"} diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/log/annotations-exclude/annotations-exclude_stdout_text.log b/src/fluent-bit/tests/runtime/data/kubernetes/log/annotations-exclude/annotations-exclude_stdout_text.log new file mode 100644 index 000000000..d5cd3af55 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/log/annotations-exclude/annotations-exclude_stdout_text.log @@ -0,0 +1,2 @@ +{"log":"Simple text\n","stream":"stdout","time":"2019-04-01T17:58:33.598656444Z"} +{"log":"Simple text\n","stream":"stderr","time":"2019-04-01T17:58:33.698656444Z"} diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/log/annotations-parser/annotations-parser_invalid_text.log b/src/fluent-bit/tests/runtime/data/kubernetes/log/annotations-parser/annotations-parser_invalid_text.log new file mode 100644 index 000000000..a857f5ab3 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/log/annotations-parser/annotations-parser_invalid_text.log @@ -0,0 +1,2 @@ +{"log":"Simple text\n","stream":"stderr","time":"2019-04-01T17:58:33.698656444Z"} +{"log":"Simple text\n","stream":"stdout","time":"2019-04-01T17:58:33.598656444Z"} diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/log/annotations-parser/annotations-parser_json-with-time_invalid-json-1.log b/src/fluent-bit/tests/runtime/data/kubernetes/log/annotations-parser/annotations-parser_json-with-time_invalid-json-1.log new file mode 100644 index 000000000..65d777867 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/log/annotations-parser/annotations-parser_json-with-time_invalid-json-1.log @@ -0,0 +1 @@ +{"log":"{\"time\":\"2019-04-05T14:54:23.584269761\",\"message\":\"Simple text\"]\n","stream":"stdout","time":"2019-04-01T17:58:33.598656444Z"} diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/log/annotations-parser/annotations-parser_json-with-time_json.log b/src/fluent-bit/tests/runtime/data/kubernetes/log/annotations-parser/annotations-parser_json-with-time_json.log new file mode 100644 index 000000000..d9dc4a391 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/log/annotations-parser/annotations-parser_json-with-time_json.log @@ -0,0 +1 @@ +{"log":"{\"time\":\"2019-04-05T14:54:23.584269761\",\"message\":\"Simple text\"}\n","stream":"stdout","time":"2019-04-01T17:58:33.598656444Z"} diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/log/annotations-parser/annotations-parser_multiple-1_container-1.log b/src/fluent-bit/tests/runtime/data/kubernetes/log/annotations-parser/annotations-parser_multiple-1_container-1.log new file mode 100644 index 000000000..db4b14a87 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/log/annotations-parser/annotations-parser_multiple-1_container-1.log @@ -0,0 +1,2 @@ +{"log":"[2019-04-05T14:54:23.584269761] Container 1 is logging on stdout\n","stream":"stdout","time":"2019-04-01T17:58:33.598656444Z"} +{"log":"[2019-04-05T14:54:23.584269761] Container 1 is logging on stderr\n","stream":"stderr","time":"2019-04-01T17:58:33.698656444Z"} diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/log/annotations-parser/annotations-parser_multiple-1_container-2.log b/src/fluent-bit/tests/runtime/data/kubernetes/log/annotations-parser/annotations-parser_multiple-1_container-2.log new file mode 100644 index 000000000..d2f01a8bc --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/log/annotations-parser/annotations-parser_multiple-1_container-2.log @@ -0,0 +1,2 @@ +{"log":"[2019-04-05T14:54:23.584269761] Container 2 is logging on stdout\n","stream":"stdout","time":"2019-04-01T17:58:33.598656444Z"} +{"log":"[2019-04-05T14:54:23.584269761] Container 2 is logging on stderr\n","stream":"stderr","time":"2019-04-01T17:58:33.698656444Z"} diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/log/annotations-parser/annotations-parser_multiple-1_container-3.log b/src/fluent-bit/tests/runtime/data/kubernetes/log/annotations-parser/annotations-parser_multiple-1_container-3.log new file mode 100644 index 000000000..edee75aa7 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/log/annotations-parser/annotations-parser_multiple-1_container-3.log @@ -0,0 +1,2 @@ +{"log":"[2019-04-05T14:54:23.584269761] Container 3 is logging on stdout\n","stream":"stdout","time":"2019-04-01T17:58:33.598656444Z"} +{"log":"[2019-04-05T14:54:23.584269761] Container 3 is logging on stderr\n","stream":"stderr","time":"2019-04-01T17:58:33.698656444Z"} diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/log/annotations-parser/annotations-parser_multiple-1_container-4.log b/src/fluent-bit/tests/runtime/data/kubernetes/log/annotations-parser/annotations-parser_multiple-1_container-4.log new file mode 100644 index 000000000..48e1dabd1 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/log/annotations-parser/annotations-parser_multiple-1_container-4.log @@ -0,0 +1,2 @@ +{"log":"[2019-04-05T14:54:23.584269761] Container 4 is logging on stdout\n","stream":"stdout","time":"2019-04-01T17:58:33.598656444Z"} +{"log":"[2019-04-05T14:54:23.584269761] Container 4 is logging on stderr\n","stream":"stderr","time":"2019-04-01T17:58:33.698656444Z"} diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/log/annotations-parser/annotations-parser_multiple-1_container-5.log b/src/fluent-bit/tests/runtime/data/kubernetes/log/annotations-parser/annotations-parser_multiple-1_container-5.log new file mode 100644 index 000000000..81e67c5f7 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/log/annotations-parser/annotations-parser_multiple-1_container-5.log @@ -0,0 +1,2 @@ +{"log":"[2019-04-05T14:54:23.584269761] Container 5 is logging on stdout\n","stream":"stdout","time":"2019-04-01T17:58:33.598656444Z"} +{"log":"[2019-04-05T14:54:23.584269761] Container 5 is logging on stderr\n","stream":"stderr","time":"2019-04-01T17:58:33.698656444Z"} diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/log/annotations-parser/annotations-parser_multiple-2_container-1.log b/src/fluent-bit/tests/runtime/data/kubernetes/log/annotations-parser/annotations-parser_multiple-2_container-1.log new file mode 100644 index 000000000..db4b14a87 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/log/annotations-parser/annotations-parser_multiple-2_container-1.log @@ -0,0 +1,2 @@ +{"log":"[2019-04-05T14:54:23.584269761] Container 1 is logging on stdout\n","stream":"stdout","time":"2019-04-01T17:58:33.598656444Z"} +{"log":"[2019-04-05T14:54:23.584269761] Container 1 is logging on stderr\n","stream":"stderr","time":"2019-04-01T17:58:33.698656444Z"} diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/log/annotations-parser/annotations-parser_multiple-2_container-2.log b/src/fluent-bit/tests/runtime/data/kubernetes/log/annotations-parser/annotations-parser_multiple-2_container-2.log new file mode 100644 index 000000000..d2f01a8bc --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/log/annotations-parser/annotations-parser_multiple-2_container-2.log @@ -0,0 +1,2 @@ +{"log":"[2019-04-05T14:54:23.584269761] Container 2 is logging on stdout\n","stream":"stdout","time":"2019-04-01T17:58:33.598656444Z"} +{"log":"[2019-04-05T14:54:23.584269761] Container 2 is logging on stderr\n","stream":"stderr","time":"2019-04-01T17:58:33.698656444Z"} diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/log/annotations-parser/annotations-parser_multiple-2_container-3.log b/src/fluent-bit/tests/runtime/data/kubernetes/log/annotations-parser/annotations-parser_multiple-2_container-3.log new file mode 100644 index 000000000..edee75aa7 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/log/annotations-parser/annotations-parser_multiple-2_container-3.log @@ -0,0 +1,2 @@ +{"log":"[2019-04-05T14:54:23.584269761] Container 3 is logging on stdout\n","stream":"stdout","time":"2019-04-01T17:58:33.598656444Z"} +{"log":"[2019-04-05T14:54:23.584269761] Container 3 is logging on stderr\n","stream":"stderr","time":"2019-04-01T17:58:33.698656444Z"} diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/log/annotations-parser/annotations-parser_multiple-2_container-4.log b/src/fluent-bit/tests/runtime/data/kubernetes/log/annotations-parser/annotations-parser_multiple-2_container-4.log new file mode 100644 index 000000000..48e1dabd1 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/log/annotations-parser/annotations-parser_multiple-2_container-4.log @@ -0,0 +1,2 @@ +{"log":"[2019-04-05T14:54:23.584269761] Container 4 is logging on stdout\n","stream":"stdout","time":"2019-04-01T17:58:33.598656444Z"} +{"log":"[2019-04-05T14:54:23.584269761] Container 4 is logging on stderr\n","stream":"stderr","time":"2019-04-01T17:58:33.698656444Z"} diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/log/annotations-parser/annotations-parser_multiple-2_container-5.log b/src/fluent-bit/tests/runtime/data/kubernetes/log/annotations-parser/annotations-parser_multiple-2_container-5.log new file mode 100644 index 000000000..81e67c5f7 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/log/annotations-parser/annotations-parser_multiple-2_container-5.log @@ -0,0 +1,2 @@ +{"log":"[2019-04-05T14:54:23.584269761] Container 5 is logging on stdout\n","stream":"stdout","time":"2019-04-01T17:58:33.598656444Z"} +{"log":"[2019-04-05T14:54:23.584269761] Container 5 is logging on stderr\n","stream":"stderr","time":"2019-04-01T17:58:33.698656444Z"} diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/log/annotations-parser/annotations-parser_regex-with-time_invalid-text-1.log b/src/fluent-bit/tests/runtime/data/kubernetes/log/annotations-parser/annotations-parser_regex-with-time_invalid-text-1.log new file mode 100644 index 000000000..8e87678b2 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/log/annotations-parser/annotations-parser_regex-with-time_invalid-text-1.log @@ -0,0 +1 @@ +{"log":"2019-04-05T14:54:23.584269761 - Simple text\n","stream":"stdout","time":"2019-04-01T17:58:33.598656444Z"} diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/log/annotations-parser/annotations-parser_regex-with-time_text.log b/src/fluent-bit/tests/runtime/data/kubernetes/log/annotations-parser/annotations-parser_regex-with-time_text.log new file mode 100644 index 000000000..6e3348354 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/log/annotations-parser/annotations-parser_regex-with-time_text.log @@ -0,0 +1 @@ +{"log":"[2019-04-05T14:54:23.584269761] Simple text\n","stream":"stdout","time":"2019-04-01T17:58:33.598656444Z"} diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/log/annotations-parser/annotations-parser_stderr_text.log b/src/fluent-bit/tests/runtime/data/kubernetes/log/annotations-parser/annotations-parser_stderr_text.log new file mode 100644 index 000000000..c6e1e05f9 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/log/annotations-parser/annotations-parser_stderr_text.log @@ -0,0 +1,2 @@ +{"log":"[2019-04-05T14:54:23.584269761] Simple text\n","stream":"stderr","time":"2019-04-01T17:58:33.698656444Z"} +{"log":"[2019-04-05T14:54:23.584269761] Simple text\n","stream":"stdout","time":"2019-04-01T17:58:33.598656444Z"} diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/log/annotations-parser/annotations-parser_stdout_text.log b/src/fluent-bit/tests/runtime/data/kubernetes/log/annotations-parser/annotations-parser_stdout_text.log new file mode 100644 index 000000000..c47aa0007 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/log/annotations-parser/annotations-parser_stdout_text.log @@ -0,0 +1,2 @@ +{"log":"[2019-04-05T14:54:23.584269761] Simple text\n","stream":"stdout","time":"2019-04-01T17:58:33.598656444Z"} +{"log":"[2019-04-05T14:54:23.584269761] Simple text\n","stream":"stderr","time":"2019-04-01T17:58:33.698656444Z"} diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/log/annotations/annotations_invalid_text.log b/src/fluent-bit/tests/runtime/data/kubernetes/log/annotations/annotations_invalid_text.log new file mode 100644 index 000000000..59ad227ff --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/log/annotations/annotations_invalid_text.log @@ -0,0 +1 @@ +{"log":"Simple text\n","stream":"stdout","time":"2019-04-01T17:58:33.598656444Z"} diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/log/core/core_base_fluent-bit.log b/src/fluent-bit/tests/runtime/data/kubernetes/log/core/core_base_fluent-bit.log new file mode 100644 index 000000000..259723131 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/log/core/core_base_fluent-bit.log @@ -0,0 +1 @@ +{"log":"Fluent Bit is logging\n","stream":"stdout","time":"2019-04-01T17:58:33.598656444Z"} diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/log/core/core_no-meta_text.log b/src/fluent-bit/tests/runtime/data/kubernetes/log/core/core_no-meta_text.log new file mode 100644 index 000000000..59ad227ff --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/log/core/core_no-meta_text.log @@ -0,0 +1 @@ +{"log":"Simple text\n","stream":"stdout","time":"2019-04-01T17:58:33.598656444Z"} diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/log/core/core_unescaping_json.log b/src/fluent-bit/tests/runtime/data/kubernetes/log/core/core_unescaping_json.log new file mode 100644 index 000000000..0cf6e4416 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/log/core/core_unescaping_json.log @@ -0,0 +1 @@ +{"log":"{\"stringified-json\":\"{\\\"stringified-json\\\":\\\"{\\\\\\\"stringified-json\\\\\\\":\\\\\\\"\\\\\\\"}\\\"}\",\"text\":\"quoted \\\"text with embedded quoted \\\\\\\"text\\\\\\\"\\\"\"}\n","stream":"stdout","time":"2019-04-01T17:58:33.598656444Z"} diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/log/core/core_unescaping_text.log b/src/fluent-bit/tests/runtime/data/kubernetes/log/core/core_unescaping_text.log new file mode 100644 index 000000000..55c673683 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/log/core/core_unescaping_text.log @@ -0,0 +1 @@ +{"log":"Text with embedded quoted \"text with embedded quoted \\\"text with embedded quoted \\\\\\\"text\\\\\\\"\\\"\"\n","stream":"stdout","time":"2019-04-01T17:58:33.598656444Z"} diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/log/options/options_k8s-logging-exclude-disabled_text.log b/src/fluent-bit/tests/runtime/data/kubernetes/log/options/options_k8s-logging-exclude-disabled_text.log new file mode 100644 index 000000000..d5cd3af55 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/log/options/options_k8s-logging-exclude-disabled_text.log @@ -0,0 +1,2 @@ +{"log":"Simple text\n","stream":"stdout","time":"2019-04-01T17:58:33.598656444Z"} +{"log":"Simple text\n","stream":"stderr","time":"2019-04-01T17:58:33.698656444Z"} diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/log/options/options_k8s-logging-parser-disabled_text.log b/src/fluent-bit/tests/runtime/data/kubernetes/log/options/options_k8s-logging-parser-disabled_text.log new file mode 100644 index 000000000..d5cd3af55 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/log/options/options_k8s-logging-parser-disabled_text.log @@ -0,0 +1,2 @@ +{"log":"Simple text\n","stream":"stdout","time":"2019-04-01T17:58:33.598656444Z"} +{"log":"Simple text\n","stream":"stderr","time":"2019-04-01T17:58:33.698656444Z"} diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/log/options/options_keep-log-disabled_json.log b/src/fluent-bit/tests/runtime/data/kubernetes/log/options/options_keep-log-disabled_json.log new file mode 100644 index 000000000..e1410b0f4 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/log/options/options_keep-log-disabled_json.log @@ -0,0 +1 @@ +{"log":"{\"text\":\"Simple text\"}\n","stream":"stdout","time":"2019-04-01T17:58:33.598656444Z"} diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/log/options/options_keep-log-enabled_json.log b/src/fluent-bit/tests/runtime/data/kubernetes/log/options/options_keep-log-enabled_json.log new file mode 100644 index 000000000..e1410b0f4 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/log/options/options_keep-log-enabled_json.log @@ -0,0 +1 @@ +{"log":"{\"text\":\"Simple text\"}\n","stream":"stdout","time":"2019-04-01T17:58:33.598656444Z"} diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/log/options/options_merge-log-disabled_json.log b/src/fluent-bit/tests/runtime/data/kubernetes/log/options/options_merge-log-disabled_json.log new file mode 100644 index 000000000..e1410b0f4 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/log/options/options_merge-log-disabled_json.log @@ -0,0 +1 @@ +{"log":"{\"text\":\"Simple text\"}\n","stream":"stdout","time":"2019-04-01T17:58:33.598656444Z"} diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/log/options/options_merge-log-enabled_invalid-json.log b/src/fluent-bit/tests/runtime/data/kubernetes/log/options/options_merge-log-enabled_invalid-json.log new file mode 100644 index 000000000..9b4e0dcb5 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/log/options/options_merge-log-enabled_invalid-json.log @@ -0,0 +1 @@ +{"log":"{\"text\":\"Simple text\"]\n","stream":"stdout","time":"2019-04-01T17:58:33.598656444Z"} diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/log/options/options_merge-log-enabled_json.log b/src/fluent-bit/tests/runtime/data/kubernetes/log/options/options_merge-log-enabled_json.log new file mode 100644 index 000000000..e1410b0f4 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/log/options/options_merge-log-enabled_json.log @@ -0,0 +1 @@ +{"log":"{\"text\":\"Simple text\"}\n","stream":"stdout","time":"2019-04-01T17:58:33.598656444Z"} diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/log/options/options_merge-log-enabled_text.log b/src/fluent-bit/tests/runtime/data/kubernetes/log/options/options_merge-log-enabled_text.log new file mode 100644 index 000000000..59ad227ff --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/log/options/options_merge-log-enabled_text.log @@ -0,0 +1 @@ +{"log":"Simple text\n","stream":"stdout","time":"2019-04-01T17:58:33.598656444Z"} diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/log/options/options_merge-log-key_json.log b/src/fluent-bit/tests/runtime/data/kubernetes/log/options/options_merge-log-key_json.log new file mode 100644 index 000000000..e1410b0f4 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/log/options/options_merge-log-key_json.log @@ -0,0 +1 @@ +{"log":"{\"text\":\"Simple text\"}\n","stream":"stdout","time":"2019-04-01T17:58:33.598656444Z"} diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/log/options/options_merge-log-trim-disabled_json.log b/src/fluent-bit/tests/runtime/data/kubernetes/log/options/options_merge-log-trim-disabled_json.log new file mode 100644 index 000000000..6dc59dc1a --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/log/options/options_merge-log-trim-disabled_json.log @@ -0,0 +1 @@ +{"log":"{\"text\":\"Simple text\n\"}\n","stream":"stdout","time":"2019-04-01T17:58:33.598656444Z"} diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/log/options/options_merge-log-trim-enabled_json.log b/src/fluent-bit/tests/runtime/data/kubernetes/log/options/options_merge-log-trim-enabled_json.log new file mode 100644 index 000000000..6dc59dc1a --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/log/options/options_merge-log-trim-enabled_json.log @@ -0,0 +1 @@ +{"log":"{\"text\":\"Simple text\n\"}\n","stream":"stdout","time":"2019-04-01T17:58:33.598656444Z"} diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/log/options/options_use-kubelet-disabled_fluent-bit.log b/src/fluent-bit/tests/runtime/data/kubernetes/log/options/options_use-kubelet-disabled_fluent-bit.log new file mode 100644 index 000000000..259723131 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/log/options/options_use-kubelet-disabled_fluent-bit.log @@ -0,0 +1 @@ +{"log":"Fluent Bit is logging\n","stream":"stdout","time":"2019-04-01T17:58:33.598656444Z"} diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/log/options/options_use-kubelet-enabled_fluent-bit.log b/src/fluent-bit/tests/runtime/data/kubernetes/log/options/options_use-kubelet-enabled_fluent-bit.log new file mode 100644 index 000000000..259723131 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/log/options/options_use-kubelet-enabled_fluent-bit.log @@ -0,0 +1 @@ +{"log":"Fluent Bit is logging\n","stream":"stdout","time":"2019-04-01T17:58:33.598656444Z"} diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/meta/annotations-exclude_default.meta b/src/fluent-bit/tests/runtime/data/kubernetes/meta/annotations-exclude_default.meta new file mode 100644 index 000000000..e986f1646 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/meta/annotations-exclude_default.meta @@ -0,0 +1,7 @@ +{ + "metadata": { + "annotations": { + "fluentbit.io/exclude": "true" + } + } +} diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/meta/annotations-exclude_invalid.meta b/src/fluent-bit/tests/runtime/data/kubernetes/meta/annotations-exclude_invalid.meta new file mode 100644 index 000000000..9f529a42e --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/meta/annotations-exclude_invalid.meta @@ -0,0 +1,7 @@ +{ + "metadata": { + "annotations": { + "fluentbit.io/exclude": "neither-true-nor-false" + } + } +} diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/meta/annotations-exclude_multiple-1.meta b/src/fluent-bit/tests/runtime/data/kubernetes/meta/annotations-exclude_multiple-1.meta new file mode 100644 index 000000000..137d88e64 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/meta/annotations-exclude_multiple-1.meta @@ -0,0 +1,9 @@ +{ + "metadata": { + "annotations": { + "fluentbit.io/exclude-container-1": "true", + "fluentbit.io/exclude_stdout-container-2": "true", + "fluentbit.io/exclude_stderr-container-3": "true" + } + } +} diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/meta/annotations-exclude_multiple-2.meta b/src/fluent-bit/tests/runtime/data/kubernetes/meta/annotations-exclude_multiple-2.meta new file mode 100644 index 000000000..f9d1ded2f --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/meta/annotations-exclude_multiple-2.meta @@ -0,0 +1,10 @@ +{ + "metadata": { + "annotations": { + "fluentbit.io/exclude": "true", + "fluentbit.io/exclude_stderr-container-2": "false", + "fluentbit.io/exclude_stdout-container-3": "false", + "fluentbit.io/exclude-container-4": "false" + } + } +} diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/meta/annotations-exclude_multiple-3.meta b/src/fluent-bit/tests/runtime/data/kubernetes/meta/annotations-exclude_multiple-3.meta new file mode 100644 index 000000000..bfcbb44bd --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/meta/annotations-exclude_multiple-3.meta @@ -0,0 +1,11 @@ +{ + "metadata": { + "annotations": { + "fluentbit.io/exclude_stdout": "true", + "fluentbit.io/exclude_stderr-container-1": "true", + "fluentbit.io/exclude_stdout-container-3": "false", + "fluentbit.io/exclude_stderr-container-3": "true", + "fluentbit.io/exclude_stdout-container-4": "false" + } + } +} diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/meta/annotations-exclude_multiple-4.meta b/src/fluent-bit/tests/runtime/data/kubernetes/meta/annotations-exclude_multiple-4.meta new file mode 100644 index 000000000..1295d0ed3 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/meta/annotations-exclude_multiple-4.meta @@ -0,0 +1,11 @@ +{ + "metadata": { + "annotations": { + "fluentbit.io/exclude_stderr": "true", + "fluentbit.io/exclude_stdout-container-1": "true", + "fluentbit.io/exclude_stdout-container-2": "true", + "fluentbit.io/exclude_stderr-container-2": "false", + "fluentbit.io/exclude_stderr-container-4": "false" + } + } +} diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/meta/annotations-exclude_stderr.meta b/src/fluent-bit/tests/runtime/data/kubernetes/meta/annotations-exclude_stderr.meta new file mode 100644 index 000000000..b57831f53 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/meta/annotations-exclude_stderr.meta @@ -0,0 +1,7 @@ +{ + "metadata": { + "annotations": { + "fluentbit.io/exclude_stderr": "true" + } + } +} diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/meta/annotations-exclude_stdout.meta b/src/fluent-bit/tests/runtime/data/kubernetes/meta/annotations-exclude_stdout.meta new file mode 100644 index 000000000..db187f2e3 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/meta/annotations-exclude_stdout.meta @@ -0,0 +1,7 @@ +{ + "metadata": { + "annotations": { + "fluentbit.io/exclude_stdout": "true" + } + } +} diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/meta/annotations-parser_invalid.meta b/src/fluent-bit/tests/runtime/data/kubernetes/meta/annotations-parser_invalid.meta new file mode 100644 index 000000000..676f20630 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/meta/annotations-parser_invalid.meta @@ -0,0 +1,7 @@ +{ + "metadata": { + "annotations": { + "fluentbit.io/parser": "invalid-parser" + } + } +} diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/meta/annotations-parser_json-with-time.meta b/src/fluent-bit/tests/runtime/data/kubernetes/meta/annotations-parser_json-with-time.meta new file mode 100644 index 000000000..9bdba6854 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/meta/annotations-parser_json-with-time.meta @@ -0,0 +1,7 @@ +{ + "metadata": { + "annotations": { + "fluentbit.io/parser": "json-parser-with-time" + } + } +} diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/meta/annotations-parser_json-without-time.meta b/src/fluent-bit/tests/runtime/data/kubernetes/meta/annotations-parser_json-without-time.meta new file mode 100644 index 000000000..2195ae0dc --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/meta/annotations-parser_json-without-time.meta @@ -0,0 +1,7 @@ +{ + "metadata": { + "annotations": { + "fluentbit.io/parser": "json-parser-without-time" + } + } +} diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/meta/annotations-parser_multiple-1.meta b/src/fluent-bit/tests/runtime/data/kubernetes/meta/annotations-parser_multiple-1.meta new file mode 100644 index 000000000..51420a872 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/meta/annotations-parser_multiple-1.meta @@ -0,0 +1,12 @@ +{ + "metadata": { + "annotations": { + "fluentbit.io/parser-container-1": "container-1-parser", + "fluentbit.io/parser_stdout-container-2": "container-2-stdout-parser", + "fluentbit.io/parser_stderr-container-2": "container-2-stderr-parser", + "fluentbit.io/parser_stdout-container-3": "container-3-stdout-parser", + "fluentbit.io/parser_stderr-container-4": "container-4-stderr-parser", + "fluentbit.io/parser": "default-parser" + } + } +} diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/meta/annotations-parser_multiple-2.meta b/src/fluent-bit/tests/runtime/data/kubernetes/meta/annotations-parser_multiple-2.meta new file mode 100644 index 000000000..b3bb2bdc0 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/meta/annotations-parser_multiple-2.meta @@ -0,0 +1,13 @@ +{ + "metadata": { + "annotations": { + "fluentbit.io/parser-container-1": "container-1-parser", + "fluentbit.io/parser_stdout-container-2": "container-2-stdout-parser", + "fluentbit.io/parser_stderr-container-2": "container-2-stderr-parser", + "fluentbit.io/parser_stdout-container-3": "container-3-stdout-parser", + "fluentbit.io/parser_stderr-container-4": "container-4-stderr-parser", + "fluentbit.io/parser_stdout": "default-stdout-parser", + "fluentbit.io/parser_stderr": "default-stderr-parser" + } + } +} diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/meta/annotations-parser_regex-with-time.meta b/src/fluent-bit/tests/runtime/data/kubernetes/meta/annotations-parser_regex-with-time.meta new file mode 100644 index 000000000..e098a81d7 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/meta/annotations-parser_regex-with-time.meta @@ -0,0 +1,7 @@ +{ + "metadata": { + "annotations": { + "fluentbit.io/parser": "regex-parser-with-time" + } + } +} diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/meta/annotations-parser_regex-without-time.meta b/src/fluent-bit/tests/runtime/data/kubernetes/meta/annotations-parser_regex-without-time.meta new file mode 100644 index 000000000..8a43354b7 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/meta/annotations-parser_regex-without-time.meta @@ -0,0 +1,7 @@ +{ + "metadata": { + "annotations": { + "fluentbit.io/parser": "regex-parser-without-time" + } + } +} diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/meta/annotations-parser_stderr.meta b/src/fluent-bit/tests/runtime/data/kubernetes/meta/annotations-parser_stderr.meta new file mode 100644 index 000000000..6a8692765 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/meta/annotations-parser_stderr.meta @@ -0,0 +1,7 @@ +{ + "metadata": { + "annotations": { + "fluentbit.io/parser_stderr": "default-stderr-parser" + } + } +} diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/meta/annotations-parser_stdout.meta b/src/fluent-bit/tests/runtime/data/kubernetes/meta/annotations-parser_stdout.meta new file mode 100644 index 000000000..1b9fd148c --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/meta/annotations-parser_stdout.meta @@ -0,0 +1,7 @@ +{ + "metadata": { + "annotations": { + "fluentbit.io/parser_stdout": "default-stdout-parser" + } + } +} diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/meta/annotations_invalid.meta b/src/fluent-bit/tests/runtime/data/kubernetes/meta/annotations_invalid.meta new file mode 100644 index 000000000..a1bc934a7 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/meta/annotations_invalid.meta @@ -0,0 +1,12 @@ +{ + "metadata": { + "annotations": { + "fluentbit.io/invalid": "true", + "fluentbit.io/exclude_": "true", + "fluentbit.io/exclude_invalid": "true", + "fluentbit.io/exclude_stdout-": "true", + "fluentbit.io/exclude-": "true", + "fluentbit.io/exclude+": "true" + } + } +} diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/meta/core_base.meta b/src/fluent-bit/tests/runtime/data/kubernetes/meta/core_base.meta new file mode 100644 index 000000000..3ccca4435 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/meta/core_base.meta @@ -0,0 +1,116 @@ +{ + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "annotations": { + "prometheus.io/path": "/api/v1/metrics/prometheus", + "prometheus.io/port": "2020", + "prometheus.io/scrape": "true" + }, + "creationTimestamp": "2019-04-03T09:29:00Z", + "labels": { + "app.kubernetes.io/name": "fluent-bit" + }, + "name": "base", + "namespace": "core", + "resourceVersion": "74466568", + "selfLink": "/api/v1/namespaces/core/pods/base", + "uid": "e9f2963f-55f2-11e9-84c5-02e422b8a84a" + }, + "spec": { + "containers": [ + { + "image": "fluent/fluent-bit", + "imagePullPolicy": "Always", + "name": "fluent-bit", + "resources": {}, + "stdin": true, + "stdinOnce": true, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "tty": true, + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "default-token-9ffht", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirst", + "nodeName": "ip-10-49-18-80.eu-west-1.compute.internal", + "restartPolicy": "Never", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "default", + "serviceAccountName": "default", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists", + "tolerationSeconds": 300 + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists", + "tolerationSeconds": 300 + } + ], + "volumes": [ + { + "name": "default-token-9ffht", + "secret": { + "defaultMode": 420, + "secretName": "default-token-9ffht" + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2019-04-03T09:29:00Z", + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2019-04-03T09:29:06Z", + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2019-04-03T09:29:00Z", + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "containerID": "docker://c9898099f6d235126d564ed38a020007ea7a6fac6e25e718de683c9dd0076c16", + "image": "fluent/fluent-bit:latest", + "imageID": "docker-pullable://fluent/fluent-bit@sha256:7ac0fd3569af866e9a6a22eb592744200d2dbe098cf066162453f8d0b06c531f", + "lastState": {}, + "name": "fluent-bit", + "ready": true, + "restartCount": 0, + "state": { + "running": { + "startedAt": "2019-04-03T09:29:05Z" + } + } + } + ], + "hostIP": "10.49.18.80", + "phase": "Running", + "podIP": "100.116.192.42", + "qosClass": "BestEffort", + "startTime": "2019-04-03T09:29:00Z" + } +} diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/meta/core_no-meta.meta b/src/fluent-bit/tests/runtime/data/kubernetes/meta/core_no-meta.meta new file mode 100644 index 000000000..0967ef424 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/meta/core_no-meta.meta @@ -0,0 +1 @@ +{} diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/meta/core_unescaping.meta b/src/fluent-bit/tests/runtime/data/kubernetes/meta/core_unescaping.meta new file mode 100644 index 000000000..4e9f5aa3e --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/meta/core_unescaping.meta @@ -0,0 +1,4 @@ +{ + "metadata": { + } +} diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/meta/default_kairosdb-914055854-b63vq.meta b/src/fluent-bit/tests/runtime/data/kubernetes/meta/default_kairosdb-914055854-b63vq.meta new file mode 100644 index 000000000..90109bfec --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/meta/default_kairosdb-914055854-b63vq.meta @@ -0,0 +1,155 @@ +{
+ "kind": "Pod",
+ "apiVersion": "v1",
+ "metadata": {
+ "name": "kairosdb-914055854-b63vq",
+ "generateName": "kairosdb-914055854-",
+ "namespace": "default",
+ "selfLink": "/api/v1/namespaces/default/pods/kairosdb-914055854-b63vq",
+ "uid": "d6c53deb-05a4-11e8-a8c4-080027435fb7",
+ "resourceVersion": "25205",
+ "creationTimestamp": "2018-01-30T10:03:37Z",
+ "labels": {
+ "name": "kairosdb",
+ "pod-template-hash": "914055854"
+ },
+ "annotations": {
+ "kubernetes.io/created-by": "{\"kind\":\"SerializedReference\",\"apiVersion\":\"v1\",\"reference\":{\"kind\":\"ReplicaSet\",\"namespace\":\"default\",\"name\":\"kairosdb-914055854\",\"uid\":\"d6c2f841-05a4-11e8-a8c4-080027435fb7\",\"apiVersion\":\"extensions\",\"resourceVersion\":\"1592\"}}\n"
+ },
+ "ownerReferences": [
+ {
+ "apiVersion": "extensions/v1beta1",
+ "kind": "ReplicaSet",
+ "name": "kairosdb-914055854",
+ "uid": "d6c2f841-05a4-11e8-a8c4-080027435fb7",
+ "controller": true,
+ "blockOwnerDeletion": true
+ }
+ ]
+ },
+ "spec": {
+ "volumes": [
+ {
+ "name": "default-token-vv11j",
+ "secret": {
+ "secretName": "default-token-vv11j",
+ "defaultMode": 420
+ }
+ }
+ ],
+ "containers": [
+ {
+ "name": "kairosdb",
+ "image": "docker.io/davradocker/docker-kairosdb:1.1.1_9",
+ "env": [
+ {
+ "name": "kairosdb_http_port",
+ "value": "58080"
+ },
+ {
+ "name": "kairosdb_datastore_cassandra_host_list",
+ "valueFrom": {
+ "configMapKeyRef": {
+ "name": "cassandra-config",
+ "key": "cassandra.nodeports"
+ }
+ }
+ },
+ {
+ "name": "kairosdb_ttl",
+ "value": "315360000"
+ },
+ {
+ "name": "_JAVA_OPTIONS",
+ "value": "-Xms128m -Xmx512m"
+ }
+ ],
+ "resources": {},
+ "volumeMounts": [
+ {
+ "name": "default-token-vv11j",
+ "readOnly": true,
+ "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount"
+ }
+ ],
+ "livenessProbe": {
+ "httpGet": {
+ "path": "/api/v1/metricnames",
+ "port": 58080,
+ "scheme": "HTTP"
+ },
+ "initialDelaySeconds": 20,
+ "timeoutSeconds": 1,
+ "periodSeconds": 1,
+ "successThreshold": 1,
+ "failureThreshold": 1
+ },
+ "terminationMessagePath": "/dev/termination-log",
+ "terminationMessagePolicy": "File",
+ "imagePullPolicy": "IfNotPresent",
+ "securityContext": {
+ "privileged": false
+ }
+ }
+ ],
+ "restartPolicy": "Always",
+ "terminationGracePeriodSeconds": 30,
+ "dnsPolicy": "ClusterFirst",
+ "serviceAccountName": "default",
+ "serviceAccount": "default",
+ "nodeName": "192.168.10.169",
+ "securityContext": {},
+ "schedulerName": "default-scheduler"
+ },
+ "status": {
+ "phase": "Running",
+ "conditions": [
+ {
+ "type": "Initialized",
+ "status": "True",
+ "lastProbeTime": null,
+ "lastTransitionTime": "2018-01-30T10:03:37Z"
+ },
+ {
+ "type": "Ready",
+ "status": "True",
+ "lastProbeTime": null,
+ "lastTransitionTime": "2018-02-27T07:28:45Z"
+ },
+ {
+ "type": "PodScheduled",
+ "status": "True",
+ "lastProbeTime": null,
+ "lastTransitionTime": "2018-01-30T10:03:37Z"
+ }
+ ],
+ "hostIP": "192.168.10.169",
+ "podIP": "10.2.39.12",
+ "startTime": "2018-01-30T10:03:37Z",
+ "containerStatuses": [
+ {
+ "name": "kairosdb",
+ "state": {
+ "running": {
+ "startedAt": "2018-02-27T07:28:45Z"
+ }
+ },
+ "lastState": {
+ "terminated": {
+ "exitCode": 0,
+ "reason": "Completed",
+ "startedAt": "2018-02-27T07:27:13Z",
+ "finishedAt": "2018-02-27T07:27:16Z",
+ "containerID": "docker://5bc9f63b02c09d7726ed3669f734b365f90e077e77a88c4a2f004459772bf912"
+ }
+ },
+ "ready": true,
+ "restartCount": 20,
+ "image": "davradocker/docker-kairosdb:1.1.1_9",
+ "imageID": "docker-pullable://davradocker/docker-kairosdb@sha256:2ee07e3f3ba61b96597c78ab0aa7b837d771b375888b020da7a7316bb524b6e8",
+ "containerID": "docker://02d45318f220ad01109d13df31fbbb0668b87fdd06ef6607abfdecf5eba4b311"
+ }
+ ],
+ "qosClass": "BestEffort"
+ }
+}
diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/meta/options_k8s-logging-exclude-disabled.meta b/src/fluent-bit/tests/runtime/data/kubernetes/meta/options_k8s-logging-exclude-disabled.meta new file mode 100644 index 000000000..e986f1646 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/meta/options_k8s-logging-exclude-disabled.meta @@ -0,0 +1,7 @@ +{ + "metadata": { + "annotations": { + "fluentbit.io/exclude": "true" + } + } +} diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/meta/options_k8s-logging-parser-disabled.meta b/src/fluent-bit/tests/runtime/data/kubernetes/meta/options_k8s-logging-parser-disabled.meta new file mode 100644 index 000000000..b922088eb --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/meta/options_k8s-logging-parser-disabled.meta @@ -0,0 +1,7 @@ +{ + "metadata": { + "annotations": { + "fluentbit.io/parser": "default-parser" + } + } +} diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/meta/options_keep-log-disabled.meta b/src/fluent-bit/tests/runtime/data/kubernetes/meta/options_keep-log-disabled.meta new file mode 100644 index 000000000..4e9f5aa3e --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/meta/options_keep-log-disabled.meta @@ -0,0 +1,4 @@ +{ + "metadata": { + } +} diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/meta/options_keep-log-enabled.meta b/src/fluent-bit/tests/runtime/data/kubernetes/meta/options_keep-log-enabled.meta new file mode 100644 index 000000000..4e9f5aa3e --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/meta/options_keep-log-enabled.meta @@ -0,0 +1,4 @@ +{ + "metadata": { + } +} diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/meta/options_merge-log-disabled.meta b/src/fluent-bit/tests/runtime/data/kubernetes/meta/options_merge-log-disabled.meta new file mode 100644 index 000000000..4e9f5aa3e --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/meta/options_merge-log-disabled.meta @@ -0,0 +1,4 @@ +{ + "metadata": { + } +} diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/meta/options_merge-log-enabled.meta b/src/fluent-bit/tests/runtime/data/kubernetes/meta/options_merge-log-enabled.meta new file mode 100644 index 000000000..4e9f5aa3e --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/meta/options_merge-log-enabled.meta @@ -0,0 +1,4 @@ +{ + "metadata": { + } +} diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/meta/options_merge-log-key.meta b/src/fluent-bit/tests/runtime/data/kubernetes/meta/options_merge-log-key.meta new file mode 100644 index 000000000..4e9f5aa3e --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/meta/options_merge-log-key.meta @@ -0,0 +1,4 @@ +{ + "metadata": { + } +} diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/meta/options_merge-log-trim-disabled.meta b/src/fluent-bit/tests/runtime/data/kubernetes/meta/options_merge-log-trim-disabled.meta new file mode 100644 index 000000000..4e9f5aa3e --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/meta/options_merge-log-trim-disabled.meta @@ -0,0 +1,4 @@ +{ + "metadata": { + } +} diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/meta/options_merge-log-trim-enabled.meta b/src/fluent-bit/tests/runtime/data/kubernetes/meta/options_merge-log-trim-enabled.meta new file mode 100644 index 000000000..4e9f5aa3e --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/meta/options_merge-log-trim-enabled.meta @@ -0,0 +1,4 @@ +{ + "metadata": { + } +} diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/meta/options_use-kubelet-disabled.meta b/src/fluent-bit/tests/runtime/data/kubernetes/meta/options_use-kubelet-disabled.meta new file mode 100644 index 000000000..ce693695e --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/meta/options_use-kubelet-disabled.meta @@ -0,0 +1,116 @@ +{ + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "annotations": { + "prometheus.io/path": "/api/v1/metrics/prometheus", + "prometheus.io/port": "2020", + "prometheus.io/scrape": "true" + }, + "creationTimestamp": "2019-04-03T09:29:00Z", + "labels": { + "app.kubernetes.io/name": "fluent-bit" + }, + "name": "use-kubelet-disabled", + "namespace": "options", + "resourceVersion": "74466568", + "selfLink": "/api/v1/namespaces/core/pods/base", + "uid": "e9f2963f-55f2-11e9-84c5-02e422b8a84a" + }, + "spec": { + "containers": [ + { + "image": "fluent/fluent-bit", + "imagePullPolicy": "Always", + "name": "fluent-bit", + "resources": {}, + "stdin": true, + "stdinOnce": true, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "tty": true, + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "default-token-9ffht", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirst", + "nodeName": "ip-10-49-18-80.eu-west-1.compute.internal", + "restartPolicy": "Never", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "default", + "serviceAccountName": "default", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists", + "tolerationSeconds": 300 + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists", + "tolerationSeconds": 300 + } + ], + "volumes": [ + { + "name": "default-token-9ffht", + "secret": { + "defaultMode": 420, + "secretName": "default-token-9ffht" + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2019-04-03T09:29:00Z", + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2019-04-03T09:29:06Z", + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2019-04-03T09:29:00Z", + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "containerID": "docker://c9898099f6d235126d564ed38a020007ea7a6fac6e25e718de683c9dd0076c16", + "image": "fluent/fluent-bit:latest", + "imageID": "docker-pullable://fluent/fluent-bit@sha256:7ac0fd3569af866e9a6a22eb592744200d2dbe098cf066162453f8d0b06c531f", + "lastState": {}, + "name": "fluent-bit", + "ready": true, + "restartCount": 0, + "state": { + "running": { + "startedAt": "2019-04-03T09:29:05Z" + } + } + } + ], + "hostIP": "10.49.18.80", + "phase": "Running", + "podIP": "100.116.192.42", + "qosClass": "BestEffort", + "startTime": "2019-04-03T09:29:00Z" + } +} diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/meta/options_use-kubelet-enabled.meta b/src/fluent-bit/tests/runtime/data/kubernetes/meta/options_use-kubelet-enabled.meta new file mode 100644 index 000000000..af9fb1a3b --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/meta/options_use-kubelet-enabled.meta @@ -0,0 +1,109 @@ +{ + "apiVersion": "v1", + "kind": "Pod", + "metadata": {}, + "items": [{ + "metadata": { + "annotations": { + "prometheus.io/path": "/api/v1/metrics/prometheus", + "prometheus.io/port": "2020", + "prometheus.io/scrape": "true" + }, + "creationTimestamp": "2019-04-03T09:29:00Z", + "labels": { + "app.kubernetes.io/name": "fluent-bit" + }, + "name": "use-kubelet-enabled", + "namespace": "options", + "resourceVersion": "74466568", + "selfLink": "/api/v1/namespaces/core/pods/base", + "uid": "e9f2963f-55f2-11e9-84c5-02e422b8a84a" + }, + "spec": { + "containers": [{ + "image": "fluent/fluent-bit", + "imagePullPolicy": "Always", + "name": "fluent-bit", + "resources": {}, + "stdin": true, + "stdinOnce": true, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "tty": true, + "volumeMounts": [{ + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "default-token-9ffht", + "readOnly": true + }] + }], + "dnsPolicy": "ClusterFirst", + "nodeName": "ip-10-49-18-80.eu-west-1.compute.internal", + "restartPolicy": "Never", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "default", + "serviceAccountName": "default", + "terminationGracePeriodSeconds": 30, + "tolerations": [{ + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists", + "tolerationSeconds": 300 + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists", + "tolerationSeconds": 300 + } + ], + "volumes": [{ + "name": "default-token-9ffht", + "secret": { + "defaultMode": 420, + "secretName": "default-token-9ffht" + } + }] + }, + "status": { + "conditions": [{ + "lastProbeTime": null, + "lastTransitionTime": "2019-04-03T09:29:00Z", + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2019-04-03T09:29:06Z", + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2019-04-03T09:29:00Z", + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [{ + "containerID": "docker://c9898099f6d235126d564ed38a020007ea7a6fac6e25e718de683c9dd0076c16", + "image": "fluent/fluent-bit:latest", + "imageID": "docker-pullable://fluent/fluent-bit@sha256:7ac0fd3569af866e9a6a22eb592744200d2dbe098cf066162453f8d0b06c531f", + "lastState": {}, + "name": "fluent-bit", + "ready": true, + "restartCount": 0, + "state": { + "running": { + "startedAt": "2019-04-03T09:29:05Z" + } + } + }], + "hostIP": "10.49.18.80", + "phase": "Running", + "podIP": "100.116.192.42", + "qosClass": "BestEffort", + "startTime": "2019-04-03T09:29:00Z" + } + }] +} diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/out/annotations-exclude/annotations-exclude_invalid_text_stderr.out b/src/fluent-bit/tests/runtime/data/kubernetes/out/annotations-exclude/annotations-exclude_invalid_text_stderr.out new file mode 100644 index 000000000..14703ca6e --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/out/annotations-exclude/annotations-exclude_invalid_text_stderr.out @@ -0,0 +1 @@ +[1554141513.698657,{"log":"Simple text\n","stream":"stderr","kubernetes":{"pod_name":"invalid","namespace_name":"annotations-exclude","annotations":{"fluentbit.io/exclude":"neither-true-nor-false"},"container_name":"text"}}] diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/out/annotations-exclude/annotations-exclude_invalid_text_stdout.out b/src/fluent-bit/tests/runtime/data/kubernetes/out/annotations-exclude/annotations-exclude_invalid_text_stdout.out new file mode 100644 index 000000000..66bcbba9b --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/out/annotations-exclude/annotations-exclude_invalid_text_stdout.out @@ -0,0 +1 @@ +[1554141513.598656,{"log":"Simple text\n","stream":"stdout","kubernetes":{"pod_name":"invalid","namespace_name":"annotations-exclude","annotations":{"fluentbit.io/exclude":"neither-true-nor-false"},"container_name":"text"}}] diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/out/annotations-exclude/annotations-exclude_multiple-1_container-2_stderr.out b/src/fluent-bit/tests/runtime/data/kubernetes/out/annotations-exclude/annotations-exclude_multiple-1_container-2_stderr.out new file mode 100644 index 000000000..8f22e1d4b --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/out/annotations-exclude/annotations-exclude_multiple-1_container-2_stderr.out @@ -0,0 +1 @@ +[1554141513.698657,{"log":"Simple text\n","stream":"stderr","kubernetes":{"pod_name":"multiple-1","namespace_name":"annotations-exclude","annotations":{"fluentbit.io/exclude-container-1":"true","fluentbit.io/exclude_stdout-container-2":"true","fluentbit.io/exclude_stderr-container-3":"true"},"container_name":"container-2"}}] diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/out/annotations-exclude/annotations-exclude_multiple-1_container-3_stdout.out b/src/fluent-bit/tests/runtime/data/kubernetes/out/annotations-exclude/annotations-exclude_multiple-1_container-3_stdout.out new file mode 100644 index 000000000..f534e361d --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/out/annotations-exclude/annotations-exclude_multiple-1_container-3_stdout.out @@ -0,0 +1 @@ +[1554141513.598656,{"log":"Simple text\n","stream":"stdout","kubernetes":{"pod_name":"multiple-1","namespace_name":"annotations-exclude","annotations":{"fluentbit.io/exclude-container-1":"true","fluentbit.io/exclude_stdout-container-2":"true","fluentbit.io/exclude_stderr-container-3":"true"},"container_name":"container-3"}}] diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/out/annotations-exclude/annotations-exclude_multiple-1_container-4_stderr.out b/src/fluent-bit/tests/runtime/data/kubernetes/out/annotations-exclude/annotations-exclude_multiple-1_container-4_stderr.out new file mode 100644 index 000000000..08ae820ac --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/out/annotations-exclude/annotations-exclude_multiple-1_container-4_stderr.out @@ -0,0 +1 @@ +[1554141513.698657,{"log":"Simple text\n","stream":"stderr","kubernetes":{"pod_name":"multiple-1","namespace_name":"annotations-exclude","annotations":{"fluentbit.io/exclude-container-1":"true","fluentbit.io/exclude_stdout-container-2":"true","fluentbit.io/exclude_stderr-container-3":"true"},"container_name":"container-4"}}] diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/out/annotations-exclude/annotations-exclude_multiple-1_container-4_stdout.out b/src/fluent-bit/tests/runtime/data/kubernetes/out/annotations-exclude/annotations-exclude_multiple-1_container-4_stdout.out new file mode 100644 index 000000000..236481afd --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/out/annotations-exclude/annotations-exclude_multiple-1_container-4_stdout.out @@ -0,0 +1 @@ +[1554141513.598656,{"log":"Simple text\n","stream":"stdout","kubernetes":{"pod_name":"multiple-1","namespace_name":"annotations-exclude","annotations":{"fluentbit.io/exclude-container-1":"true","fluentbit.io/exclude_stdout-container-2":"true","fluentbit.io/exclude_stderr-container-3":"true"},"container_name":"container-4"}}] diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/out/annotations-exclude/annotations-exclude_multiple-2_container-2_stderr.out b/src/fluent-bit/tests/runtime/data/kubernetes/out/annotations-exclude/annotations-exclude_multiple-2_container-2_stderr.out new file mode 100644 index 000000000..50520e92b --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/out/annotations-exclude/annotations-exclude_multiple-2_container-2_stderr.out @@ -0,0 +1 @@ +[1554141513.698657,{"log":"Simple text\n","stream":"stderr","kubernetes":{"pod_name":"multiple-2","namespace_name":"annotations-exclude","annotations":{"fluentbit.io/exclude":"true","fluentbit.io/exclude_stderr-container-2":"false","fluentbit.io/exclude_stdout-container-3":"false","fluentbit.io/exclude-container-4":"false"},"container_name":"container-2"}}] diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/out/annotations-exclude/annotations-exclude_multiple-2_container-3_stdout.out b/src/fluent-bit/tests/runtime/data/kubernetes/out/annotations-exclude/annotations-exclude_multiple-2_container-3_stdout.out new file mode 100644 index 000000000..73e7e5db1 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/out/annotations-exclude/annotations-exclude_multiple-2_container-3_stdout.out @@ -0,0 +1 @@ +[1554141513.598656,{"log":"Simple text\n","stream":"stdout","kubernetes":{"pod_name":"multiple-2","namespace_name":"annotations-exclude","annotations":{"fluentbit.io/exclude":"true","fluentbit.io/exclude_stderr-container-2":"false","fluentbit.io/exclude_stdout-container-3":"false","fluentbit.io/exclude-container-4":"false"},"container_name":"container-3"}}] diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/out/annotations-exclude/annotations-exclude_multiple-2_container-4_stderr.out b/src/fluent-bit/tests/runtime/data/kubernetes/out/annotations-exclude/annotations-exclude_multiple-2_container-4_stderr.out new file mode 100644 index 000000000..0b7da35cd --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/out/annotations-exclude/annotations-exclude_multiple-2_container-4_stderr.out @@ -0,0 +1 @@ +[1554141513.698657,{"log":"Simple text\n","stream":"stderr","kubernetes":{"pod_name":"multiple-2","namespace_name":"annotations-exclude","annotations":{"fluentbit.io/exclude":"true","fluentbit.io/exclude_stderr-container-2":"false","fluentbit.io/exclude_stdout-container-3":"false","fluentbit.io/exclude-container-4":"false"},"container_name":"container-4"}}] diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/out/annotations-exclude/annotations-exclude_multiple-2_container-4_stdout.out b/src/fluent-bit/tests/runtime/data/kubernetes/out/annotations-exclude/annotations-exclude_multiple-2_container-4_stdout.out new file mode 100644 index 000000000..427b67c7c --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/out/annotations-exclude/annotations-exclude_multiple-2_container-4_stdout.out @@ -0,0 +1 @@ +[1554141513.598656,{"log":"Simple text\n","stream":"stdout","kubernetes":{"pod_name":"multiple-2","namespace_name":"annotations-exclude","annotations":{"fluentbit.io/exclude":"true","fluentbit.io/exclude_stderr-container-2":"false","fluentbit.io/exclude_stdout-container-3":"false","fluentbit.io/exclude-container-4":"false"},"container_name":"container-4"}}] diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/out/annotations-exclude/annotations-exclude_multiple-3_container-2_stderr.out b/src/fluent-bit/tests/runtime/data/kubernetes/out/annotations-exclude/annotations-exclude_multiple-3_container-2_stderr.out new file mode 100644 index 000000000..de5af8a15 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/out/annotations-exclude/annotations-exclude_multiple-3_container-2_stderr.out @@ -0,0 +1 @@ +[1554141513.698657,{"log":"Simple text\n","stream":"stderr","kubernetes":{"pod_name":"multiple-3","namespace_name":"annotations-exclude","annotations":{"fluentbit.io/exclude_stdout":"true","fluentbit.io/exclude_stderr-container-1":"true","fluentbit.io/exclude_stdout-container-3":"false","fluentbit.io/exclude_stderr-container-3":"true","fluentbit.io/exclude_stdout-container-4":"false"},"container_name":"container-2"}}] diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/out/annotations-exclude/annotations-exclude_multiple-3_container-3_stdout.out b/src/fluent-bit/tests/runtime/data/kubernetes/out/annotations-exclude/annotations-exclude_multiple-3_container-3_stdout.out new file mode 100644 index 000000000..de7d598ed --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/out/annotations-exclude/annotations-exclude_multiple-3_container-3_stdout.out @@ -0,0 +1 @@ +[1554141513.598656,{"log":"Simple text\n","stream":"stdout","kubernetes":{"pod_name":"multiple-3","namespace_name":"annotations-exclude","annotations":{"fluentbit.io/exclude_stdout":"true","fluentbit.io/exclude_stderr-container-1":"true","fluentbit.io/exclude_stdout-container-3":"false","fluentbit.io/exclude_stderr-container-3":"true","fluentbit.io/exclude_stdout-container-4":"false"},"container_name":"container-3"}}] diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/out/annotations-exclude/annotations-exclude_multiple-3_container-4_stderr.out b/src/fluent-bit/tests/runtime/data/kubernetes/out/annotations-exclude/annotations-exclude_multiple-3_container-4_stderr.out new file mode 100644 index 000000000..cff4f21e7 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/out/annotations-exclude/annotations-exclude_multiple-3_container-4_stderr.out @@ -0,0 +1 @@ +[1554141513.698657,{"log":"Simple text\n","stream":"stderr","kubernetes":{"pod_name":"multiple-3","namespace_name":"annotations-exclude","annotations":{"fluentbit.io/exclude_stdout":"true","fluentbit.io/exclude_stderr-container-1":"true","fluentbit.io/exclude_stdout-container-3":"false","fluentbit.io/exclude_stderr-container-3":"true","fluentbit.io/exclude_stdout-container-4":"false"},"container_name":"container-4"}}] diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/out/annotations-exclude/annotations-exclude_multiple-3_container-4_stdout.out b/src/fluent-bit/tests/runtime/data/kubernetes/out/annotations-exclude/annotations-exclude_multiple-3_container-4_stdout.out new file mode 100644 index 000000000..0329d2181 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/out/annotations-exclude/annotations-exclude_multiple-3_container-4_stdout.out @@ -0,0 +1 @@ +[1554141513.598656,{"log":"Simple text\n","stream":"stdout","kubernetes":{"pod_name":"multiple-3","namespace_name":"annotations-exclude","annotations":{"fluentbit.io/exclude_stdout":"true","fluentbit.io/exclude_stderr-container-1":"true","fluentbit.io/exclude_stdout-container-3":"false","fluentbit.io/exclude_stderr-container-3":"true","fluentbit.io/exclude_stdout-container-4":"false"},"container_name":"container-4"}}] diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/out/annotations-exclude/annotations-exclude_multiple-4_container-2_stderr.out b/src/fluent-bit/tests/runtime/data/kubernetes/out/annotations-exclude/annotations-exclude_multiple-4_container-2_stderr.out new file mode 100644 index 000000000..ad9fa8d41 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/out/annotations-exclude/annotations-exclude_multiple-4_container-2_stderr.out @@ -0,0 +1 @@ +[1554141513.698657,{"log":"Simple text\n","stream":"stderr","kubernetes":{"pod_name":"multiple-4","namespace_name":"annotations-exclude","annotations":{"fluentbit.io/exclude_stderr":"true","fluentbit.io/exclude_stdout-container-1":"true","fluentbit.io/exclude_stdout-container-2":"true","fluentbit.io/exclude_stderr-container-2":"false","fluentbit.io/exclude_stderr-container-4":"false"},"container_name":"container-2"}}] diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/out/annotations-exclude/annotations-exclude_multiple-4_container-3_stdout.out b/src/fluent-bit/tests/runtime/data/kubernetes/out/annotations-exclude/annotations-exclude_multiple-4_container-3_stdout.out new file mode 100644 index 000000000..730abcc71 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/out/annotations-exclude/annotations-exclude_multiple-4_container-3_stdout.out @@ -0,0 +1 @@ +[1554141513.598656,{"log":"Simple text\n","stream":"stdout","kubernetes":{"pod_name":"multiple-4","namespace_name":"annotations-exclude","annotations":{"fluentbit.io/exclude_stderr":"true","fluentbit.io/exclude_stdout-container-1":"true","fluentbit.io/exclude_stdout-container-2":"true","fluentbit.io/exclude_stderr-container-2":"false","fluentbit.io/exclude_stderr-container-4":"false"},"container_name":"container-3"}}] diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/out/annotations-exclude/annotations-exclude_multiple-4_container-4_stderr.out b/src/fluent-bit/tests/runtime/data/kubernetes/out/annotations-exclude/annotations-exclude_multiple-4_container-4_stderr.out new file mode 100644 index 000000000..228657629 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/out/annotations-exclude/annotations-exclude_multiple-4_container-4_stderr.out @@ -0,0 +1 @@ +[1554141513.698657,{"log":"Simple text\n","stream":"stderr","kubernetes":{"pod_name":"multiple-4","namespace_name":"annotations-exclude","annotations":{"fluentbit.io/exclude_stderr":"true","fluentbit.io/exclude_stdout-container-1":"true","fluentbit.io/exclude_stdout-container-2":"true","fluentbit.io/exclude_stderr-container-2":"false","fluentbit.io/exclude_stderr-container-4":"false"},"container_name":"container-4"}}] diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/out/annotations-exclude/annotations-exclude_multiple-4_container-4_stdout.out b/src/fluent-bit/tests/runtime/data/kubernetes/out/annotations-exclude/annotations-exclude_multiple-4_container-4_stdout.out new file mode 100644 index 000000000..75b6ae3a5 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/out/annotations-exclude/annotations-exclude_multiple-4_container-4_stdout.out @@ -0,0 +1 @@ +[1554141513.598656,{"log":"Simple text\n","stream":"stdout","kubernetes":{"pod_name":"multiple-4","namespace_name":"annotations-exclude","annotations":{"fluentbit.io/exclude_stderr":"true","fluentbit.io/exclude_stdout-container-1":"true","fluentbit.io/exclude_stdout-container-2":"true","fluentbit.io/exclude_stderr-container-2":"false","fluentbit.io/exclude_stderr-container-4":"false"},"container_name":"container-4"}}] diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/out/annotations-exclude/annotations-exclude_stderr_text_stdout.out b/src/fluent-bit/tests/runtime/data/kubernetes/out/annotations-exclude/annotations-exclude_stderr_text_stdout.out new file mode 100644 index 000000000..0e237f7a8 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/out/annotations-exclude/annotations-exclude_stderr_text_stdout.out @@ -0,0 +1 @@ +[1554141513.598656,{"log":"Simple text\n","stream":"stdout","kubernetes":{"pod_name":"stderr","namespace_name":"annotations-exclude","annotations":{"fluentbit.io/exclude_stderr":"true"},"container_name":"text"}}] diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/out/annotations-exclude/annotations-exclude_stdout_text_stderr.out b/src/fluent-bit/tests/runtime/data/kubernetes/out/annotations-exclude/annotations-exclude_stdout_text_stderr.out new file mode 100644 index 000000000..8204d04df --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/out/annotations-exclude/annotations-exclude_stdout_text_stderr.out @@ -0,0 +1 @@ +[1554141513.698657,{"log":"Simple text\n","stream":"stderr","kubernetes":{"pod_name":"stdout","namespace_name":"annotations-exclude","annotations":{"fluentbit.io/exclude_stdout":"true"},"container_name":"text"}}] diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/out/annotations-parser/annotations-parser_invalid_text_stderr.out b/src/fluent-bit/tests/runtime/data/kubernetes/out/annotations-parser/annotations-parser_invalid_text_stderr.out new file mode 100644 index 000000000..dbb0d7a0e --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/out/annotations-parser/annotations-parser_invalid_text_stderr.out @@ -0,0 +1 @@ +[1554141513.698657,{"log":"Simple text\n","stream":"stderr","kubernetes":{"pod_name":"invalid","namespace_name":"annotations-parser","annotations":{"fluentbit.io/parser":"invalid-parser"},"container_name":"text"}}] diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/out/annotations-parser/annotations-parser_invalid_text_stdout.out b/src/fluent-bit/tests/runtime/data/kubernetes/out/annotations-parser/annotations-parser_invalid_text_stdout.out new file mode 100644 index 000000000..5bb03765c --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/out/annotations-parser/annotations-parser_invalid_text_stdout.out @@ -0,0 +1 @@ +[1554141513.598656,{"log":"Simple text\n","stream":"stdout","kubernetes":{"pod_name":"invalid","namespace_name":"annotations-parser","annotations":{"fluentbit.io/parser":"invalid-parser"},"container_name":"text"}}] diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/out/annotations-parser/annotations-parser_json-with-time_invalid-json-1.out b/src/fluent-bit/tests/runtime/data/kubernetes/out/annotations-parser/annotations-parser_json-with-time_invalid-json-1.out new file mode 100644 index 000000000..168358274 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/out/annotations-parser/annotations-parser_json-with-time_invalid-json-1.out @@ -0,0 +1 @@ +[1554141513.598656,{"log":"{\"time\":\"2019-04-05T14:54:23.584269761\",\"message\":\"Simple text\"]\n","stream":"stdout","kubernetes":{"pod_name":"json-with-time","namespace_name":"annotations-parser","annotations":{"fluentbit.io/parser":"json-parser-with-time"},"container_name":"invalid-json-1"}}] diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/out/annotations-parser/annotations-parser_json-with-time_json.out b/src/fluent-bit/tests/runtime/data/kubernetes/out/annotations-parser/annotations-parser_json-with-time_json.out new file mode 100644 index 000000000..2a472dad8 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/out/annotations-parser/annotations-parser_json-with-time_json.out @@ -0,0 +1 @@ +[1554476063.584270,{"stream":"stdout","message":"Simple text","kubernetes":{"pod_name":"json-with-time","namespace_name":"annotations-parser","annotations":{"fluentbit.io/parser":"json-parser-with-time"},"container_name":"json"}}] diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/out/annotations-parser/annotations-parser_multiple-1_container-1_stderr.out b/src/fluent-bit/tests/runtime/data/kubernetes/out/annotations-parser/annotations-parser_multiple-1_container-1_stderr.out new file mode 100644 index 000000000..937c3add5 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/out/annotations-parser/annotations-parser_multiple-1_container-1_stderr.out @@ -0,0 +1 @@ +[1554476063.584270,{"stream":"stderr","container_1_parser":"Container 1 is logging on stderr","kubernetes":{"pod_name":"multiple-1","namespace_name":"annotations-parser","annotations":{"fluentbit.io/parser-container-1":"container-1-parser","fluentbit.io/parser_stdout-container-2":"container-2-stdout-parser","fluentbit.io/parser_stderr-container-2":"container-2-stderr-parser","fluentbit.io/parser_stdout-container-3":"container-3-stdout-parser","fluentbit.io/parser_stderr-container-4":"container-4-stderr-parser","fluentbit.io/parser":"default-parser"},"container_name":"container-1"}}] diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/out/annotations-parser/annotations-parser_multiple-1_container-1_stdout.out b/src/fluent-bit/tests/runtime/data/kubernetes/out/annotations-parser/annotations-parser_multiple-1_container-1_stdout.out new file mode 100644 index 000000000..c672970cb --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/out/annotations-parser/annotations-parser_multiple-1_container-1_stdout.out @@ -0,0 +1 @@ +[1554476063.584270,{"stream":"stdout","container_1_parser":"Container 1 is logging on stdout","kubernetes":{"pod_name":"multiple-1","namespace_name":"annotations-parser","annotations":{"fluentbit.io/parser-container-1":"container-1-parser","fluentbit.io/parser_stdout-container-2":"container-2-stdout-parser","fluentbit.io/parser_stderr-container-2":"container-2-stderr-parser","fluentbit.io/parser_stdout-container-3":"container-3-stdout-parser","fluentbit.io/parser_stderr-container-4":"container-4-stderr-parser","fluentbit.io/parser":"default-parser"},"container_name":"container-1"}}] diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/out/annotations-parser/annotations-parser_multiple-1_container-2_stderr.out b/src/fluent-bit/tests/runtime/data/kubernetes/out/annotations-parser/annotations-parser_multiple-1_container-2_stderr.out new file mode 100644 index 000000000..02f9ce662 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/out/annotations-parser/annotations-parser_multiple-1_container-2_stderr.out @@ -0,0 +1 @@ +[1554476063.584270,{"stream":"stderr","container_2_stderr_parser":"Container 2 is logging on stderr","kubernetes":{"pod_name":"multiple-1","namespace_name":"annotations-parser","annotations":{"fluentbit.io/parser-container-1":"container-1-parser","fluentbit.io/parser_stdout-container-2":"container-2-stdout-parser","fluentbit.io/parser_stderr-container-2":"container-2-stderr-parser","fluentbit.io/parser_stdout-container-3":"container-3-stdout-parser","fluentbit.io/parser_stderr-container-4":"container-4-stderr-parser","fluentbit.io/parser":"default-parser"},"container_name":"container-2"}}] diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/out/annotations-parser/annotations-parser_multiple-1_container-2_stdout.out b/src/fluent-bit/tests/runtime/data/kubernetes/out/annotations-parser/annotations-parser_multiple-1_container-2_stdout.out new file mode 100644 index 000000000..9d3d358cf --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/out/annotations-parser/annotations-parser_multiple-1_container-2_stdout.out @@ -0,0 +1 @@ +[1554476063.584270,{"stream":"stdout","container_2_stdout_parser":"Container 2 is logging on stdout","kubernetes":{"pod_name":"multiple-1","namespace_name":"annotations-parser","annotations":{"fluentbit.io/parser-container-1":"container-1-parser","fluentbit.io/parser_stdout-container-2":"container-2-stdout-parser","fluentbit.io/parser_stderr-container-2":"container-2-stderr-parser","fluentbit.io/parser_stdout-container-3":"container-3-stdout-parser","fluentbit.io/parser_stderr-container-4":"container-4-stderr-parser","fluentbit.io/parser":"default-parser"},"container_name":"container-2"}}] diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/out/annotations-parser/annotations-parser_multiple-1_container-3_stderr.out b/src/fluent-bit/tests/runtime/data/kubernetes/out/annotations-parser/annotations-parser_multiple-1_container-3_stderr.out new file mode 100644 index 000000000..3908d07e5 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/out/annotations-parser/annotations-parser_multiple-1_container-3_stderr.out @@ -0,0 +1 @@ +[1554476063.584270,{"stream":"stderr","default_parser":"Container 3 is logging on stderr","kubernetes":{"pod_name":"multiple-1","namespace_name":"annotations-parser","annotations":{"fluentbit.io/parser-container-1":"container-1-parser","fluentbit.io/parser_stdout-container-2":"container-2-stdout-parser","fluentbit.io/parser_stderr-container-2":"container-2-stderr-parser","fluentbit.io/parser_stdout-container-3":"container-3-stdout-parser","fluentbit.io/parser_stderr-container-4":"container-4-stderr-parser","fluentbit.io/parser":"default-parser"},"container_name":"container-3"}}] diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/out/annotations-parser/annotations-parser_multiple-1_container-3_stdout.out b/src/fluent-bit/tests/runtime/data/kubernetes/out/annotations-parser/annotations-parser_multiple-1_container-3_stdout.out new file mode 100644 index 000000000..77ca2ce0b --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/out/annotations-parser/annotations-parser_multiple-1_container-3_stdout.out @@ -0,0 +1 @@ +[1554476063.584270,{"stream":"stdout","container_3_stdout_parser":"Container 3 is logging on stdout","kubernetes":{"pod_name":"multiple-1","namespace_name":"annotations-parser","annotations":{"fluentbit.io/parser-container-1":"container-1-parser","fluentbit.io/parser_stdout-container-2":"container-2-stdout-parser","fluentbit.io/parser_stderr-container-2":"container-2-stderr-parser","fluentbit.io/parser_stdout-container-3":"container-3-stdout-parser","fluentbit.io/parser_stderr-container-4":"container-4-stderr-parser","fluentbit.io/parser":"default-parser"},"container_name":"container-3"}}] diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/out/annotations-parser/annotations-parser_multiple-1_container-4_stderr.out b/src/fluent-bit/tests/runtime/data/kubernetes/out/annotations-parser/annotations-parser_multiple-1_container-4_stderr.out new file mode 100644 index 000000000..f3752f595 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/out/annotations-parser/annotations-parser_multiple-1_container-4_stderr.out @@ -0,0 +1 @@ +[1554476063.584270,{"stream":"stderr","container_4_stderr_parser":"Container 4 is logging on stderr","kubernetes":{"pod_name":"multiple-1","namespace_name":"annotations-parser","annotations":{"fluentbit.io/parser-container-1":"container-1-parser","fluentbit.io/parser_stdout-container-2":"container-2-stdout-parser","fluentbit.io/parser_stderr-container-2":"container-2-stderr-parser","fluentbit.io/parser_stdout-container-3":"container-3-stdout-parser","fluentbit.io/parser_stderr-container-4":"container-4-stderr-parser","fluentbit.io/parser":"default-parser"},"container_name":"container-4"}}] diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/out/annotations-parser/annotations-parser_multiple-1_container-4_stdout.out b/src/fluent-bit/tests/runtime/data/kubernetes/out/annotations-parser/annotations-parser_multiple-1_container-4_stdout.out new file mode 100644 index 000000000..bf81796a4 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/out/annotations-parser/annotations-parser_multiple-1_container-4_stdout.out @@ -0,0 +1 @@ +[1554476063.584270,{"stream":"stdout","default_parser":"Container 4 is logging on stdout","kubernetes":{"pod_name":"multiple-1","namespace_name":"annotations-parser","annotations":{"fluentbit.io/parser-container-1":"container-1-parser","fluentbit.io/parser_stdout-container-2":"container-2-stdout-parser","fluentbit.io/parser_stderr-container-2":"container-2-stderr-parser","fluentbit.io/parser_stdout-container-3":"container-3-stdout-parser","fluentbit.io/parser_stderr-container-4":"container-4-stderr-parser","fluentbit.io/parser":"default-parser"},"container_name":"container-4"}}] diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/out/annotations-parser/annotations-parser_multiple-1_container-5_stderr.out b/src/fluent-bit/tests/runtime/data/kubernetes/out/annotations-parser/annotations-parser_multiple-1_container-5_stderr.out new file mode 100644 index 000000000..b633eb0e9 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/out/annotations-parser/annotations-parser_multiple-1_container-5_stderr.out @@ -0,0 +1 @@ +[1554476063.584270,{"stream":"stderr","default_parser":"Container 5 is logging on stderr","kubernetes":{"pod_name":"multiple-1","namespace_name":"annotations-parser","annotations":{"fluentbit.io/parser-container-1":"container-1-parser","fluentbit.io/parser_stdout-container-2":"container-2-stdout-parser","fluentbit.io/parser_stderr-container-2":"container-2-stderr-parser","fluentbit.io/parser_stdout-container-3":"container-3-stdout-parser","fluentbit.io/parser_stderr-container-4":"container-4-stderr-parser","fluentbit.io/parser":"default-parser"},"container_name":"container-5"}}] diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/out/annotations-parser/annotations-parser_multiple-1_container-5_stdout.out b/src/fluent-bit/tests/runtime/data/kubernetes/out/annotations-parser/annotations-parser_multiple-1_container-5_stdout.out new file mode 100644 index 000000000..f61377997 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/out/annotations-parser/annotations-parser_multiple-1_container-5_stdout.out @@ -0,0 +1 @@ +[1554476063.584270,{"stream":"stdout","default_parser":"Container 5 is logging on stdout","kubernetes":{"pod_name":"multiple-1","namespace_name":"annotations-parser","annotations":{"fluentbit.io/parser-container-1":"container-1-parser","fluentbit.io/parser_stdout-container-2":"container-2-stdout-parser","fluentbit.io/parser_stderr-container-2":"container-2-stderr-parser","fluentbit.io/parser_stdout-container-3":"container-3-stdout-parser","fluentbit.io/parser_stderr-container-4":"container-4-stderr-parser","fluentbit.io/parser":"default-parser"},"container_name":"container-5"}}] diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/out/annotations-parser/annotations-parser_multiple-2_container-1_stderr.out b/src/fluent-bit/tests/runtime/data/kubernetes/out/annotations-parser/annotations-parser_multiple-2_container-1_stderr.out new file mode 100644 index 000000000..d4d401d05 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/out/annotations-parser/annotations-parser_multiple-2_container-1_stderr.out @@ -0,0 +1 @@ +[1554476063.584270,{"stream":"stderr","container_1_parser":"Container 1 is logging on stderr","kubernetes":{"pod_name":"multiple-2","namespace_name":"annotations-parser","annotations":{"fluentbit.io/parser-container-1":"container-1-parser","fluentbit.io/parser_stdout-container-2":"container-2-stdout-parser","fluentbit.io/parser_stderr-container-2":"container-2-stderr-parser","fluentbit.io/parser_stdout-container-3":"container-3-stdout-parser","fluentbit.io/parser_stderr-container-4":"container-4-stderr-parser","fluentbit.io/parser_stdout":"default-stdout-parser","fluentbit.io/parser_stderr":"default-stderr-parser"},"container_name":"container-1"}}] diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/out/annotations-parser/annotations-parser_multiple-2_container-1_stdout.out b/src/fluent-bit/tests/runtime/data/kubernetes/out/annotations-parser/annotations-parser_multiple-2_container-1_stdout.out new file mode 100644 index 000000000..fc0de55cb --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/out/annotations-parser/annotations-parser_multiple-2_container-1_stdout.out @@ -0,0 +1 @@ +[1554476063.584270,{"stream":"stdout","container_1_parser":"Container 1 is logging on stdout","kubernetes":{"pod_name":"multiple-2","namespace_name":"annotations-parser","annotations":{"fluentbit.io/parser-container-1":"container-1-parser","fluentbit.io/parser_stdout-container-2":"container-2-stdout-parser","fluentbit.io/parser_stderr-container-2":"container-2-stderr-parser","fluentbit.io/parser_stdout-container-3":"container-3-stdout-parser","fluentbit.io/parser_stderr-container-4":"container-4-stderr-parser","fluentbit.io/parser_stdout":"default-stdout-parser","fluentbit.io/parser_stderr":"default-stderr-parser"},"container_name":"container-1"}}] diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/out/annotations-parser/annotations-parser_multiple-2_container-2_stderr.out b/src/fluent-bit/tests/runtime/data/kubernetes/out/annotations-parser/annotations-parser_multiple-2_container-2_stderr.out new file mode 100644 index 000000000..38214f3c0 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/out/annotations-parser/annotations-parser_multiple-2_container-2_stderr.out @@ -0,0 +1 @@ +[1554476063.584270,{"stream":"stderr","container_2_stderr_parser":"Container 2 is logging on stderr","kubernetes":{"pod_name":"multiple-2","namespace_name":"annotations-parser","annotations":{"fluentbit.io/parser-container-1":"container-1-parser","fluentbit.io/parser_stdout-container-2":"container-2-stdout-parser","fluentbit.io/parser_stderr-container-2":"container-2-stderr-parser","fluentbit.io/parser_stdout-container-3":"container-3-stdout-parser","fluentbit.io/parser_stderr-container-4":"container-4-stderr-parser","fluentbit.io/parser_stdout":"default-stdout-parser","fluentbit.io/parser_stderr":"default-stderr-parser"},"container_name":"container-2"}}] diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/out/annotations-parser/annotations-parser_multiple-2_container-2_stdout.out b/src/fluent-bit/tests/runtime/data/kubernetes/out/annotations-parser/annotations-parser_multiple-2_container-2_stdout.out new file mode 100644 index 000000000..6c4dc1d72 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/out/annotations-parser/annotations-parser_multiple-2_container-2_stdout.out @@ -0,0 +1 @@ +[1554476063.584270,{"stream":"stdout","container_2_stdout_parser":"Container 2 is logging on stdout","kubernetes":{"pod_name":"multiple-2","namespace_name":"annotations-parser","annotations":{"fluentbit.io/parser-container-1":"container-1-parser","fluentbit.io/parser_stdout-container-2":"container-2-stdout-parser","fluentbit.io/parser_stderr-container-2":"container-2-stderr-parser","fluentbit.io/parser_stdout-container-3":"container-3-stdout-parser","fluentbit.io/parser_stderr-container-4":"container-4-stderr-parser","fluentbit.io/parser_stdout":"default-stdout-parser","fluentbit.io/parser_stderr":"default-stderr-parser"},"container_name":"container-2"}}] diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/out/annotations-parser/annotations-parser_multiple-2_container-3_stderr.out b/src/fluent-bit/tests/runtime/data/kubernetes/out/annotations-parser/annotations-parser_multiple-2_container-3_stderr.out new file mode 100644 index 000000000..8d5e8cf76 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/out/annotations-parser/annotations-parser_multiple-2_container-3_stderr.out @@ -0,0 +1 @@ +[1554476063.584270,{"stream":"stderr","default_stderr_parser":"Container 3 is logging on stderr","kubernetes":{"pod_name":"multiple-2","namespace_name":"annotations-parser","annotations":{"fluentbit.io/parser-container-1":"container-1-parser","fluentbit.io/parser_stdout-container-2":"container-2-stdout-parser","fluentbit.io/parser_stderr-container-2":"container-2-stderr-parser","fluentbit.io/parser_stdout-container-3":"container-3-stdout-parser","fluentbit.io/parser_stderr-container-4":"container-4-stderr-parser","fluentbit.io/parser_stdout":"default-stdout-parser","fluentbit.io/parser_stderr":"default-stderr-parser"},"container_name":"container-3"}}] diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/out/annotations-parser/annotations-parser_multiple-2_container-3_stdout.out b/src/fluent-bit/tests/runtime/data/kubernetes/out/annotations-parser/annotations-parser_multiple-2_container-3_stdout.out new file mode 100644 index 000000000..1371805ec --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/out/annotations-parser/annotations-parser_multiple-2_container-3_stdout.out @@ -0,0 +1 @@ +[1554476063.584270,{"stream":"stdout","container_3_stdout_parser":"Container 3 is logging on stdout","kubernetes":{"pod_name":"multiple-2","namespace_name":"annotations-parser","annotations":{"fluentbit.io/parser-container-1":"container-1-parser","fluentbit.io/parser_stdout-container-2":"container-2-stdout-parser","fluentbit.io/parser_stderr-container-2":"container-2-stderr-parser","fluentbit.io/parser_stdout-container-3":"container-3-stdout-parser","fluentbit.io/parser_stderr-container-4":"container-4-stderr-parser","fluentbit.io/parser_stdout":"default-stdout-parser","fluentbit.io/parser_stderr":"default-stderr-parser"},"container_name":"container-3"}}] diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/out/annotations-parser/annotations-parser_multiple-2_container-4_stderr.out b/src/fluent-bit/tests/runtime/data/kubernetes/out/annotations-parser/annotations-parser_multiple-2_container-4_stderr.out new file mode 100644 index 000000000..bca461e51 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/out/annotations-parser/annotations-parser_multiple-2_container-4_stderr.out @@ -0,0 +1 @@ +[1554476063.584270,{"stream":"stderr","container_4_stderr_parser":"Container 4 is logging on stderr","kubernetes":{"pod_name":"multiple-2","namespace_name":"annotations-parser","annotations":{"fluentbit.io/parser-container-1":"container-1-parser","fluentbit.io/parser_stdout-container-2":"container-2-stdout-parser","fluentbit.io/parser_stderr-container-2":"container-2-stderr-parser","fluentbit.io/parser_stdout-container-3":"container-3-stdout-parser","fluentbit.io/parser_stderr-container-4":"container-4-stderr-parser","fluentbit.io/parser_stdout":"default-stdout-parser","fluentbit.io/parser_stderr":"default-stderr-parser"},"container_name":"container-4"}}] diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/out/annotations-parser/annotations-parser_multiple-2_container-4_stdout.out b/src/fluent-bit/tests/runtime/data/kubernetes/out/annotations-parser/annotations-parser_multiple-2_container-4_stdout.out new file mode 100644 index 000000000..79aa4d99c --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/out/annotations-parser/annotations-parser_multiple-2_container-4_stdout.out @@ -0,0 +1 @@ +[1554476063.584270,{"stream":"stdout","default_stdout_parser":"Container 4 is logging on stdout","kubernetes":{"pod_name":"multiple-2","namespace_name":"annotations-parser","annotations":{"fluentbit.io/parser-container-1":"container-1-parser","fluentbit.io/parser_stdout-container-2":"container-2-stdout-parser","fluentbit.io/parser_stderr-container-2":"container-2-stderr-parser","fluentbit.io/parser_stdout-container-3":"container-3-stdout-parser","fluentbit.io/parser_stderr-container-4":"container-4-stderr-parser","fluentbit.io/parser_stdout":"default-stdout-parser","fluentbit.io/parser_stderr":"default-stderr-parser"},"container_name":"container-4"}}] diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/out/annotations-parser/annotations-parser_multiple-2_container-5_stderr.out b/src/fluent-bit/tests/runtime/data/kubernetes/out/annotations-parser/annotations-parser_multiple-2_container-5_stderr.out new file mode 100644 index 000000000..860dbeba8 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/out/annotations-parser/annotations-parser_multiple-2_container-5_stderr.out @@ -0,0 +1 @@ +[1554476063.584270,{"stream":"stderr","default_stderr_parser":"Container 5 is logging on stderr","kubernetes":{"pod_name":"multiple-2","namespace_name":"annotations-parser","annotations":{"fluentbit.io/parser-container-1":"container-1-parser","fluentbit.io/parser_stdout-container-2":"container-2-stdout-parser","fluentbit.io/parser_stderr-container-2":"container-2-stderr-parser","fluentbit.io/parser_stdout-container-3":"container-3-stdout-parser","fluentbit.io/parser_stderr-container-4":"container-4-stderr-parser","fluentbit.io/parser_stdout":"default-stdout-parser","fluentbit.io/parser_stderr":"default-stderr-parser"},"container_name":"container-5"}}] diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/out/annotations-parser/annotations-parser_multiple-2_container-5_stdout.out b/src/fluent-bit/tests/runtime/data/kubernetes/out/annotations-parser/annotations-parser_multiple-2_container-5_stdout.out new file mode 100644 index 000000000..141fc4228 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/out/annotations-parser/annotations-parser_multiple-2_container-5_stdout.out @@ -0,0 +1 @@ +[1554476063.584270,{"stream":"stdout","default_stdout_parser":"Container 5 is logging on stdout","kubernetes":{"pod_name":"multiple-2","namespace_name":"annotations-parser","annotations":{"fluentbit.io/parser-container-1":"container-1-parser","fluentbit.io/parser_stdout-container-2":"container-2-stdout-parser","fluentbit.io/parser_stderr-container-2":"container-2-stderr-parser","fluentbit.io/parser_stdout-container-3":"container-3-stdout-parser","fluentbit.io/parser_stderr-container-4":"container-4-stderr-parser","fluentbit.io/parser_stdout":"default-stdout-parser","fluentbit.io/parser_stderr":"default-stderr-parser"},"container_name":"container-5"}}] diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/out/annotations-parser/annotations-parser_regex-with-time_invalid-text-1.out b/src/fluent-bit/tests/runtime/data/kubernetes/out/annotations-parser/annotations-parser_regex-with-time_invalid-text-1.out new file mode 100644 index 000000000..a563c4e46 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/out/annotations-parser/annotations-parser_regex-with-time_invalid-text-1.out @@ -0,0 +1 @@ +[1554141513.598656,{"log":"2019-04-05T14:54:23.584269761 - Simple text\n","stream":"stdout","kubernetes":{"pod_name":"regex-with-time","namespace_name":"annotations-parser","annotations":{"fluentbit.io/parser":"regex-parser-with-time"},"container_name":"invalid-text-1"}}] diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/out/annotations-parser/annotations-parser_regex-with-time_text.out b/src/fluent-bit/tests/runtime/data/kubernetes/out/annotations-parser/annotations-parser_regex-with-time_text.out new file mode 100644 index 000000000..dad7fcda5 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/out/annotations-parser/annotations-parser_regex-with-time_text.out @@ -0,0 +1 @@ +[1554476063.584270,{"stream":"stdout","message":"Simple text","kubernetes":{"pod_name":"regex-with-time","namespace_name":"annotations-parser","annotations":{"fluentbit.io/parser":"regex-parser-with-time"},"container_name":"text"}}] diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/out/annotations-parser/annotations-parser_stderr_text_stderr.out b/src/fluent-bit/tests/runtime/data/kubernetes/out/annotations-parser/annotations-parser_stderr_text_stderr.out new file mode 100644 index 000000000..85954f7ba --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/out/annotations-parser/annotations-parser_stderr_text_stderr.out @@ -0,0 +1 @@ +[1554476063.584270,{"stream":"stderr","default_stderr_parser":"Simple text","kubernetes":{"pod_name":"stderr","namespace_name":"annotations-parser","annotations":{"fluentbit.io/parser_stderr":"default-stderr-parser"},"container_name":"text"}}] diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/out/annotations-parser/annotations-parser_stderr_text_stdout.out b/src/fluent-bit/tests/runtime/data/kubernetes/out/annotations-parser/annotations-parser_stderr_text_stdout.out new file mode 100644 index 000000000..a56cb8092 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/out/annotations-parser/annotations-parser_stderr_text_stdout.out @@ -0,0 +1 @@ +[1554141513.598656,{"log":"[2019-04-05T14:54:23.584269761] Simple text\n","stream":"stdout","kubernetes":{"pod_name":"stderr","namespace_name":"annotations-parser","annotations":{"fluentbit.io/parser_stderr":"default-stderr-parser"},"container_name":"text"}}] diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/out/annotations-parser/annotations-parser_stdout_text_stderr.out b/src/fluent-bit/tests/runtime/data/kubernetes/out/annotations-parser/annotations-parser_stdout_text_stderr.out new file mode 100644 index 000000000..65e31d0d1 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/out/annotations-parser/annotations-parser_stdout_text_stderr.out @@ -0,0 +1 @@ +[1554141513.698657,{"log":"[2019-04-05T14:54:23.584269761] Simple text\n","stream":"stderr","kubernetes":{"pod_name":"stdout","namespace_name":"annotations-parser","annotations":{"fluentbit.io/parser_stdout":"default-stdout-parser"},"container_name":"text"}}] diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/out/annotations-parser/annotations-parser_stdout_text_stdout.out b/src/fluent-bit/tests/runtime/data/kubernetes/out/annotations-parser/annotations-parser_stdout_text_stdout.out new file mode 100644 index 000000000..07c2d9134 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/out/annotations-parser/annotations-parser_stdout_text_stdout.out @@ -0,0 +1 @@ +[1554476063.584270,{"stream":"stdout","default_stdout_parser":"Simple text","kubernetes":{"pod_name":"stdout","namespace_name":"annotations-parser","annotations":{"fluentbit.io/parser_stdout":"default-stdout-parser"},"container_name":"text"}}] diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/out/annotations/annotations_invalid_text.out b/src/fluent-bit/tests/runtime/data/kubernetes/out/annotations/annotations_invalid_text.out new file mode 100644 index 000000000..900560d8a --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/out/annotations/annotations_invalid_text.out @@ -0,0 +1 @@ +[1554141513.598656,{"log":"Simple text\n","stream":"stdout","kubernetes":{"pod_name":"invalid","namespace_name":"annotations","annotations":{"fluentbit.io/invalid":"true","fluentbit.io/exclude_":"true","fluentbit.io/exclude_invalid":"true","fluentbit.io/exclude_stdout-":"true","fluentbit.io/exclude-":"true","fluentbit.io/exclude+":"true"},"container_name":"text"}}] diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/out/core/core_base_fluent-bit.out b/src/fluent-bit/tests/runtime/data/kubernetes/out/core/core_base_fluent-bit.out new file mode 100644 index 000000000..6a1a3ac5c --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/out/core/core_base_fluent-bit.out @@ -0,0 +1 @@ +[1554141513.598656,{"log":"Fluent Bit is logging\n","stream":"stdout","kubernetes":{"pod_name":"base","namespace_name":"core","pod_id":"e9f2963f-55f2-11e9-84c5-02e422b8a84a","labels":{"app.kubernetes.io/name":"fluent-bit"},"annotations":{"prometheus.io/path":"/api/v1/metrics/prometheus","prometheus.io/port":"2020","prometheus.io/scrape":"true"},"host":"ip-10-49-18-80.eu-west-1.compute.internal","container_name":"fluent-bit","docker_id":"c9898099f6d235126d564ed38a020007ea7a6fac6e25e718de683c9dd0076c16","container_hash":"fluent/fluent-bit@sha256:7ac0fd3569af866e9a6a22eb592744200d2dbe098cf066162453f8d0b06c531f","container_image":"fluent/fluent-bit:latest"}}] diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/out/core/core_no-meta_text.out b/src/fluent-bit/tests/runtime/data/kubernetes/out/core/core_no-meta_text.out new file mode 100644 index 000000000..f72dc98ba --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/out/core/core_no-meta_text.out @@ -0,0 +1 @@ +[1554141513.598656,{"log":"Simple text\n","stream":"stdout","kubernetes":{"pod_name":"no-meta","namespace_name":"core","container_name":"text"}}] diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/out/core/core_unescaping_json.out b/src/fluent-bit/tests/runtime/data/kubernetes/out/core/core_unescaping_json.out new file mode 100644 index 000000000..8063a7fa3 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/out/core/core_unescaping_json.out @@ -0,0 +1 @@ +[1554141513.598656,{"log":"{\"stringified-json\":\"{\\\"stringified-json\\\":\\\"{\\\\\\\"stringified-json\\\\\\\":\\\\\\\"\\\\\\\"}\\\"}\",\"text\":\"quoted \\\"text with embedded quoted \\\\\\\"text\\\\\\\"\\\"\"}\n","stream":"stdout","kubernetes":{"pod_name":"unescaping","namespace_name":"core","container_name":"json"}}] diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/out/core/core_unescaping_text.out b/src/fluent-bit/tests/runtime/data/kubernetes/out/core/core_unescaping_text.out new file mode 100644 index 000000000..f9dad5be2 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/out/core/core_unescaping_text.out @@ -0,0 +1 @@ +[1554141513.598656,{"log":"Text with embedded quoted \"text with embedded quoted \\\"text with embedded quoted \\\\\\\"text\\\\\\\"\\\"\"\n","stream":"stdout","kubernetes":{"pod_name":"unescaping","namespace_name":"core","container_name":"text"}}] diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/out/kairosdb-914055854-b63vq.out b/src/fluent-bit/tests/runtime/data/kubernetes/out/kairosdb-914055854-b63vq.out new file mode 100644 index 000000000..efaaf0d8b --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/out/kairosdb-914055854-b63vq.out @@ -0,0 +1 @@ +[1541358747.464754,{"_UID":"1000","_GID":"1000","_CAP_EFFECTIVE":"0","_SELINUX_CONTEXT":"unconfined\n","_AUDIT_LOGINUID":"1000","_SYSTEMD_OWNER_UID":"1000","_SYSTEMD_SLICE":"user-1000.slice","_SYSTEMD_USER_SLICE":"-.slice","_BOOT_ID":"e3abb4093a904a169e2ac21af241392f","_MACHINE_ID":"28d81df105e641e9b7d85e9d41433662","_HOSTNAME":"cube","PRIORITY":"6","CODE_FILE":"filter_kubernetes.c","CODE_LINE":"456","CODE_FUNC":"flb_test_systemd_logs","CONTAINER_NAME":"k8s_kairosdb_kairosdb-914055854-b63vq_default_d6c53deb-05a4-11e8-a8c4-080027435fb7_23","CONTAINER_TAG":"","CONTAINER_ID":"56e257661383","CONTAINER_ID_FULL":"56e257661383836fac4cd90a23ee8a7a02ee1538c8f35657d1a90f3de1065a22","MESSAGE":"08:58:45.839 [qtp151442075-47] DEBUG [HttpParser.java:281] - filled 157/157","KUBE_TEST":"2018","SYSLOG_IDENTIFIER":"flb-rt-filter_kubernetes","_TRANSPORT":"journal","_PID":"32318","_COMM":"flb-rt-filter_k","_EXE":"build/bin/flb-rt-filter_kubernetes","_CMDLINE":"bin/flb-rt-filter_kubernetes","_AUDIT_SESSION":"7417","_SYSTEMD_CGROUP":"/user.slice/user-1000.slice/session-7417.scope","_SYSTEMD_SESSION":"7417","_SYSTEMD_UNIT":"session-7417.scope","_SYSTEMD_INVOCATION_ID":"899c0f7241ae4db0af3cb4088dca76b0","_SOURCE_REALTIME_TIMESTAMP":"1541358747464738","kubernetes":{"pod_name":"kairosdb-914055854-b63vq","namespace_name":"default","pod_id":"d6c53deb-05a4-11e8-a8c4-080027435fb7","labels":{"name":"kairosdb","pod-template-hash":"914055854"},"annotations":{"kubernetes.io/created-by":"{\"kind\":\"SerializedReference\",\"apiVersion\":\"v1\",\"reference\":{\"kind\":\"ReplicaSet\",\"namespace\":\"default\",\"name\":\"kairosdb-914055854\",\"uid\":\"d6c2f841-05a4-11e8-a8c4-080027435fb7\",\"apiVersion\":\"extensions\",\"resourceVersion\":\"1592\"}}\n"},"host":"192.168.10.169","container_name":"kairosdb","docker_id":"02d45318f220ad01109d13df31fbbb0668b87fdd06ef6607abfdecf5eba4b311","container_hash":"davradocker/docker-kairosdb@sha256:2ee07e3f3ba61b96597c78ab0aa7b837d771b375888b020da7a7316bb524b6e8","container_image":"davradocker/docker-kairosdb:1.1.1_9"}}] diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/out/options/options_k8s-logging-exclude-disabled_text_stderr.out b/src/fluent-bit/tests/runtime/data/kubernetes/out/options/options_k8s-logging-exclude-disabled_text_stderr.out new file mode 100644 index 000000000..2b148329d --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/out/options/options_k8s-logging-exclude-disabled_text_stderr.out @@ -0,0 +1 @@ +[1554141513.698657,{"log":"Simple text\n","stream":"stderr","kubernetes":{"pod_name":"k8s-logging-exclude-disabled","namespace_name":"options","annotations":{"fluentbit.io/exclude":"true"},"container_name":"text"}}] diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/out/options/options_k8s-logging-exclude-disabled_text_stdout.out b/src/fluent-bit/tests/runtime/data/kubernetes/out/options/options_k8s-logging-exclude-disabled_text_stdout.out new file mode 100644 index 000000000..4a87d121a --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/out/options/options_k8s-logging-exclude-disabled_text_stdout.out @@ -0,0 +1 @@ +[1554141513.598656,{"log":"Simple text\n","stream":"stdout","kubernetes":{"pod_name":"k8s-logging-exclude-disabled","namespace_name":"options","annotations":{"fluentbit.io/exclude":"true"},"container_name":"text"}}] diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/out/options/options_k8s-logging-parser-disabled_text_stderr.out b/src/fluent-bit/tests/runtime/data/kubernetes/out/options/options_k8s-logging-parser-disabled_text_stderr.out new file mode 100644 index 000000000..0c35402fd --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/out/options/options_k8s-logging-parser-disabled_text_stderr.out @@ -0,0 +1 @@ +[1554141513.698657,{"log":"Simple text\n","stream":"stderr","kubernetes":{"pod_name":"k8s-logging-parser-disabled","namespace_name":"options","annotations":{"fluentbit.io/parser":"default-parser"},"container_name":"text"}}] diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/out/options/options_k8s-logging-parser-disabled_text_stdout.out b/src/fluent-bit/tests/runtime/data/kubernetes/out/options/options_k8s-logging-parser-disabled_text_stdout.out new file mode 100644 index 000000000..5eb809663 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/out/options/options_k8s-logging-parser-disabled_text_stdout.out @@ -0,0 +1 @@ +[1554141513.598656,{"log":"Simple text\n","stream":"stdout","kubernetes":{"pod_name":"k8s-logging-parser-disabled","namespace_name":"options","annotations":{"fluentbit.io/parser":"default-parser"},"container_name":"text"}}] diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/out/options/options_keep-log-disabled_json.out b/src/fluent-bit/tests/runtime/data/kubernetes/out/options/options_keep-log-disabled_json.out new file mode 100644 index 000000000..6f95c3ba3 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/out/options/options_keep-log-disabled_json.out @@ -0,0 +1 @@ +[1554141513.598656,{"stream":"stdout","text":"Simple text","kubernetes":{"pod_name":"keep-log-disabled","namespace_name":"options","container_name":"json"}}] diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/out/options/options_keep-log-enabled_json.out b/src/fluent-bit/tests/runtime/data/kubernetes/out/options/options_keep-log-enabled_json.out new file mode 100644 index 000000000..b09efae85 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/out/options/options_keep-log-enabled_json.out @@ -0,0 +1 @@ +[1554141513.598656,{"log":"{\"text\":\"Simple text\"}\n","stream":"stdout","text":"Simple text","kubernetes":{"pod_name":"keep-log-enabled","namespace_name":"options","container_name":"json"}}] diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/out/options/options_merge-log-disabled_json.out b/src/fluent-bit/tests/runtime/data/kubernetes/out/options/options_merge-log-disabled_json.out new file mode 100644 index 000000000..7d28a9c9b --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/out/options/options_merge-log-disabled_json.out @@ -0,0 +1 @@ +[1554141513.598656,{"log":"{\"text\":\"Simple text\"}\n","stream":"stdout","kubernetes":{"pod_name":"merge-log-disabled","namespace_name":"options","container_name":"json"}}] diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/out/options/options_merge-log-enabled_invalid-json.out b/src/fluent-bit/tests/runtime/data/kubernetes/out/options/options_merge-log-enabled_invalid-json.out new file mode 100644 index 000000000..66cfe5474 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/out/options/options_merge-log-enabled_invalid-json.out @@ -0,0 +1 @@ +[1554141513.598656,{"log":"{\"text\":\"Simple text\"]\n","stream":"stdout","kubernetes":{"pod_name":"merge-log-enabled","namespace_name":"options","container_name":"invalid-json"}}] diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/out/options/options_merge-log-enabled_json.out b/src/fluent-bit/tests/runtime/data/kubernetes/out/options/options_merge-log-enabled_json.out new file mode 100644 index 000000000..257177172 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/out/options/options_merge-log-enabled_json.out @@ -0,0 +1 @@ +[1554141513.598656,{"log":"{\"text\":\"Simple text\"}\n","stream":"stdout","text":"Simple text","kubernetes":{"pod_name":"merge-log-enabled","namespace_name":"options","container_name":"json"}}] diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/out/options/options_merge-log-enabled_text.out b/src/fluent-bit/tests/runtime/data/kubernetes/out/options/options_merge-log-enabled_text.out new file mode 100644 index 000000000..7c6363bc4 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/out/options/options_merge-log-enabled_text.out @@ -0,0 +1 @@ +[1554141513.598656,{"log":"Simple text\n","stream":"stdout","kubernetes":{"pod_name":"merge-log-enabled","namespace_name":"options","container_name":"text"}}] diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/out/options/options_merge-log-key_json.out b/src/fluent-bit/tests/runtime/data/kubernetes/out/options/options_merge-log-key_json.out new file mode 100644 index 000000000..130e92f28 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/out/options/options_merge-log-key_json.out @@ -0,0 +1 @@ +[1554141513.598656,{"log":"{\"text\":\"Simple text\"}\n","stream":"stdout","merge-log-key":{"text":"Simple text"},"kubernetes":{"pod_name":"merge-log-key","namespace_name":"options","container_name":"json"}}] diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/out/options/options_merge-log-trim-disabled_json.out b/src/fluent-bit/tests/runtime/data/kubernetes/out/options/options_merge-log-trim-disabled_json.out new file mode 100644 index 000000000..0fee86454 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/out/options/options_merge-log-trim-disabled_json.out @@ -0,0 +1 @@ +[1554141513.598656,{"log":"{\"text\":\"Simple text\n\"}\n","stream":"stdout","text":"Simple text\n","kubernetes":{"pod_name":"merge-log-trim-disabled","namespace_name":"options","container_name":"json"}}] diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/out/options/options_merge-log-trim-enabled_json.out b/src/fluent-bit/tests/runtime/data/kubernetes/out/options/options_merge-log-trim-enabled_json.out new file mode 100644 index 000000000..adcdf765b --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/out/options/options_merge-log-trim-enabled_json.out @@ -0,0 +1 @@ +[1554141513.598656,{"log":"{\"text\":\"Simple text\n\"}\n","stream":"stdout","text":"Simple text","kubernetes":{"pod_name":"merge-log-trim-enabled","namespace_name":"options","container_name":"json"}}] diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/out/options/options_use-kubelet-disabled_fluent-bit.out b/src/fluent-bit/tests/runtime/data/kubernetes/out/options/options_use-kubelet-disabled_fluent-bit.out new file mode 100644 index 000000000..699da8bd6 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/out/options/options_use-kubelet-disabled_fluent-bit.out @@ -0,0 +1 @@ +[1554141513.598656,{"log":"Fluent Bit is logging\n","stream":"stdout","kubernetes":{"pod_name":"use-kubelet-disabled","namespace_name":"options","pod_id":"e9f2963f-55f2-11e9-84c5-02e422b8a84a","labels":{"app.kubernetes.io/name":"fluent-bit"},"annotations":{"prometheus.io/path":"/api/v1/metrics/prometheus","prometheus.io/port":"2020","prometheus.io/scrape":"true"},"host":"ip-10-49-18-80.eu-west-1.compute.internal","container_name":"fluent-bit","docker_id":"c9898099f6d235126d564ed38a020007ea7a6fac6e25e718de683c9dd0076c16","container_hash":"fluent/fluent-bit@sha256:7ac0fd3569af866e9a6a22eb592744200d2dbe098cf066162453f8d0b06c531f","container_image":"fluent/fluent-bit:latest"}}] diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/out/options/options_use-kubelet-enabled_fluent-bit.out b/src/fluent-bit/tests/runtime/data/kubernetes/out/options/options_use-kubelet-enabled_fluent-bit.out new file mode 100644 index 000000000..dbd97f1ae --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/out/options/options_use-kubelet-enabled_fluent-bit.out @@ -0,0 +1 @@ +[1554141513.598656,{"log":"Fluent Bit is logging\n","stream":"stdout","kubernetes":{"pod_name":"use-kubelet-enabled","namespace_name":"options","pod_id":"e9f2963f-55f2-11e9-84c5-02e422b8a84a","labels":{"app.kubernetes.io/name":"fluent-bit"},"annotations":{"prometheus.io/path":"/api/v1/metrics/prometheus","prometheus.io/port":"2020","prometheus.io/scrape":"true"},"host":"ip-10-49-18-80.eu-west-1.compute.internal","container_name":"fluent-bit","docker_id":"c9898099f6d235126d564ed38a020007ea7a6fac6e25e718de683c9dd0076c16","container_hash":"fluent/fluent-bit@sha256:7ac0fd3569af866e9a6a22eb592744200d2dbe098cf066162453f8d0b06c531f","container_image":"fluent/fluent-bit:latest"}}] diff --git a/src/fluent-bit/tests/runtime/data/kubernetes/parsers.conf b/src/fluent-bit/tests/runtime/data/kubernetes/parsers.conf new file mode 100644 index 000000000..40014ec36 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/kubernetes/parsers.conf @@ -0,0 +1,88 @@ +[PARSER] + Name docker + Format json + Time_Key time + Time_Format %Y-%m-%dT%H:%M:%S.%L%z + +[PARSER] + Name kubernetes-tag + Format regex + Regex ^(?<namespace_name>[^.]+).(?<pod_name>[^.]+).(?<container_name>[^.]+)$ + +[PARSER] + Name regex-parser-without-time + Format regex + Regex ^(?<message>.+)$ + +[PARSER] + Name regex-parser-with-time + Format regex + Regex ^\[(?<time>[^\]]+)\] (?<message>.+)$ + Time_Key time + Time_Format %Y-%m-%dT%H:%M:%S.%L + +[PARSER] + Name json-parser-without-time + Format json + +[PARSER] + Name json-parser-with-time + Format json + Time_Key time + Time_Format %Y-%m-%dT%H:%M:%S.%L + +[PARSER] + Name default-parser + Format regex + Regex ^\[(?<time>[^\]]+)\] (?<default_parser>.+)$ + Time_Key time + Time_Format %Y-%m-%dT%H:%M:%S.%L + +[PARSER] + Name default-stdout-parser + Format regex + Regex ^\[(?<time>[^\]]+)\] (?<default_stdout_parser>.+)$ + Time_Key time + Time_Format %Y-%m-%dT%H:%M:%S.%L + +[PARSER] + Name default-stderr-parser + Format regex + Regex ^\[(?<time>[^\]]+)\] (?<default_stderr_parser>.+)$ + Time_Key time + Time_Format %Y-%m-%dT%H:%M:%S.%L + +[PARSER] + Name container-1-parser + Format regex + Regex ^\[(?<time>[^\]]+)\] (?<container_1_parser>.+)$ + Time_Key time + Time_Format %Y-%m-%dT%H:%M:%S.%L + +[PARSER] + Name container-2-stdout-parser + Format regex + Regex ^\[(?<time>[^\]]+)\] (?<container_2_stdout_parser>.+)$ + Time_Key time + Time_Format %Y-%m-%dT%H:%M:%S.%L + +[PARSER] + Name container-2-stderr-parser + Format regex + Regex ^\[(?<time>[^\]]+)\] (?<container_2_stderr_parser>.+)$ + Time_Key time + Time_Format %Y-%m-%dT%H:%M:%S.%L + +[PARSER] + Name container-3-stdout-parser + Format regex + Regex ^\[(?<time>[^\]]+)\] (?<container_3_stdout_parser>.+)$ + Time_Key time + Time_Format %Y-%m-%dT%H:%M:%S.%L + +[PARSER] + Name container-4-stderr-parser + Format regex + Regex ^\[(?<time>[^\]]+)\] (?<container_4_stderr_parser>.+)$ + Time_Key time + Time_Format %Y-%m-%dT%H:%M:%S.%L diff --git a/src/fluent-bit/tests/runtime/data/loki/labelmap.json b/src/fluent-bit/tests/runtime/data/loki/labelmap.json new file mode 100644 index 000000000..b24ad434c --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/loki/labelmap.json @@ -0,0 +1,10 @@ +{ + "kubernetes": { + "container_name": "container", + "pod_name": "pod", + "namespace_name": "namespace", + "labels" : { + "team": "team" + } + } +} diff --git a/src/fluent-bit/tests/runtime/data/podman/cgroupv2/42/net/dev b/src/fluent-bit/tests/runtime/data/podman/cgroupv2/42/net/dev new file mode 100644 index 000000000..835131a90 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/podman/cgroupv2/42/net/dev @@ -0,0 +1,6 @@ +Inter-| Receive | Transmit + face |bytes packets errs drop fifo frame compressed multicast|bytes packets errs drop fifo colls carrier compressed + lo: 34275905 6362 0 0 0 0 0 0 34275905 6362 0 0 0 0 0 0 +enp6s0: 9089298401 6228223 0 0 0 0 0 5669 173774157 2534993 0 0 0 0 0 0 +wlp5s0: 2802033 11801 0 0 0 0 0 0 790512 6492 0 0 0 0 0 0 +virbr0: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 diff --git a/src/fluent-bit/tests/runtime/data/podman/cgroupv2/cgroup.controllers b/src/fluent-bit/tests/runtime/data/podman/cgroupv2/cgroup.controllers new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/podman/cgroupv2/cgroup.controllers diff --git a/src/fluent-bit/tests/runtime/data/podman/cgroupv2/config.json b/src/fluent-bit/tests/runtime/data/podman/cgroupv2/config.json new file mode 100644 index 000000000..dcf6ab4ac --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/podman/cgroupv2/config.json @@ -0,0 +1 @@ +[{"id":"8a19d6058bfbe88cd0548eba9047d94c70161f5d74b545c7504b2f27491686d9","names":["determined_mcnulty"],"image":"27941809078cc9b2802deb2b0bb6feed6c236cde01e487f200e24653533701ee","layer":"633c06ce1a553c72c4fbf12686acd5129a453f57b73385f8e7afc975fa657d86","metadata":"{\"image-name\":\"docker.io/library/ubuntu:latest\",\"image-id\":\"27941809078cc9b2802deb2b0bb6feed6c236cde01e487f200e24653533701ee\",\"name\":\"determined_mcnulty\",\"created-at\":1657980361}","created":"2022-07-16T14:06:01.521179687Z","flags":{"MountLabel":"system_u:object_r:container_file_t:s0:c664,c969","ProcessLabel":"system_u:system_r:container_t:s0:c664,c969"}}] diff --git a/src/fluent-bit/tests/runtime/data/podman/cgroupv2/libpod-8a19d6058bfbe88cd0548eba9047d94c70161f5d74b545c7504b2f27491686d9/containers/cgroup.procs b/src/fluent-bit/tests/runtime/data/podman/cgroupv2/libpod-8a19d6058bfbe88cd0548eba9047d94c70161f5d74b545c7504b2f27491686d9/containers/cgroup.procs new file mode 100644 index 000000000..d81cc0710 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/podman/cgroupv2/libpod-8a19d6058bfbe88cd0548eba9047d94c70161f5d74b545c7504b2f27491686d9/containers/cgroup.procs @@ -0,0 +1 @@ +42 diff --git a/src/fluent-bit/tests/runtime/data/podman/cgroupv2/libpod-8a19d6058bfbe88cd0548eba9047d94c70161f5d74b545c7504b2f27491686d9/memory.current b/src/fluent-bit/tests/runtime/data/podman/cgroupv2/libpod-8a19d6058bfbe88cd0548eba9047d94c70161f5d74b545c7504b2f27491686d9/memory.current new file mode 100644 index 000000000..d81cc0710 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/podman/cgroupv2/libpod-8a19d6058bfbe88cd0548eba9047d94c70161f5d74b545c7504b2f27491686d9/memory.current @@ -0,0 +1 @@ +42 diff --git a/src/fluent-bit/tests/runtime/data/podman/garbage/42/net/dev b/src/fluent-bit/tests/runtime/data/podman/garbage/42/net/dev new file mode 100644 index 000000000..9ce7649da --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/podman/garbage/42/net/dev @@ -0,0 +1 @@ +garbage diff --git a/src/fluent-bit/tests/runtime/data/podman/garbage/config.json b/src/fluent-bit/tests/runtime/data/podman/garbage/config.json new file mode 100644 index 000000000..dcf6ab4ac --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/podman/garbage/config.json @@ -0,0 +1 @@ +[{"id":"8a19d6058bfbe88cd0548eba9047d94c70161f5d74b545c7504b2f27491686d9","names":["determined_mcnulty"],"image":"27941809078cc9b2802deb2b0bb6feed6c236cde01e487f200e24653533701ee","layer":"633c06ce1a553c72c4fbf12686acd5129a453f57b73385f8e7afc975fa657d86","metadata":"{\"image-name\":\"docker.io/library/ubuntu:latest\",\"image-id\":\"27941809078cc9b2802deb2b0bb6feed6c236cde01e487f200e24653533701ee\",\"name\":\"determined_mcnulty\",\"created-at\":1657980361}","created":"2022-07-16T14:06:01.521179687Z","flags":{"MountLabel":"system_u:object_r:container_file_t:s0:c664,c969","ProcessLabel":"system_u:system_r:container_t:s0:c664,c969"}}] diff --git a/src/fluent-bit/tests/runtime/data/podman/garbage/libpod-8a19d6058bfbe88cd0548eba9047d94c70161f5d74b545c7504b2f27491686d9/memory.current b/src/fluent-bit/tests/runtime/data/podman/garbage/libpod-8a19d6058bfbe88cd0548eba9047d94c70161f5d74b545c7504b2f27491686d9/memory.current new file mode 100644 index 000000000..9ce7649da --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/podman/garbage/libpod-8a19d6058bfbe88cd0548eba9047d94c70161f5d74b545c7504b2f27491686d9/memory.current @@ -0,0 +1 @@ +garbage diff --git a/src/fluent-bit/tests/runtime/data/podman/garbage/memory/libpod-8a19d6058bfbe88cd0548eba9047d94c70161f5d74b545c7504b2f27491686d9/memory.usage_in_bytes b/src/fluent-bit/tests/runtime/data/podman/garbage/memory/libpod-8a19d6058bfbe88cd0548eba9047d94c70161f5d74b545c7504b2f27491686d9/memory.usage_in_bytes new file mode 100644 index 000000000..9ce7649da --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/podman/garbage/memory/libpod-8a19d6058bfbe88cd0548eba9047d94c70161f5d74b545c7504b2f27491686d9/memory.usage_in_bytes @@ -0,0 +1 @@ +garbage diff --git a/src/fluent-bit/tests/runtime/data/podman/garbage_config/config.json b/src/fluent-bit/tests/runtime/data/podman/garbage_config/config.json new file mode 100644 index 000000000..9ce7649da --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/podman/garbage_config/config.json @@ -0,0 +1 @@ +garbage diff --git a/src/fluent-bit/tests/runtime/data/podman/no_config/libpod-8a19d6058bfbe88cd0548eba9047d94c70161f5d74b545c7504b2f27491686d9/memory.current b/src/fluent-bit/tests/runtime/data/podman/no_config/libpod-8a19d6058bfbe88cd0548eba9047d94c70161f5d74b545c7504b2f27491686d9/memory.current new file mode 100644 index 000000000..d81cc0710 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/podman/no_config/libpod-8a19d6058bfbe88cd0548eba9047d94c70161f5d74b545c7504b2f27491686d9/memory.current @@ -0,0 +1 @@ +42 diff --git a/src/fluent-bit/tests/runtime/data/podman/no_config/libpod-8a19d6058bfbe88cd0548eba9047d94c70161f5d74b545c7504b2f27491686d9/memory.usage_in_bytes b/src/fluent-bit/tests/runtime/data/podman/no_config/libpod-8a19d6058bfbe88cd0548eba9047d94c70161f5d74b545c7504b2f27491686d9/memory.usage_in_bytes new file mode 100644 index 000000000..d81cc0710 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/podman/no_config/libpod-8a19d6058bfbe88cd0548eba9047d94c70161f5d74b545c7504b2f27491686d9/memory.usage_in_bytes @@ -0,0 +1 @@ +42 diff --git a/src/fluent-bit/tests/runtime/data/podman/no_config/memory/libpod-8a19d6058bfbe88cd0548eba9047d94c70161f5d74b545c7504b2f27491686d9/memory.usage_in_bytes b/src/fluent-bit/tests/runtime/data/podman/no_config/memory/libpod-8a19d6058bfbe88cd0548eba9047d94c70161f5d74b545c7504b2f27491686d9/memory.usage_in_bytes new file mode 100644 index 000000000..d81cc0710 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/podman/no_config/memory/libpod-8a19d6058bfbe88cd0548eba9047d94c70161f5d74b545c7504b2f27491686d9/memory.usage_in_bytes @@ -0,0 +1 @@ +42 diff --git a/src/fluent-bit/tests/runtime/data/podman/no_proc/config.json b/src/fluent-bit/tests/runtime/data/podman/no_proc/config.json new file mode 100644 index 000000000..dcf6ab4ac --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/podman/no_proc/config.json @@ -0,0 +1 @@ +[{"id":"8a19d6058bfbe88cd0548eba9047d94c70161f5d74b545c7504b2f27491686d9","names":["determined_mcnulty"],"image":"27941809078cc9b2802deb2b0bb6feed6c236cde01e487f200e24653533701ee","layer":"633c06ce1a553c72c4fbf12686acd5129a453f57b73385f8e7afc975fa657d86","metadata":"{\"image-name\":\"docker.io/library/ubuntu:latest\",\"image-id\":\"27941809078cc9b2802deb2b0bb6feed6c236cde01e487f200e24653533701ee\",\"name\":\"determined_mcnulty\",\"created-at\":1657980361}","created":"2022-07-16T14:06:01.521179687Z","flags":{"MountLabel":"system_u:object_r:container_file_t:s0:c664,c969","ProcessLabel":"system_u:system_r:container_t:s0:c664,c969"}}] diff --git a/src/fluent-bit/tests/runtime/data/podman/no_proc/libpod-8a19d6058bfbe88cd0548eba9047d94c70161f5d74b545c7504b2f27491686d9/memory.current b/src/fluent-bit/tests/runtime/data/podman/no_proc/libpod-8a19d6058bfbe88cd0548eba9047d94c70161f5d74b545c7504b2f27491686d9/memory.current new file mode 100644 index 000000000..d81cc0710 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/podman/no_proc/libpod-8a19d6058bfbe88cd0548eba9047d94c70161f5d74b545c7504b2f27491686d9/memory.current @@ -0,0 +1 @@ +42 diff --git a/src/fluent-bit/tests/runtime/data/podman/no_proc/memory/libpod-8a19d6058bfbe88cd0548eba9047d94c70161f5d74b545c7504b2f27491686d9/memory.usage_in_bytes b/src/fluent-bit/tests/runtime/data/podman/no_proc/memory/libpod-8a19d6058bfbe88cd0548eba9047d94c70161f5d74b545c7504b2f27491686d9/memory.usage_in_bytes new file mode 100644 index 000000000..d81cc0710 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/podman/no_proc/memory/libpod-8a19d6058bfbe88cd0548eba9047d94c70161f5d74b545c7504b2f27491686d9/memory.usage_in_bytes @@ -0,0 +1 @@ +42 diff --git a/src/fluent-bit/tests/runtime/data/podman/no_sysfs/42/net/dev b/src/fluent-bit/tests/runtime/data/podman/no_sysfs/42/net/dev new file mode 100644 index 000000000..835131a90 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/podman/no_sysfs/42/net/dev @@ -0,0 +1,6 @@ +Inter-| Receive | Transmit + face |bytes packets errs drop fifo frame compressed multicast|bytes packets errs drop fifo colls carrier compressed + lo: 34275905 6362 0 0 0 0 0 0 34275905 6362 0 0 0 0 0 0 +enp6s0: 9089298401 6228223 0 0 0 0 0 5669 173774157 2534993 0 0 0 0 0 0 +wlp5s0: 2802033 11801 0 0 0 0 0 0 790512 6492 0 0 0 0 0 0 +virbr0: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 diff --git a/src/fluent-bit/tests/runtime/data/podman/no_sysfs/config.json b/src/fluent-bit/tests/runtime/data/podman/no_sysfs/config.json new file mode 100644 index 000000000..dcf6ab4ac --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/podman/no_sysfs/config.json @@ -0,0 +1 @@ +[{"id":"8a19d6058bfbe88cd0548eba9047d94c70161f5d74b545c7504b2f27491686d9","names":["determined_mcnulty"],"image":"27941809078cc9b2802deb2b0bb6feed6c236cde01e487f200e24653533701ee","layer":"633c06ce1a553c72c4fbf12686acd5129a453f57b73385f8e7afc975fa657d86","metadata":"{\"image-name\":\"docker.io/library/ubuntu:latest\",\"image-id\":\"27941809078cc9b2802deb2b0bb6feed6c236cde01e487f200e24653533701ee\",\"name\":\"determined_mcnulty\",\"created-at\":1657980361}","created":"2022-07-16T14:06:01.521179687Z","flags":{"MountLabel":"system_u:object_r:container_file_t:s0:c664,c969","ProcessLabel":"system_u:system_r:container_t:s0:c664,c969"}}] diff --git a/src/fluent-bit/tests/runtime/data/podman/regular/42/net/dev b/src/fluent-bit/tests/runtime/data/podman/regular/42/net/dev new file mode 100644 index 000000000..835131a90 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/podman/regular/42/net/dev @@ -0,0 +1,6 @@ +Inter-| Receive | Transmit + face |bytes packets errs drop fifo frame compressed multicast|bytes packets errs drop fifo colls carrier compressed + lo: 34275905 6362 0 0 0 0 0 0 34275905 6362 0 0 0 0 0 0 +enp6s0: 9089298401 6228223 0 0 0 0 0 5669 173774157 2534993 0 0 0 0 0 0 +wlp5s0: 2802033 11801 0 0 0 0 0 0 790512 6492 0 0 0 0 0 0 +virbr0: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 diff --git a/src/fluent-bit/tests/runtime/data/podman/regular/config.json b/src/fluent-bit/tests/runtime/data/podman/regular/config.json new file mode 100644 index 000000000..dcf6ab4ac --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/podman/regular/config.json @@ -0,0 +1 @@ +[{"id":"8a19d6058bfbe88cd0548eba9047d94c70161f5d74b545c7504b2f27491686d9","names":["determined_mcnulty"],"image":"27941809078cc9b2802deb2b0bb6feed6c236cde01e487f200e24653533701ee","layer":"633c06ce1a553c72c4fbf12686acd5129a453f57b73385f8e7afc975fa657d86","metadata":"{\"image-name\":\"docker.io/library/ubuntu:latest\",\"image-id\":\"27941809078cc9b2802deb2b0bb6feed6c236cde01e487f200e24653533701ee\",\"name\":\"determined_mcnulty\",\"created-at\":1657980361}","created":"2022-07-16T14:06:01.521179687Z","flags":{"MountLabel":"system_u:object_r:container_file_t:s0:c664,c969","ProcessLabel":"system_u:system_r:container_t:s0:c664,c969"}}] diff --git a/src/fluent-bit/tests/runtime/data/podman/regular/libpod-8a19d6058bfbe88cd0548eba9047d94c70161f5d74b545c7504b2f27491686d9/memory.current b/src/fluent-bit/tests/runtime/data/podman/regular/libpod-8a19d6058bfbe88cd0548eba9047d94c70161f5d74b545c7504b2f27491686d9/memory.current new file mode 100644 index 000000000..d81cc0710 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/podman/regular/libpod-8a19d6058bfbe88cd0548eba9047d94c70161f5d74b545c7504b2f27491686d9/memory.current @@ -0,0 +1 @@ +42 diff --git a/src/fluent-bit/tests/runtime/data/podman/regular/memory/libpod-8a19d6058bfbe88cd0548eba9047d94c70161f5d74b545c7504b2f27491686d9/memory.usage_in_bytes b/src/fluent-bit/tests/runtime/data/podman/regular/memory/libpod-8a19d6058bfbe88cd0548eba9047d94c70161f5d74b545c7504b2f27491686d9/memory.usage_in_bytes new file mode 100644 index 000000000..d81cc0710 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/podman/regular/memory/libpod-8a19d6058bfbe88cd0548eba9047d94c70161f5d74b545c7504b2f27491686d9/memory.usage_in_bytes @@ -0,0 +1 @@ +42 diff --git a/src/fluent-bit/tests/runtime/data/podman/regular/systemd/libpod-8a19d6058bfbe88cd0548eba9047d94c70161f5d74b545c7504b2f27491686d9/cgroup.procs b/src/fluent-bit/tests/runtime/data/podman/regular/systemd/libpod-8a19d6058bfbe88cd0548eba9047d94c70161f5d74b545c7504b2f27491686d9/cgroup.procs new file mode 100644 index 000000000..a2a31c002 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/podman/regular/systemd/libpod-8a19d6058bfbe88cd0548eba9047d94c70161f5d74b545c7504b2f27491686d9/cgroup.procs @@ -0,0 +1,3 @@ +42 +73 +12 diff --git a/src/fluent-bit/tests/runtime/data/podman/reversed/42/net/dev b/src/fluent-bit/tests/runtime/data/podman/reversed/42/net/dev new file mode 100644 index 000000000..835131a90 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/podman/reversed/42/net/dev @@ -0,0 +1,6 @@ +Inter-| Receive | Transmit + face |bytes packets errs drop fifo frame compressed multicast|bytes packets errs drop fifo colls carrier compressed + lo: 34275905 6362 0 0 0 0 0 0 34275905 6362 0 0 0 0 0 0 +enp6s0: 9089298401 6228223 0 0 0 0 0 5669 173774157 2534993 0 0 0 0 0 0 +wlp5s0: 2802033 11801 0 0 0 0 0 0 790512 6492 0 0 0 0 0 0 +virbr0: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 diff --git a/src/fluent-bit/tests/runtime/data/podman/reversed/config.json b/src/fluent-bit/tests/runtime/data/podman/reversed/config.json new file mode 100644 index 000000000..6cbcc6907 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/podman/reversed/config.json @@ -0,0 +1 @@ +[{"id":"8a19d6058bfbe88cd0548eba9047d94c70161f5d74b545c7504b2f27491686d9","names":["determined_mcnulty"],"image":"27941809078cc9b2802deb2b0bb6feed6c236cde01e487f200e24653533701ee","layer":"633c06ce1a553c72c4fbf12686acd5129a453f57b73385f8e7afc975fa657d86","metadata":"{\"image-id\":\"27941809078cc9b2802deb2b0bb6feed6c236cde01e487f200e24653533701ee\",\"name\":\"determined_mcnulty\",\"created-at\":1657980361,\"image-name\":\"docker.io/library/ubuntu:latest\"}","created":"2022-07-16T14:06:01.521179687Z","flags":{"MountLabel":"system_u:object_r:container_file_t:s0:c664,c969","ProcessLabel":"system_u:system_r:container_t:s0:c664,c969"}}] diff --git a/src/fluent-bit/tests/runtime/data/podman/reversed/libpod-8a19d6058bfbe88cd0548eba9047d94c70161f5d74b545c7504b2f27491686d9/memory.current b/src/fluent-bit/tests/runtime/data/podman/reversed/libpod-8a19d6058bfbe88cd0548eba9047d94c70161f5d74b545c7504b2f27491686d9/memory.current new file mode 100644 index 000000000..d81cc0710 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/podman/reversed/libpod-8a19d6058bfbe88cd0548eba9047d94c70161f5d74b545c7504b2f27491686d9/memory.current @@ -0,0 +1 @@ +42 diff --git a/src/fluent-bit/tests/runtime/data/podman/reversed/memory/libpod-8a19d6058bfbe88cd0548eba9047d94c70161f5d74b545c7504b2f27491686d9/memory.usage_in_bytes b/src/fluent-bit/tests/runtime/data/podman/reversed/memory/libpod-8a19d6058bfbe88cd0548eba9047d94c70161f5d74b545c7504b2f27491686d9/memory.usage_in_bytes new file mode 100644 index 000000000..d81cc0710 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/podman/reversed/memory/libpod-8a19d6058bfbe88cd0548eba9047d94c70161f5d74b545c7504b2f27491686d9/memory.usage_in_bytes @@ -0,0 +1 @@ +42 diff --git a/src/fluent-bit/tests/runtime/data/podman/reversed/systemd/libpod-8a19d6058bfbe88cd0548eba9047d94c70161f5d74b545c7504b2f27491686d9/cgroup.procs b/src/fluent-bit/tests/runtime/data/podman/reversed/systemd/libpod-8a19d6058bfbe88cd0548eba9047d94c70161f5d74b545c7504b2f27491686d9/cgroup.procs new file mode 100644 index 000000000..a2a31c002 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/podman/reversed/systemd/libpod-8a19d6058bfbe88cd0548eba9047d94c70161f5d74b545c7504b2f27491686d9/cgroup.procs @@ -0,0 +1,3 @@ +42 +73 +12 diff --git a/src/fluent-bit/tests/runtime/data/stackdriver/json.h b/src/fluent-bit/tests/runtime/data/stackdriver/json.h new file mode 100755 index 000000000..146dd5324 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/stackdriver/json.h @@ -0,0 +1,12 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +#define JSON "[" \ + "1448403340," \ + "{" \ + "\"key_0\": false," \ + "\"key_1\": true," \ + "\"key_2\": \"some string\"," \ + "\"key_3\": 0.12345678," \ + "\"key.4\": 5000," \ + "\"END_KEY\": \"JSON_END\"" \ + "}]" diff --git a/src/fluent-bit/tests/runtime/data/stackdriver/stackdriver-credentials.json b/src/fluent-bit/tests/runtime/data/stackdriver/stackdriver-credentials.json new file mode 100644 index 000000000..1ea8afd92 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/stackdriver/stackdriver-credentials.json @@ -0,0 +1,12 @@ +{ + "type": "service_account", + "project_id": "fluent-bit", + "private_key_id": "837364849384b143e4e21243a2cbcbd26f4fabcdfe", + "private_key": "-----BEGIN PRIVATE KEY-----\nasldjkasdjksa98327498327498327489qajdsijasdjlksajdlksajdlks7+TK\n-----END PRIVATE KEY-----\n", + "client_email": "stackdriver@fluent-bit.iam.gserviceaccount.com", + "client_id": "0123456789", + "auth_uri": "https://accounts.google.com/o/oauth2/auth", + "token_uri": "https://accounts.google.com/o/oauth2/token", + "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs", + "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/stackdriver%40fluent-bit.iam.gserviceaccount.com" +} diff --git a/src/fluent-bit/tests/runtime/data/stackdriver/stackdriver_multi_entries_severity.log b/src/fluent-bit/tests/runtime/data/stackdriver/stackdriver_multi_entries_severity.log new file mode 100644 index 000000000..157cbea64 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/stackdriver/stackdriver_multi_entries_severity.log @@ -0,0 +1,4 @@ +{"message":"test severity1", "severity":"INFO"} +{"message":"test severity2"} +{"message":"test severity3", "severity":"DEBUG"} +{"message":"test severity4"} diff --git a/src/fluent-bit/tests/runtime/data/stackdriver/stackdriver_test_http_request.h b/src/fluent-bit/tests/runtime/data/stackdriver/stackdriver_test_http_request.h new file mode 100644 index 000000000..2b7ad82d3 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/stackdriver/stackdriver_test_http_request.h @@ -0,0 +1,134 @@ +#define HTTPREQUEST_COMMON_CASE "[" \ + "1591111124," \ + "{" \ + "\"logging.googleapis.com/http_request\": " \ + "{" \ + "\"requestMethod\": \"test_requestMethod\"," \ + "\"requestUrl\": \"test_requestUrl\"," \ + "\"userAgent\": \"test_userAgent\"," \ + "\"remoteIp\": \"test_remoteIp\"," \ + "\"serverIp\": \"test_serverIp\"," \ + "\"referer\": \"test_referer\"," \ + "\"latency\": \"0s\"," \ + "\"protocol\": \"test_protocol\"," \ + "\"requestSize\": 123," \ + "\"responseSize\": 123," \ + "\"status\": 200," \ + "\"cacheFillBytes\": 123," \ + "\"cacheLookup\": true," \ + "\"cacheHit\": true," \ + "\"cacheValidatedWithOriginServer\": true" \ + "}" \ + "}]" + +#define EMPTY_HTTPREQUEST "[" \ + "1591111124," \ + "{" \ + "\"logging.googleapis.com/http_request\": " \ + "{" \ + "}" \ + "}]" + +#define HTTPREQUEST_IN_STRING "[" \ + "1591111124," \ + "{" \ + "\"logging.googleapis.com/http_request\": \"some string\"" \ + "}]" + +#define PARTIAL_HTTPREQUEST "[" \ + "1591111124," \ + "{" \ + "\"logging.googleapis.com/http_request\": " \ + "{" \ + "\"cacheLookup\": true," \ + "\"cacheHit\": true," \ + "\"cacheValidatedWithOriginServer\": true" \ + "}" \ + "}]" + +#define HTTPREQUEST_SUBFIELDS_IN_INCORRECT_TYPE "[" \ + "1591111124," \ + "{" \ + "\"logging.googleapis.com/http_request\": " \ + "{" \ + "\"requestMethod\": 123," \ + "\"requestUrl\": 123," \ + "\"userAgent\": 123," \ + "\"remoteIp\": 123," \ + "\"serverIp\": true," \ + "\"referer\": true," \ + "\"latency\": false," \ + "\"protocol\": false," \ + "\"requestSize\": \"some string\"," \ + "\"responseSize\": true," \ + "\"status\": false," \ + "\"cacheFillBytes\": false," \ + "\"cacheLookup\": \"some string\"," \ + "\"cacheHit\": 123," \ + "\"cacheValidatedWithOriginServer\": 123" \ + "}" \ + "}]" + +#define HTTPREQUEST_EXTRA_SUBFIELDS_EXISTED "[" \ + "1591111124," \ + "{" \ + "\"logging.googleapis.com/http_request\": " \ + "{" \ + "\"requestMethod\": \"test_requestMethod\"," \ + "\"requestUrl\": \"test_requestUrl\"," \ + "\"userAgent\": \"test_userAgent\"," \ + "\"remoteIp\": \"test_remoteIp\"," \ + "\"serverIp\": \"test_serverIp\"," \ + "\"referer\": \"test_referer\"," \ + "\"latency\": \"0s\"," \ + "\"protocol\": \"test_protocol\"," \ + "\"requestSize\": 123," \ + "\"responseSize\": 123," \ + "\"status\": 200," \ + "\"cacheFillBytes\": 123," \ + "\"cacheLookup\": true," \ + "\"cacheHit\": true," \ + "\"cacheValidatedWithOriginServer\": true," \ + "\"extra_key1\": \"extra_val1\"," \ + "\"extra_key2\": 123," \ + "\"extra_key3\": true" \ + "}" \ + "}]" + + +/* Tests for 'latency' */ +#define HTTPREQUEST_LATENCY_COMMON_CASE "[" \ + "1591111124," \ + "{" \ + "\"logging.googleapis.com/http_request\": " \ + "{" \ + "\"latency\": \" 100.00 s \"" \ + "}" \ + "}]" + +#define HTTPREQUEST_LATENCY_INVALID_SPACES "[" \ + "1591111124," \ + "{" \ + "\"logging.googleapis.com/http_request\": " \ + "{" \ + "\"latency\": \" 100. 00 s \"" \ + "}" \ + "}]" + +#define HTTPREQUEST_LATENCY_INVALID_STRING "[" \ + "1591111124," \ + "{" \ + "\"logging.googleapis.com/http_request\": " \ + "{" \ + "\"latency\": \" s100.00 s \"" \ + "}" \ + "}]" + +#define HTTPREQUEST_LATENCY_INVALID_END "[" \ + "1591111124," \ + "{" \ + "\"logging.googleapis.com/http_request\": " \ + "{" \ + "\"latency\": \" 100.00 \"" \ + "}" \ + "}]" diff --git a/src/fluent-bit/tests/runtime/data/stackdriver/stackdriver_test_insert_id.h b/src/fluent-bit/tests/runtime/data/stackdriver/stackdriver_test_insert_id.h new file mode 100644 index 000000000..9cdf3f545 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/stackdriver/stackdriver_test_insert_id.h @@ -0,0 +1,26 @@ + +#define INSERTID_COMMON_CASE "[" \ + "1591111124," \ + "{" \ + "\"logging.googleapis.com/insertId\": \"test_insertId\" " \ + "}]" + +#define EMPTY_INSERTID "[" \ + "1591111124," \ + "{" \ + "\"logging.googleapis.com/insertId\": \"\" " \ + "}]" + +#define INSERTID_INCORRECT_TYPE_INT "[" \ + "1591111124," \ + "{" \ + "\"logging.googleapis.com/insertId\": 123 " \ + "}]" + +#define INSERTID_INCORRECT_TYPE_MAP "[" \ + "1591111124," \ + "{" \ + "\"logging.googleapis.com/insertId\": " \ + "{" \ + "}" \ + "}]" diff --git a/src/fluent-bit/tests/runtime/data/stackdriver/stackdriver_test_k8s_resource.h b/src/fluent-bit/tests/runtime/data/stackdriver/stackdriver_test_k8s_resource.h new file mode 100644 index 000000000..9fe1604a9 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/stackdriver/stackdriver_test_k8s_resource.h @@ -0,0 +1,89 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* k8s_container */ +#define K8S_CONTAINER_COMMON "[" \ + "1591649196," \ + "{" \ + "\"message\": \"K8S_CONTAINER_COMMON\"," \ + "\"logging.googleapis.com/local_resource_id\": \"k8s_container.testnamespace.testpod.testctr\"" \ + "}]" + +#define K8S_CONTAINER_COMMON_DIFF_TAGS "[" \ + "1591649196," \ + "{" \ + "\"message\": \"K8S_CONTAINER_COMMON\"," \ + "\"logging.googleapis.com/local_resource_id\": \"k8s_container.diffnamespace.diffpod.diffctr\"" \ + "}]" + +#define K8S_CONTAINER_NO_LOCAL_RESOURCE_ID "[" \ + "1591649196," \ + "{" \ + "\"message\": \"K8S_CONTAINER_COMMON_NO_LOCAL_RESOURCE_ID\"" \ + "}]" + +/* k8s_node */ +#define K8S_NODE_COMMON "[" \ + "1591649196," \ + "{" \ + "\"message\": \"K8S_NODE_COMMON\"," \ + "\"logging.googleapis.com/local_resource_id\": \"k8s_node.testnode\"," \ + "\"PRIORITY\": 6," \ + "\"SYSLOG_FACILITY\": 3," \ + "\"_CAP_EFFECTIVE\": \"3fffffffff\"," \ + "\"_PID\": 1387," \ + "\"_SYSTEMD_UNIT\": \"docker.service\"," \ + "\"END_KEY\": \"JSON_END\"" \ + "}]" + +#define K8S_NODE_NO_LOCAL_RESOURCE_ID "[" \ + "1591649196," \ + "{" \ + "\"message\": \"K8S_NODE_NO_LOCAL_RESOURCE_ID\"," \ + "\"PRIORITY\": 6," \ + "\"SYSLOG_FACILITY\": 3," \ + "\"_CAP_EFFECTIVE\": \"3fffffffff\"," \ + "\"_PID\": 1387," \ + "\"_SYSTEMD_UNIT\": \"docker.service\"," \ + "\"END_KEY\": \"JSON_END\"" \ + "}]" + +#define K8S_NODE_LOCAL_RESOURCE_ID_WITH_DOT "[" \ + "1591649196," \ + "{" \ + "\"message\": \"K8S_NODE_LOCAL_RESOURCE_ID_WITH_DOT\"," \ + "\"logging.googleapis.com/local_resource_id\": \"k8s_node.testnode.withdot.dot\"," \ + "\"PRIORITY\": 6," \ + "\"SYSLOG_FACILITY\": 3," \ + "\"_CAP_EFFECTIVE\": \"3fffffffff\"," \ + "\"_PID\": 1387," \ + "\"_SYSTEMD_UNIT\": \"docker.service\"," \ + "\"END_KEY\": \"JSON_END\"" \ + "}]" + +/* k8s_pod */ +#define K8S_POD_COMMON "[" \ + "1591649196," \ + "{" \ + "\"message\": \"K8S_POD_COMMON\"," \ + "\"logging.googleapis.com/local_resource_id\": \"k8s_pod.testnamespace.testpod\"," \ + "\"key_0\": false," \ + "\"key_1\": true," \ + "\"key_2\": \"some string\"," \ + "\"key_3\": 0.12345678," \ + "\"key_4\": 5000," \ + "\"END_KEY\": \"JSON_END\"" \ + "}]" + +#define K8S_POD_NO_LOCAL_RESOURCE_ID "[" \ + "1591649196," \ + "{" \ + "\"message\": \"K8S_POD_NO_LOCAL_RESOURCE_ID\"," \ + "\"key_0\": false," \ + "\"key_1\": true," \ + "\"key_2\": \"some string\"," \ + "\"key_3\": 0.12345678," \ + "\"key_4\": 5000," \ + "\"END_KEY\": \"JSON_END\"" \ + "}]" + +
\ No newline at end of file diff --git a/src/fluent-bit/tests/runtime/data/stackdriver/stackdriver_test_labels.h b/src/fluent-bit/tests/runtime/data/stackdriver/stackdriver_test_labels.h new file mode 100644 index 000000000..ec3d80b3a --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/stackdriver/stackdriver_test_labels.h @@ -0,0 +1,44 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +#define DEFAULT_LABELS "[" \ + "1591649196," \ + "{" \ + "\"logging.googleapis.com/labels\": " \ + "{" \ + "\"testA\": \"valA\"," \ + "\"testB\": \"valB\"" \ + "}" \ + "}]" + +#define CUSTOM_LABELS "[" \ + "1591649196," \ + "{" \ + "\"logging.googleapis.com/customlabels\": " \ + "{" \ + "\"testA\": \"valA\"," \ + "\"testB\": \"valB\"" \ + "}" \ + "}]" + +#define DEFAULT_LABELS_K8S_RESOURCE_TYPE "[" \ + "1591649196," \ + "{" \ + "\"logging.googleapis.com/local_resource_id\": \"k8s_container.testnamespace.testpod.testctr\"," \ + "\"logging.googleapis.com/labels\": " \ + "{" \ + "\"testA\": \"valA\"," \ + "\"testB\": \"valB\"" \ + "}" \ + "}]" + +#define CUSTOM_LABELS_K8S_RESOURCE_TYPE "[" \ + "1591649196," \ + "{" \ + "\"logging.googleapis.com/local_resource_id\": \"k8s_container.testnamespace.testpod.testctr\"," \ + "\"logging.googleapis.com/customlabels\": " \ + "{" \ + "\"testA\": \"valA\"," \ + "\"testB\": \"valB\"" \ + "}" \ + "}]" + diff --git a/src/fluent-bit/tests/runtime/data/stackdriver/stackdriver_test_log_name.h b/src/fluent-bit/tests/runtime/data/stackdriver/stackdriver_test_log_name.h new file mode 100644 index 000000000..4d39e242e --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/stackdriver/stackdriver_test_log_name.h @@ -0,0 +1,10 @@ +#define LOG_NAME_OVERRIDE "[" \ + "1591111124," \ + "{" \ + "\"custom_log_name_key\": \"custom_log_name\"" \ + "}]" + +#define LOG_NAME_NO_OVERRIDE "[" \ + "1591111124," \ + "{" \ + "}]" diff --git a/src/fluent-bit/tests/runtime/data/stackdriver/stackdriver_test_monitored_resource.h b/src/fluent-bit/tests/runtime/data/stackdriver/stackdriver_test_monitored_resource.h new file mode 100644 index 000000000..d18749287 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/stackdriver/stackdriver_test_monitored_resource.h @@ -0,0 +1,46 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ +#define MONITORED_RESOURCE_COMMON_CASE "[" \ + "1591649196," \ + "{" \ + "\"logging.googleapis.com/monitored_resource\": " \ + "{" \ + "\"labels\": " \ + "{" \ + "\"project_id\": \"monitored_resource_project_id\"," \ + "\"location\": \"monitored_resource_location\"," \ + "\"testA\": \"valA\"" \ + "}" \ + "}" \ + "}]" + +#define MONITORED_RESOURCE_PRIORITY_HIGHER_THAN_LOCAL_RESOURCE_ID "[" \ + "1591649196," \ + "{" \ + "\"logging.googleapis.com/monitored_resource\": " \ + "{" \ + "\"labels\": " \ + "{" \ + "\"project_id\": \"monitored_resource_project_id\"," \ + "\"location\": \"monitored_resource_location\"," \ + "\"cluster_name\": \"monitored_resource_cluster_name\"," \ + "\"namespace_name\": \"monitored_resource_namespace_name\"," \ + "\"pod_name\": \"monitored_resource_pod_name\"," \ + "\"container_name\": \"monitored_resource_container_name\"" \ + "}" \ + "}," \ + "\"logging.googleapis.com/local_resource_id\": \"k8s_container.testnamespace.testpod.testctr\"" \ + "}]" + +#define MONITORED_RESOURCE_PRIORITY_HIGHER_THAN_GCE_INSTANCE "[" \ + "1448403340," \ + "{" \ + "\"logging.googleapis.com/monitored_resource\": " \ + "{" \ + "\"labels\": " \ + "{" \ + "\"project_id\": \"monitored_resource_project_id\"," \ + "\"zone\": \"monitored_resource_zone\"," \ + "\"instance_id\": \"monitored_resource_instance_id\"" \ + "}" \ + "}" \ + "}]" diff --git a/src/fluent-bit/tests/runtime/data/stackdriver/stackdriver_test_operation.h b/src/fluent-bit/tests/runtime/data/stackdriver/stackdriver_test_operation.h new file mode 100644 index 000000000..b3af37225 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/stackdriver/stackdriver_test_operation.h @@ -0,0 +1,62 @@ +#define OPERATION_COMMON_CASE "[" \ + "1591111124," \ + "{" \ + "\"logging.googleapis.com/operation\": " \ + "{" \ + "\"id\": \"test_id\"," \ + "\"producer\": \"test_producer\"," \ + "\"first\": true," \ + "\"last\": true" \ + "}" \ + "}]" + +#define EMPTY_OPERATION "[" \ + "1591111124," \ + "{" \ + "\"logging.googleapis.com/operation\": " \ + "{" \ + "}" \ + "}]" + +#define OPERATION_IN_STRING "[" \ + "1591111124," \ + "{" \ + "\"logging.googleapis.com/operation\": \"some string\"" \ + "}]" + +#define PARTIAL_SUBFIELDS "[" \ + "1591111124," \ + "{" \ + "\"logging.googleapis.com/operation\": " \ + "{" \ + "\"first\": false," \ + "\"last\": false" \ + "}" \ + "}]" + +#define SUBFIELDS_IN_INCORRECT_TYPE "[" \ + "1591111124," \ + "{" \ + "\"logging.googleapis.com/operation\": " \ + "{" \ + "\"id\": 123," \ + "\"producer\": true," \ + "\"first\": \"some string\"," \ + "\"last\": 123" \ + "}" \ + "}]" + +#define EXTRA_SUBFIELDS_EXISTED "[" \ + "1591111124," \ + "{" \ + "\"logging.googleapis.com/operation\": " \ + "{" \ + "\"id\": \"test_id\"," \ + "\"producer\": \"test_producer\"," \ + "\"first\": true," \ + "\"last\": true," \ + "\"extra_key1\": \"extra_val1\"," \ + "\"extra_key2\": 123," \ + "\"extra_key3\": true" \ + "}" \ + "}]" diff --git a/src/fluent-bit/tests/runtime/data/stackdriver/stackdriver_test_resource_labels.h b/src/fluent-bit/tests/runtime/data/stackdriver/stackdriver_test_resource_labels.h new file mode 100644 index 000000000..a65a47140 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/stackdriver/stackdriver_test_resource_labels.h @@ -0,0 +1,37 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +#define ONE_FIELD "[" \ + "1591649196," \ + "{" \ + "\"keyA\": \"valA\"" \ + "}]" + +#define MULTIPLE_FIELDS "[" \ + "1591649196," \ + "{" \ + "\"keyA\": \"valA\"," \ + "\"keyB\": \"valB\"" \ + "}]" + +#define NESTED_FIELDS "[" \ + "1591649196," \ + "{" \ + "\"toplevel\": " \ + "{" \ + "\"keyA\": \"valA\"," \ + "\"keyB\": \"valB\"" \ + "}" \ + "}]" + +#define LAYERED_NESTED_FIELDS "[" \ + "1591649196," \ + "{" \ + "\"toplevel\": " \ + "{" \ + "\"midlevel\": " \ + "{" \ + "\"keyA\": \"valA\"" \ + "}," \ + "\"keyB\": \"valB\"" \ + "}" \ + "}]" diff --git a/src/fluent-bit/tests/runtime/data/stackdriver/stackdriver_test_source_location.h b/src/fluent-bit/tests/runtime/data/stackdriver/stackdriver_test_source_location.h new file mode 100644 index 000000000..0bf419f5a --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/stackdriver/stackdriver_test_source_location.h @@ -0,0 +1,70 @@ +#define SOURCELOCATION_COMMON_CASE "[" \ + "1591111124," \ + "{" \ + "\"logging.googleapis.com/sourceLocation\": " \ + "{" \ + "\"file\": \"test_file\"," \ + "\"line\": 123," \ + "\"function\": \"test_function\"" \ + "}" \ + "}]" + +#define SOURCELOCATION_COMMON_CASE_LINE_IN_STRING "[" \ + "1591111124," \ + "{" \ + "\"logging.googleapis.com/sourceLocation\": " \ + "{" \ + "\"file\": \"test_file\"," \ + "\"line\": \"123\"," \ + "\"function\": \"test_function\"" \ + "}" \ + "}]" + +#define EMPTY_SOURCELOCATION "[" \ + "1591111124," \ + "{" \ + "\"logging.googleapis.com/sourceLocation\": " \ + "{" \ + "}" \ + "}]" + +#define SOURCELOCATION_IN_STRING "[" \ + "1591111124," \ + "{" \ + "\"logging.googleapis.com/sourceLocation\": \"some string\"" \ + "}]" + +#define PARTIAL_SOURCELOCATION "[" \ + "1591111124," \ + "{" \ + "\"logging.googleapis.com/sourceLocation\": " \ + "{" \ + "\"function\": \"test_function\"" \ + "}" \ + "}]" + +#define SOURCELOCATION_SUBFIELDS_IN_INCORRECT_TYPE "[" \ + "1591111124," \ + "{" \ + "\"logging.googleapis.com/sourceLocation\": " \ + "{" \ + "\"file\": 123," \ + "\"line\": \"some string\"," \ + "\"function\": true" \ + "}" \ + "}]" + +#define SOURCELOCATION_EXTRA_SUBFIELDS_EXISTED "[" \ + "1591111124," \ + "{" \ + "\"logging.googleapis.com/sourceLocation\": " \ + "{" \ + "\"file\": \"test_file\"," \ + "\"line\": 123," \ + "\"function\": \"test_function\"," \ + "\"extra_key1\": \"extra_val1\"," \ + "\"extra_key2\": 123," \ + "\"extra_key3\": true" \ + "}" \ + "}]" +
\ No newline at end of file diff --git a/src/fluent-bit/tests/runtime/data/stackdriver/stackdriver_test_span_id.h b/src/fluent-bit/tests/runtime/data/stackdriver/stackdriver_test_span_id.h new file mode 100644 index 000000000..5b7b823e2 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/stackdriver/stackdriver_test_span_id.h @@ -0,0 +1,6 @@ +#define SPAN_ID_COMMON_CASE "[" \ + "1591111124," \ + "{" \ + "\"logging.googleapis.com/spanId\": \"000000000000004a\"" \ + "}]" +
\ No newline at end of file diff --git a/src/fluent-bit/tests/runtime/data/stackdriver/stackdriver_test_timestamp.h b/src/fluent-bit/tests/runtime/data/stackdriver/stackdriver_test_timestamp.h new file mode 100644 index 000000000..f80cc37d6 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/stackdriver/stackdriver_test_timestamp.h @@ -0,0 +1,62 @@ +/* timestamp after parsing: 2020-07-21T16:40:42.000012345Z */ +#define TIMESTAMP_FORMAT_OBJECT_COMMON_CASE "[" \ + "1591111124," \ + "{" \ + "\"timestamp\": " \ + "{" \ + "\"seconds\": \"1595349642\"," \ + "\"nanos\": \"12345\"" \ + "}" \ + "}]" + +/* "1595349600" in RFC3339 format: 2020-07-21T16:40:00Z */ +#define TIMESTAMP_FORMAT_OBJECT_NOT_A_MAP "[" \ + "1595349600," \ + "{" \ + "\"timestamp\": \"string\"" \ + "}]" + +/* "1595349600" in RFC3339 format: 2020-07-21T16:40:00Z */ +#define TIMESTAMP_FORMAT_OBJECT_MISSING_SUBFIELD "[" \ + "1595349600," \ + "{" \ + "\"timestamp\": " \ + "{" \ + "\"nanos\": \"12345\"" \ + "}" \ + "}]" + +/* "1595349600" in RFC3339 format: 2020-07-21T16:40:00Z */ +#define TIMESTAMP_FORMAT_OBJECT_INCORRECT_TYPE_SUBFIELDS "[" \ + "1595349600," \ + "{" \ + "\"timestamp\": " \ + "{" \ + "\"seconds\": \"string\"," \ + "\"nanos\": true" \ + "}" \ + "}]" + + +/* timestamp after parsing: 2020-07-21T16:40:42.000012345Z */ +#define TIMESTAMP_FORMAT_DUO_FIELDS_COMMON_CASE "[" \ + "1595349600," \ + "{" \ + "\"timestampSeconds\": \"1595349642\"," \ + "\"timestampNanos\": \"12345\"" \ + "}]" + +/* "1595349600" in RFC3339 format: 2020-07-21T16:40:00Z */ +#define TIMESTAMP_FORMAT_DUO_FIELDS_MISSING_NANOS "[" \ + "1595349600," \ + "{" \ + "\"timestampSeconds\": \"1595349642\"" \ + "}]" + +/* "1595349600" in RFC3339 format: 2020-07-21T16:40:00Z */ +#define TIMESTAMP_FORMAT_DUO_FIELDS_INCORRECT_TYPE "[" \ + "1595349600," \ + "{" \ + "\"timestampSeconds\": \"string\"," \ + "\"timestampNanos\": true" \ + "}]" diff --git a/src/fluent-bit/tests/runtime/data/stackdriver/stackdriver_test_trace.h b/src/fluent-bit/tests/runtime/data/stackdriver/stackdriver_test_trace.h new file mode 100644 index 000000000..15c7bc911 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/stackdriver/stackdriver_test_trace.h @@ -0,0 +1,6 @@ +#define TRACE_COMMON_CASE "[" \ + "1591111124," \ + "{" \ + "\"trace\": \"test-trace-id-xyz\"" \ + "}]" +
\ No newline at end of file diff --git a/src/fluent-bit/tests/runtime/data/stackdriver/stackdriver_test_trace_sampled.h b/src/fluent-bit/tests/runtime/data/stackdriver/stackdriver_test_trace_sampled.h new file mode 100644 index 000000000..fad628e93 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/stackdriver/stackdriver_test_trace_sampled.h @@ -0,0 +1,11 @@ +#define TRACE_SAMPLED_CASE_TRUE "[" \ + "1591111124," \ + "{" \ + "\"logging.googleapis.com/traceSampled\": true" \ + "}]" + +#define TRACE_SAMPLED_CASE_FALSE "[" \ + "1591111124," \ + "{" \ + "\"logging.googleapis.com/traceSampled\": false" \ + "}]" diff --git a/src/fluent-bit/tests/runtime/data/tail/log/3943.log b/src/fluent-bit/tests/runtime/data/tail/log/3943.log new file mode 100644 index 000000000..8f3d91d9c --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/tail/log/3943.log @@ -0,0 +1,2 @@ +bbb
+ diff --git a/src/fluent-bit/tests/runtime/data/tail/log/dockermode.log b/src/fluent-bit/tests/runtime/data/tail/log/dockermode.log new file mode 100644 index 000000000..a5a05ee46 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/tail/log/dockermode.log @@ -0,0 +1,3 @@ +{"log":"Single log\n"} +{"log":"Second log\n"} +{"log":"Third log\n"} diff --git a/src/fluent-bit/tests/runtime/data/tail/log/dockermode_firstline_detection.log b/src/fluent-bit/tests/runtime/data/tail/log/dockermode_firstline_detection.log new file mode 100644 index 000000000..56a2711aa --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/tail/log/dockermode_firstline_detection.log @@ -0,0 +1,9 @@ +{"log":"Single log\n"} +{"log":"Second log\n"} +{"log":"Third log\n"} +{"log":"2020-03-24 Multiple lines: \n"} +{"log":"first bullet point\n"} +{"log":"second bullet point\n"} +{"log":"third bullet point\n"} +{"log":"fourth bullet point\n"} +{"log":"2020-03-24 Single line\n"} diff --git a/src/fluent-bit/tests/runtime/data/tail/log/dockermode_multiple_lines.log b/src/fluent-bit/tests/runtime/data/tail/log/dockermode_multiple_lines.log new file mode 100644 index 000000000..564dabbb3 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/tail/log/dockermode_multiple_lines.log @@ -0,0 +1,6 @@ +{"log":"2020-03-24 Multiple lines: \n"} +{"log":"first bullet point\n"} +{"log":"second bullet point\n"} +{"log":"third bullet point\n"} +{"log":"fourth bullet point\n"} +{"log":"2020-03-24 Single line\n"} diff --git a/src/fluent-bit/tests/runtime/data/tail/log/dockermode_splitted_line.log b/src/fluent-bit/tests/runtime/data/tail/log/dockermode_splitted_line.log new file mode 100644 index 000000000..8f80f1b32 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/tail/log/dockermode_splitted_line.log @@ -0,0 +1,6 @@ +{"log":"2020-03-24 Single "} +{"log":"li"} +{"log":"ne\n"} +{"log":"2020-03-24 Another "} +{"log":"single "} +{"log":"line\n"} diff --git a/src/fluent-bit/tests/runtime/data/tail/log/dockermode_splitted_multiple_lines.log b/src/fluent-bit/tests/runtime/data/tail/log/dockermode_splitted_multiple_lines.log new file mode 100644 index 000000000..2c7d36dd7 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/tail/log/dockermode_splitted_multiple_lines.log @@ -0,0 +1,8 @@ +{"log":"2020-03-24 Multiple lines: \n"} +{"log":"first bullet point\n"} +{"log":"second bullet point\n"} +{"log":"third "} +{"log":"bullet "} +{"log":"point\n"} +{"log":"fourth bullet point\n"} +{"log":"2020-03-24 Single line\n"} diff --git a/src/fluent-bit/tests/runtime/data/tail/log/multiline_001.log b/src/fluent-bit/tests/runtime/data/tail/log/multiline_001.log new file mode 100644 index 000000000..986223d87 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/tail/log/multiline_001.log @@ -0,0 +1,6 @@ +{"key1": 12345, "key2": "abc", "time": "2006-07-28T13:22:04Z","message":"Dec 14 06:41:08 Exception in thread main java.lang.RuntimeException: Something has gone wrong, aborting!"} +{"key1": 12345, "key2": "abc", "time": "2006-07-28T13:22:04Z","message":" at com.myproject.module.MyProject.badMethod(MyProject.java:22)"} +{"key1": 12345, "key2": "abc", "time": "2006-07-28T13:22:04Z","message":" at com.myproject.module.MyProject.oneMoreMethod(MyProject.java:18)"} +{"key1": 12345, "key2": "abc", "time": "2006-07-28T13:22:04Z","message":" at com.myproject.module.MyProject.anotherMethod(MyProject.java:14)"} +{"key1": 12345, "key2": "abc", "time": "2006-07-28T13:22:04Z","message":" at com.myproject.module.MyProject.oneMoreMethod(MyProject.java:13)"} +{"key1": 12345, "key2": "abc", "time": "2006-07-28T13:22:04Z","message":" at com.myproject.module.MyProject.oneMoreMethod(MyProject.java:12)"} diff --git a/src/fluent-bit/tests/runtime/data/tail/out/3943.out b/src/fluent-bit/tests/runtime/data/tail/out/3943.out new file mode 100644 index 000000000..52a0f10a9 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/tail/out/3943.out @@ -0,0 +1,2 @@ +{"log":"bbb"} +{"log":""}
\ No newline at end of file diff --git a/src/fluent-bit/tests/runtime/data/tail/out/dockermode.out b/src/fluent-bit/tests/runtime/data/tail/out/dockermode.out new file mode 100644 index 000000000..a5a05ee46 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/tail/out/dockermode.out @@ -0,0 +1,3 @@ +{"log":"Single log\n"} +{"log":"Second log\n"} +{"log":"Third log\n"} diff --git a/src/fluent-bit/tests/runtime/data/tail/out/dockermode_firstline_detection.out b/src/fluent-bit/tests/runtime/data/tail/out/dockermode_firstline_detection.out new file mode 100644 index 000000000..00eecce98 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/tail/out/dockermode_firstline_detection.out @@ -0,0 +1,5 @@ +{"log":"Single log\n"} +{"log":"Second log\n"} +{"log":"Third log\n"} +{"log":"2020-03-24 Multiple lines: \nfirst bullet point\nsecond bullet point\nthird bullet point\nfourth bullet point\n"} +{"log":"2020-03-24 Single line\n"} diff --git a/src/fluent-bit/tests/runtime/data/tail/out/dockermode_multiple_lines.out b/src/fluent-bit/tests/runtime/data/tail/out/dockermode_multiple_lines.out new file mode 100644 index 000000000..27fba1b98 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/tail/out/dockermode_multiple_lines.out @@ -0,0 +1,2 @@ +{"log":"2020-03-24 Multiple lines: \nfirst bullet point\nsecond bullet point\nthird bullet point\nfourth bullet point\n"} +{"log":"2020-03-24 Single line\n"} diff --git a/src/fluent-bit/tests/runtime/data/tail/out/dockermode_splitted_line.out b/src/fluent-bit/tests/runtime/data/tail/out/dockermode_splitted_line.out new file mode 100644 index 000000000..14d0aa8a2 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/tail/out/dockermode_splitted_line.out @@ -0,0 +1,2 @@ +{"log":"2020-03-24 Single line\n"} +{"log":"2020-03-24 Another single line\n"} diff --git a/src/fluent-bit/tests/runtime/data/tail/out/dockermode_splitted_multiple_lines.out b/src/fluent-bit/tests/runtime/data/tail/out/dockermode_splitted_multiple_lines.out new file mode 100644 index 000000000..27fba1b98 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/tail/out/dockermode_splitted_multiple_lines.out @@ -0,0 +1,2 @@ +{"log":"2020-03-24 Multiple lines: \nfirst bullet point\nsecond bullet point\nthird bullet point\nfourth bullet point\n"} +{"log":"2020-03-24 Single line\n"} diff --git a/src/fluent-bit/tests/runtime/data/tail/out/multiline_001.out b/src/fluent-bit/tests/runtime/data/tail/out/multiline_001.out new file mode 100644 index 000000000..73f48580d --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/tail/out/multiline_001.out @@ -0,0 +1 @@ +{"key1":12345,"key2":"abc","time":"2006-07-28T13:22:04Z","message":"Dec 14 06:41:08 Exception in thread main java.lang.RuntimeException: Something has gone wrong, aborting!\n at com.myproject.module.MyProject.badMethod(MyProject.java:22)\n at com.myproject.module.MyProject.oneMoreMethod(MyProject.java:18)\n at com.myproject.module.MyProject.anotherMethod(MyProject.java:14)\n at com.myproject.module.MyProject.oneMoreMethod(MyProject.java:13)\n at com.myproject.module.MyProject.oneMoreMethod(MyProject.java:12)"} diff --git a/src/fluent-bit/tests/runtime/data/tail/out/skip_long_lines.out b/src/fluent-bit/tests/runtime/data/tail/out/skip_long_lines.out new file mode 100644 index 000000000..c7c32913c --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/tail/out/skip_long_lines.out @@ -0,0 +1,2 @@ +{"log":"before_long_line"} +{"log":"after_long_line"}
\ No newline at end of file diff --git a/src/fluent-bit/tests/runtime/data/tail/parsers.conf b/src/fluent-bit/tests/runtime/data/tail/parsers.conf new file mode 100644 index 000000000..28bde4751 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/tail/parsers.conf @@ -0,0 +1,10 @@ +[PARSER] + Name docker + Format json + Time_Key time + Time_Format %Y-%m-%dT%H:%M:%S.%L%z + +[PARSER] + Name docker_multiline + Format regex + Regex (?<log>^{"log":"\d{4}-\d{2}-\d{2}.*) diff --git a/src/fluent-bit/tests/runtime/data/tail/parsers_multiline_json.conf b/src/fluent-bit/tests/runtime/data/tail/parsers_multiline_json.conf new file mode 100644 index 000000000..1ed1cfdf3 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/tail/parsers_multiline_json.conf @@ -0,0 +1,24 @@ +[PARSER] + Name json + Format json + +[MULTILINE_PARSER] + name multiline-json-regex + key_content message + type regex + parser json + flush_timeout 2000 + + # + # Regex rules for multiline parsing + # --------------------------------- + # + # configuration hints: + # + # - first state always has the name: start_state + # - every field in the rule must be inside double quotes + # + # rules | state name | regex pattern | next state + # ------|---------------|-------------------------------------------- + rule "start_state" "/(Dec \d+ \d+\:\d+\:\d+)(.*)/" "cont" + rule "cont" "/^\s+at.*/" "cont" diff --git a/src/fluent-bit/tests/runtime/data/td/json_td.h b/src/fluent-bit/tests/runtime/data/td/json_td.h new file mode 100755 index 000000000..c9f4e01da --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/td/json_td.h @@ -0,0 +1,506 @@ +#define JSON_TD "[" \ + "1448403340," \ + "{" \ + "\"key_0\": \"val_0\"," \ + "\"key_1\": \"val_1\"," \ + "\"key_2\": \"val_2\"," \ + "\"key_3\": \"val_3\"," \ + "\"key_4\": \"val_4\"," \ + "\"key_5\": \"val_5\"," \ + "\"key_6\": \"val_6\"," \ + "\"key_7\": \"val_7\"," \ + "\"key_8\": \"val_8\"," \ + "\"key_9\": \"val_9\"," \ + "\"key_10\": \"val_10\"," \ + "\"key_11\": \"val_11\"," \ + "\"key_12\": \"val_12\"," \ + "\"key_13\": \"val_13\"," \ + "\"key_14\": \"val_14\"," \ + "\"key_15\": \"val_15\"," \ + "\"key_16\": \"val_16\"," \ + "\"key_17\": \"val_17\"," \ + "\"key_18\": \"val_18\"," \ + "\"key_19\": \"val_19\"," \ + "\"key_20\": \"val_20\"," \ + "\"key_21\": \"val_21\"," \ + "\"key_22\": \"val_22\"," \ + "\"key_23\": \"val_23\"," \ + "\"key_24\": \"val_24\"," \ + "\"key_25\": \"val_25\"," \ + "\"key_26\": \"val_26\"," \ + "\"key_27\": \"val_27\"," \ + "\"key_28\": \"val_28\"," \ + "\"key_29\": \"val_29\"," \ + "\"key_30\": \"val_30\"," \ + "\"key_31\": \"val_31\"," \ + "\"key_32\": \"val_32\"," \ + "\"key_33\": \"val_33\"," \ + "\"key_34\": \"val_34\"," \ + "\"key_35\": \"val_35\"," \ + "\"key_36\": \"val_36\"," \ + "\"key_37\": \"val_37\"," \ + "\"key_38\": \"val_38\"," \ + "\"key_39\": \"val_39\"," \ + "\"key_40\": \"val_40\"," \ + "\"key_41\": \"val_41\"," \ + "\"key_42\": \"val_42\"," \ + "\"key_43\": \"val_43\"," \ + "\"key_44\": \"val_44\"," \ + "\"key_45\": \"val_45\"," \ + "\"key_46\": \"val_46\"," \ + "\"key_47\": \"val_47\"," \ + "\"key_48\": \"val_48\"," \ + "\"key_49\": \"val_49\"," \ + "\"key_50\": \"val_50\"," \ + "\"key_51\": \"val_51\"," \ + "\"key_52\": \"val_52\"," \ + "\"key_53\": \"val_53\"," \ + "\"key_54\": \"val_54\"," \ + "\"key_55\": \"val_55\"," \ + "\"key_56\": \"val_56\"," \ + "\"key_57\": \"val_57\"," \ + "\"key_58\": \"val_58\"," \ + "\"key_59\": \"val_59\"," \ + "\"key_60\": \"val_60\"," \ + "\"key_61\": \"val_61\"," \ + "\"key_62\": \"val_62\"," \ + "\"key_63\": \"val_63\"," \ + "\"key_64\": \"val_64\"," \ + "\"key_65\": \"val_65\"," \ + "\"key_66\": \"val_66\"," \ + "\"key_67\": \"val_67\"," \ + "\"key_68\": \"val_68\"," \ + "\"key_69\": \"val_69\"," \ + "\"key_70\": \"val_70\"," \ + "\"key_71\": \"val_71\"," \ + "\"key_72\": \"val_72\"," \ + "\"key_73\": \"val_73\"," \ + "\"key_74\": \"val_74\"," \ + "\"key_75\": \"val_75\"," \ + "\"key_76\": \"val_76\"," \ + "\"key_77\": \"val_77\"," \ + "\"key_78\": \"val_78\"," \ + "\"key_79\": \"val_79\"," \ + "\"key_80\": \"val_80\"," \ + "\"key_81\": \"val_81\"," \ + "\"key_82\": \"val_82\"," \ + "\"key_83\": \"val_83\"," \ + "\"key_84\": \"val_84\"," \ + "\"key_85\": \"val_85\"," \ + "\"key_86\": \"val_86\"," \ + "\"key_87\": \"val_87\"," \ + "\"key_88\": \"val_88\"," \ + "\"key_89\": \"val_89\"," \ + "\"key_90\": \"val_90\"," \ + "\"key_91\": \"val_91\"," \ + "\"key_92\": \"val_92\"," \ + "\"key_93\": \"val_93\"," \ + "\"key_94\": \"val_94\"," \ + "\"key_95\": \"val_95\"," \ + "\"key_96\": \"val_96\"," \ + "\"key_97\": \"val_97\"," \ + "\"key_98\": \"val_98\"," \ + "\"key_99\": \"val_99\"," \ + "\"key_100\": \"val_100\"," \ + "\"key_101\": \"val_101\"," \ + "\"key_102\": \"val_102\"," \ + "\"key_103\": \"val_103\"," \ + "\"key_104\": \"val_104\"," \ + "\"key_105\": \"val_105\"," \ + "\"key_106\": \"val_106\"," \ + "\"key_107\": \"val_107\"," \ + "\"key_108\": \"val_108\"," \ + "\"key_109\": \"val_109\"," \ + "\"key_110\": \"val_110\"," \ + "\"key_111\": \"val_111\"," \ + "\"key_112\": \"val_112\"," \ + "\"key_113\": \"val_113\"," \ + "\"key_114\": \"val_114\"," \ + "\"key_115\": \"val_115\"," \ + "\"key_116\": \"val_116\"," \ + "\"key_117\": \"val_117\"," \ + "\"key_118\": \"val_118\"," \ + "\"key_119\": \"val_119\"," \ + "\"key_120\": \"val_120\"," \ + "\"key_121\": \"val_121\"," \ + "\"key_122\": \"val_122\"," \ + "\"key_123\": \"val_123\"," \ + "\"key_124\": \"val_124\"," \ + "\"key_125\": \"val_125\"," \ + "\"key_126\": \"val_126\"," \ + "\"key_127\": \"val_127\"," \ + "\"key_128\": \"val_128\"," \ + "\"key_129\": \"val_129\"," \ + "\"key_130\": \"val_130\"," \ + "\"key_131\": \"val_131\"," \ + "\"key_132\": \"val_132\"," \ + "\"key_133\": \"val_133\"," \ + "\"key_134\": \"val_134\"," \ + "\"key_135\": \"val_135\"," \ + "\"key_136\": \"val_136\"," \ + "\"key_137\": \"val_137\"," \ + "\"key_138\": \"val_138\"," \ + "\"key_139\": \"val_139\"," \ + "\"key_140\": \"val_140\"," \ + "\"key_141\": \"val_141\"," \ + "\"key_142\": \"val_142\"," \ + "\"key_143\": \"val_143\"," \ + "\"key_144\": \"val_144\"," \ + "\"key_145\": \"val_145\"," \ + "\"key_146\": \"val_146\"," \ + "\"key_147\": \"val_147\"," \ + "\"key_148\": \"val_148\"," \ + "\"key_149\": \"val_149\"," \ + "\"key_150\": \"val_150\"," \ + "\"key_151\": \"val_151\"," \ + "\"key_152\": \"val_152\"," \ + "\"key_153\": \"val_153\"," \ + "\"key_154\": \"val_154\"," \ + "\"key_155\": \"val_155\"," \ + "\"key_156\": \"val_156\"," \ + "\"key_157\": \"val_157\"," \ + "\"key_158\": \"val_158\"," \ + "\"key_159\": \"val_159\"," \ + "\"key_160\": \"val_160\"," \ + "\"key_161\": \"val_161\"," \ + "\"key_162\": \"val_162\"," \ + "\"key_163\": \"val_163\"," \ + "\"key_164\": \"val_164\"," \ + "\"key_165\": \"val_165\"," \ + "\"key_166\": \"val_166\"," \ + "\"key_167\": \"val_167\"," \ + "\"key_168\": \"val_168\"," \ + "\"key_169\": \"val_169\"," \ + "\"key_170\": \"val_170\"," \ + "\"key_171\": \"val_171\"," \ + "\"key_172\": \"val_172\"," \ + "\"key_173\": \"val_173\"," \ + "\"key_174\": \"val_174\"," \ + "\"key_175\": \"val_175\"," \ + "\"key_176\": \"val_176\"," \ + "\"key_177\": \"val_177\"," \ + "\"key_178\": \"val_178\"," \ + "\"key_179\": \"val_179\"," \ + "\"key_180\": \"val_180\"," \ + "\"key_181\": \"val_181\"," \ + "\"key_182\": \"val_182\"," \ + "\"key_183\": \"val_183\"," \ + "\"key_184\": \"val_184\"," \ + "\"key_185\": \"val_185\"," \ + "\"key_186\": \"val_186\"," \ + "\"key_187\": \"val_187\"," \ + "\"key_188\": \"val_188\"," \ + "\"key_189\": \"val_189\"," \ + "\"key_190\": \"val_190\"," \ + "\"key_191\": \"val_191\"," \ + "\"key_192\": \"val_192\"," \ + "\"key_193\": \"val_193\"," \ + "\"key_194\": \"val_194\"," \ + "\"key_195\": \"val_195\"," \ + "\"key_196\": \"val_196\"," \ + "\"key_197\": \"val_197\"," \ + "\"key_198\": \"val_198\"," \ + "\"key_199\": \"val_199\"," \ + "\"key_200\": \"val_200\"," \ + "\"key_201\": \"val_201\"," \ + "\"key_202\": \"val_202\"," \ + "\"key_203\": \"val_203\"," \ + "\"key_204\": \"val_204\"," \ + "\"key_205\": \"val_205\"," \ + "\"key_206\": \"val_206\"," \ + "\"key_207\": \"val_207\"," \ + "\"key_208\": \"val_208\"," \ + "\"key_209\": \"val_209\"," \ + "\"key_210\": \"val_210\"," \ + "\"key_211\": \"val_211\"," \ + "\"key_212\": \"val_212\"," \ + "\"key_213\": \"val_213\"," \ + "\"key_214\": \"val_214\"," \ + "\"key_215\": \"val_215\"," \ + "\"key_216\": \"val_216\"," \ + "\"key_217\": \"val_217\"," \ + "\"key_218\": \"val_218\"," \ + "\"key_219\": \"val_219\"," \ + "\"key_220\": \"val_220\"," \ + "\"key_221\": \"val_221\"," \ + "\"key_222\": \"val_222\"," \ + "\"key_223\": \"val_223\"," \ + "\"key_224\": \"val_224\"," \ + "\"key_225\": \"val_225\"," \ + "\"key_226\": \"val_226\"," \ + "\"key_227\": \"val_227\"," \ + "\"key_228\": \"val_228\"," \ + "\"key_229\": \"val_229\"," \ + "\"key_230\": \"val_230\"," \ + "\"key_231\": \"val_231\"," \ + "\"key_232\": \"val_232\"," \ + "\"key_233\": \"val_233\"," \ + "\"key_234\": \"val_234\"," \ + "\"key_235\": \"val_235\"," \ + "\"key_236\": \"val_236\"," \ + "\"key_237\": \"val_237\"," \ + "\"key_238\": \"val_238\"," \ + "\"key_239\": \"val_239\"," \ + "\"key_240\": \"val_240\"," \ + "\"key_241\": \"val_241\"," \ + "\"key_242\": \"val_242\"," \ + "\"key_243\": \"val_243\"," \ + "\"key_244\": \"val_244\"," \ + "\"key_245\": \"val_245\"," \ + "\"key_246\": \"val_246\"," \ + "\"key_247\": \"val_247\"," \ + "\"key_248\": \"val_248\"," \ + "\"key_249\": \"val_249\"," \ + "\"key_250\": \"val_250\"," \ + "\"key_251\": \"val_251\"," \ + "\"key_252\": \"val_252\"," \ + "\"key_253\": \"val_253\"," \ + "\"key_254\": \"val_254\"," \ + "\"key_255\": \"val_255\"," \ + "\"key_256\": \"val_256\"," \ + "\"key_257\": \"val_257\"," \ + "\"key_258\": \"val_258\"," \ + "\"key_259\": \"val_259\"," \ + "\"key_260\": \"val_260\"," \ + "\"key_261\": \"val_261\"," \ + "\"key_262\": \"val_262\"," \ + "\"key_263\": \"val_263\"," \ + "\"key_264\": \"val_264\"," \ + "\"key_265\": \"val_265\"," \ + "\"key_266\": \"val_266\"," \ + "\"key_267\": \"val_267\"," \ + "\"key_268\": \"val_268\"," \ + "\"key_269\": \"val_269\"," \ + "\"key_270\": \"val_270\"," \ + "\"key_271\": \"val_271\"," \ + "\"key_272\": \"val_272\"," \ + "\"key_273\": \"val_273\"," \ + "\"key_274\": \"val_274\"," \ + "\"key_275\": \"val_275\"," \ + "\"key_276\": \"val_276\"," \ + "\"key_277\": \"val_277\"," \ + "\"key_278\": \"val_278\"," \ + "\"key_279\": \"val_279\"," \ + "\"key_280\": \"val_280\"," \ + "\"key_281\": \"val_281\"," \ + "\"key_282\": \"val_282\"," \ + "\"key_283\": \"val_283\"," \ + "\"key_284\": \"val_284\"," \ + "\"key_285\": \"val_285\"," \ + "\"key_286\": \"val_286\"," \ + "\"key_287\": \"val_287\"," \ + "\"key_288\": \"val_288\"," \ + "\"key_289\": \"val_289\"," \ + "\"key_290\": \"val_290\"," \ + "\"key_291\": \"val_291\"," \ + "\"key_292\": \"val_292\"," \ + "\"key_293\": \"val_293\"," \ + "\"key_294\": \"val_294\"," \ + "\"key_295\": \"val_295\"," \ + "\"key_296\": \"val_296\"," \ + "\"key_297\": \"val_297\"," \ + "\"key_298\": \"val_298\"," \ + "\"key_299\": \"val_299\"," \ + "\"key_300\": \"val_300\"," \ + "\"key_301\": \"val_301\"," \ + "\"key_302\": \"val_302\"," \ + "\"key_303\": \"val_303\"," \ + "\"key_304\": \"val_304\"," \ + "\"key_305\": \"val_305\"," \ + "\"key_306\": \"val_306\"," \ + "\"key_307\": \"val_307\"," \ + "\"key_308\": \"val_308\"," \ + "\"key_309\": \"val_309\"," \ + "\"key_310\": \"val_310\"," \ + "\"key_311\": \"val_311\"," \ + "\"key_312\": \"val_312\"," \ + "\"key_313\": \"val_313\"," \ + "\"key_314\": \"val_314\"," \ + "\"key_315\": \"val_315\"," \ + "\"key_316\": \"val_316\"," \ + "\"key_317\": \"val_317\"," \ + "\"key_318\": \"val_318\"," \ + "\"key_319\": \"val_319\"," \ + "\"key_320\": \"val_320\"," \ + "\"key_321\": \"val_321\"," \ + "\"key_322\": \"val_322\"," \ + "\"key_323\": \"val_323\"," \ + "\"key_324\": \"val_324\"," \ + "\"key_325\": \"val_325\"," \ + "\"key_326\": \"val_326\"," \ + "\"key_327\": \"val_327\"," \ + "\"key_328\": \"val_328\"," \ + "\"key_329\": \"val_329\"," \ + "\"key_330\": \"val_330\"," \ + "\"key_331\": \"val_331\"," \ + "\"key_332\": \"val_332\"," \ + "\"key_333\": \"val_333\"," \ + "\"key_334\": \"val_334\"," \ + "\"key_335\": \"val_335\"," \ + "\"key_336\": \"val_336\"," \ + "\"key_337\": \"val_337\"," \ + "\"key_338\": \"val_338\"," \ + "\"key_339\": \"val_339\"," \ + "\"key_340\": \"val_340\"," \ + "\"key_341\": \"val_341\"," \ + "\"key_342\": \"val_342\"," \ + "\"key_343\": \"val_343\"," \ + "\"key_344\": \"val_344\"," \ + "\"key_345\": \"val_345\"," \ + "\"key_346\": \"val_346\"," \ + "\"key_347\": \"val_347\"," \ + "\"key_348\": \"val_348\"," \ + "\"key_349\": \"val_349\"," \ + "\"key_350\": \"val_350\"," \ + "\"key_351\": \"val_351\"," \ + "\"key_352\": \"val_352\"," \ + "\"key_353\": \"val_353\"," \ + "\"key_354\": \"val_354\"," \ + "\"key_355\": \"val_355\"," \ + "\"key_356\": \"val_356\"," \ + "\"key_357\": \"val_357\"," \ + "\"key_358\": \"val_358\"," \ + "\"key_359\": \"val_359\"," \ + "\"key_360\": \"val_360\"," \ + "\"key_361\": \"val_361\"," \ + "\"key_362\": \"val_362\"," \ + "\"key_363\": \"val_363\"," \ + "\"key_364\": \"val_364\"," \ + "\"key_365\": \"val_365\"," \ + "\"key_366\": \"val_366\"," \ + "\"key_367\": \"val_367\"," \ + "\"key_368\": \"val_368\"," \ + "\"key_369\": \"val_369\"," \ + "\"key_370\": \"val_370\"," \ + "\"key_371\": \"val_371\"," \ + "\"key_372\": \"val_372\"," \ + "\"key_373\": \"val_373\"," \ + "\"key_374\": \"val_374\"," \ + "\"key_375\": \"val_375\"," \ + "\"key_376\": \"val_376\"," \ + "\"key_377\": \"val_377\"," \ + "\"key_378\": \"val_378\"," \ + "\"key_379\": \"val_379\"," \ + "\"key_380\": \"val_380\"," \ + "\"key_381\": \"val_381\"," \ + "\"key_382\": \"val_382\"," \ + "\"key_383\": \"val_383\"," \ + "\"key_384\": \"val_384\"," \ + "\"key_385\": \"val_385\"," \ + "\"key_386\": \"val_386\"," \ + "\"key_387\": \"val_387\"," \ + "\"key_388\": \"val_388\"," \ + "\"key_389\": \"val_389\"," \ + "\"key_390\": \"val_390\"," \ + "\"key_391\": \"val_391\"," \ + "\"key_392\": \"val_392\"," \ + "\"key_393\": \"val_393\"," \ + "\"key_394\": \"val_394\"," \ + "\"key_395\": \"val_395\"," \ + "\"key_396\": \"val_396\"," \ + "\"key_397\": \"val_397\"," \ + "\"key_398\": \"val_398\"," \ + "\"key_399\": \"val_399\"," \ + "\"key_400\": \"val_400\"," \ + "\"key_401\": \"val_401\"," \ + "\"key_402\": \"val_402\"," \ + "\"key_403\": \"val_403\"," \ + "\"key_404\": \"val_404\"," \ + "\"key_405\": \"val_405\"," \ + "\"key_406\": \"val_406\"," \ + "\"key_407\": \"val_407\"," \ + "\"key_408\": \"val_408\"," \ + "\"key_409\": \"val_409\"," \ + "\"key_410\": \"val_410\"," \ + "\"key_411\": \"val_411\"," \ + "\"key_412\": \"val_412\"," \ + "\"key_413\": \"val_413\"," \ + "\"key_414\": \"val_414\"," \ + "\"key_415\": \"val_415\"," \ + "\"key_416\": \"val_416\"," \ + "\"key_417\": \"val_417\"," \ + "\"key_418\": \"val_418\"," \ + "\"key_419\": \"val_419\"," \ + "\"key_420\": \"val_420\"," \ + "\"key_421\": \"val_421\"," \ + "\"key_422\": \"val_422\"," \ + "\"key_423\": \"val_423\"," \ + "\"key_424\": \"val_424\"," \ + "\"key_425\": \"val_425\"," \ + "\"key_426\": \"val_426\"," \ + "\"key_427\": \"val_427\"," \ + "\"key_428\": \"val_428\"," \ + "\"key_429\": \"val_429\"," \ + "\"key_430\": \"val_430\"," \ + "\"key_431\": \"val_431\"," \ + "\"key_432\": \"val_432\"," \ + "\"key_433\": \"val_433\"," \ + "\"key_434\": \"val_434\"," \ + "\"key_435\": \"val_435\"," \ + "\"key_436\": \"val_436\"," \ + "\"key_437\": \"val_437\"," \ + "\"key_438\": \"val_438\"," \ + "\"key_439\": \"val_439\"," \ + "\"key_440\": \"val_440\"," \ + "\"key_441\": \"val_441\"," \ + "\"key_442\": \"val_442\"," \ + "\"key_443\": \"val_443\"," \ + "\"key_444\": \"val_444\"," \ + "\"key_445\": \"val_445\"," \ + "\"key_446\": \"val_446\"," \ + "\"key_447\": \"val_447\"," \ + "\"key_448\": \"val_448\"," \ + "\"key_449\": \"val_449\"," \ + "\"key_450\": \"val_450\"," \ + "\"key_451\": \"val_451\"," \ + "\"key_452\": \"val_452\"," \ + "\"key_453\": \"val_453\"," \ + "\"key_454\": \"val_454\"," \ + "\"key_455\": \"val_455\"," \ + "\"key_456\": \"val_456\"," \ + "\"key_457\": \"val_457\"," \ + "\"key_458\": \"val_458\"," \ + "\"key_459\": \"val_459\"," \ + "\"key_460\": \"val_460\"," \ + "\"key_461\": \"val_461\"," \ + "\"key_462\": \"val_462\"," \ + "\"key_463\": \"val_463\"," \ + "\"key_464\": \"val_464\"," \ + "\"key_465\": \"val_465\"," \ + "\"key_466\": \"val_466\"," \ + "\"key_467\": \"val_467\"," \ + "\"key_468\": \"val_468\"," \ + "\"key_469\": \"val_469\"," \ + "\"key_470\": \"val_470\"," \ + "\"key_471\": \"val_471\"," \ + "\"key_472\": \"val_472\"," \ + "\"key_473\": \"val_473\"," \ + "\"key_474\": \"val_474\"," \ + "\"key_475\": \"val_475\"," \ + "\"key_476\": \"val_476\"," \ + "\"key_477\": \"val_477\"," \ + "\"key_478\": \"val_478\"," \ + "\"key_479\": \"val_479\"," \ + "\"key_480\": \"val_480\"," \ + "\"key_481\": \"val_481\"," \ + "\"key_482\": \"val_482\"," \ + "\"key_483\": \"val_483\"," \ + "\"key_484\": \"val_484\"," \ + "\"key_485\": \"val_485\"," \ + "\"key_486\": \"val_486\"," \ + "\"key_487\": \"val_487\"," \ + "\"key_488\": \"val_488\"," \ + "\"key_489\": \"val_489\"," \ + "\"key_490\": \"val_490\"," \ + "\"key_491\": \"val_491\"," \ + "\"key_492\": \"val_492\"," \ + "\"key_493\": \"val_493\"," \ + "\"key_494\": \"val_494\"," \ + "\"key_495\": \"val_495\"," \ + "\"key_496\": \"val_496\"," \ + "\"key_497\": \"val_497\"," \ + "\"key_498\": \"val_498\"," \ + "\"key_499\": \"val_499\"," \ + "\"END_KEY\": \"JSON_END\"" \ + "}]" + diff --git a/src/fluent-bit/tests/runtime/data/tls/certificate.pem b/src/fluent-bit/tests/runtime/data/tls/certificate.pem new file mode 100644 index 000000000..b79719873 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/tls/certificate.pem @@ -0,0 +1,22 @@ +-----BEGIN CERTIFICATE----- +MIIDlzCCAn+gAwIBAgIUCl24aoQh3HuyObboivsozKKOb3cwDQYJKoZIhvcNAQEL +BQAwWzELMAkGA1UEBhMCQVUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoM +GEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDEUMBIGA1UEAwwLbGVvLnZjYXAubWUw +HhcNMjIwODE3MTM0OTAyWhcNMjMwODE3MTM0OTAyWjBbMQswCQYDVQQGEwJBVTET +MBEGA1UECAwKU29tZS1TdGF0ZTEhMB8GA1UECgwYSW50ZXJuZXQgV2lkZ2l0cyBQ +dHkgTHRkMRQwEgYDVQQDDAtsZW8udmNhcC5tZTCCASIwDQYJKoZIhvcNAQEBBQAD +ggEPADCCAQoCggEBAJ2cbLtGhSeGPAXKitw+PN7U8CkaSGL7BssPFSuqtuVkJ1AH +puluHq62FHm4UgTyJB/Iundf1kBxiOehGFr9E1fGp6kUety6iBZK3mmmX0NdXY2A +ObqfvxTgPNVPEj+5Qp0CJW6fEBRhHrFgpf+OdKzoKnjSCbl1EyJ1wFZeSNDTx98I +h951xEOCobcXj1qXHDd0snRW5QNKTdSnj+3amqrNqV5DrkCyyb5Rq/VQX9LIyGV8 +tOyq50mUKeK+vOu/oNwfkZqgZ5wlY/AUyPMNvd8UKitnWeQMiMmr3ImQRZLkFnoE +2mi2Xr6ZQhmsweWbTmmUYxr+ej7L5WvBVQEkbd0CAwEAAaNTMFEwHQYDVR0OBBYE +FGliHqbvEsRN36C2w1cCq5M/gGCRMB8GA1UdIwQYMBaAFGliHqbvEsRN36C2w1cC +q5M/gGCRMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBACWCNohz +YbnnDzGXMhGlfV5PywH6+gguqTrtzJlfJB/FGHDMybBvEp9gda5ssPZfPdv6aAvo +hz2XC+uwJqYn76ZcX+LOixsqhoXJ367pT0ggMSUB/UNvGkzMFWuGerxcB5Q7xvsh +0Ui37Ojok7Ozj9Ofk/zroJB1WG3kPDtwoo+GwNmJaDtXhOutRPxCqp5Dw+dC0N0c +TlPCBRB17hx49Zjw+h9sv1VmOUtQcplNZgRL9lguv7eXB6y0sSdpDyDLlTBbRMGl +uVnQ20uKnqqALRfpXDWFL8g01xPNEIfV3KctpJfu1inLWtPuklerl0yeyUh9M1f0 +zCAi0URsGEHMFA0= +-----END CERTIFICATE----- diff --git a/src/fluent-bit/tests/runtime/data/tls/private_key.pem b/src/fluent-bit/tests/runtime/data/tls/private_key.pem new file mode 100644 index 000000000..b6a705395 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/tls/private_key.pem @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCdnGy7RoUnhjwF +yorcPjze1PApGkhi+wbLDxUrqrblZCdQB6bpbh6uthR5uFIE8iQfyLp3X9ZAcYjn +oRha/RNXxqepFHrcuogWSt5ppl9DXV2NgDm6n78U4DzVTxI/uUKdAiVunxAUYR6x +YKX/jnSs6Cp40gm5dRMidcBWXkjQ08ffCIfedcRDgqG3F49alxw3dLJ0VuUDSk3U +p4/t2pqqzaleQ65Assm+Uav1UF/SyMhlfLTsqudJlCnivrzrv6DcH5GaoGecJWPw +FMjzDb3fFCorZ1nkDIjJq9yJkEWS5BZ6BNpotl6+mUIZrMHlm05plGMa/no+y+Vr +wVUBJG3dAgMBAAECggEANXQ/AEkLkfsZ0lD+RXIqTNzlUttiH4fJpwbHhFbSzvvn +xWHC/zpk15ZTXXDhCGJjVBBNBX2QeazH5N8jFoDslYF/jX2vqbrturnLswNFHeDF +gN8zNRNGyDrBBwtZQhl/+SYoMdtqpa7GrRv9UK4s7hOTjASYXbjSM4bCI8i4Y3JX +5iKYnejTJ4GNXm1S5g4H8QpgXxrxYU7HY1UlMBGni1BSDPb5SxANnFvgw0oypaDG +4a0KpnUmziBtZo8wIQ7+UeStS4xikIeaT2SQM/ilplPNx0HgPHbTMeTBHQylG1JU +ztEE04Y1F9UF7cqZnxqQLlKmIqsOepLeDOkRXjDtNQKBgQDQtWYbewCmGuo1+AR7 +wStOXpfhqdu6NxAwX+S06ZdAV7f4duMFb//6uJ/c997qwRlNvRvpPlJBdcT3qziS +zjvR8uk2YN5N9D+ZN0Yi11vltUYVPeGpoAk4L7+Ruj5RTVqk5iZEZ1RbAv5cRIm0 +D24H4W0Cj/aldDED2pwTb9ukEwKBgQDBUv/s7gUNM3LCkNLc1EDg2adSzoUopxHU +jIQCitXt5iFpY2Lv2tpaIz4ux4HQGwhmao2MiUYOMit0++/pFaKur66tgRQRsr7F +uS1fpjqtS51/5uFuc1xcWjVquidZXH9u9iChrYvim6SBkxzFsbu4kXNOj4I562cm +E2VWH5GETwKBgQCo5ePX4VbJFYbsXeXi8JRHO63V5Uv4Co+DVlcTQOYyH8q1vCBE +Sjrxf29/tugjOllr29o2i0StzMy1UU7bHyKx6M5qP0In+71sFJshnv6ziltI3Wc9 +ilFrstho6jt8OAle4RGe0bAmZunJaX22xbXZkshRBognpTv1Tnh4ElHBGQKBgHIW +nVohjXGg7xTLiuUvjaokSI6hugunrOoWksE9VcqziPw83uJV8Y5IRiYtLvq1OVvX +ffl1+ZXfHa5ID+kqD3uvyhIynrljFxpwkcpkuzQR77zPcDJSeis2QVfey+H8qGe/ +cLp5RJhS6d5eBxjULshZbgbqwhuURKc/wwn0T1gZAoGAXu2wqwBE8ikhpaDVFYZR +EZe0G1iq03CtabFL0oPiddytV9jJy4Mg457tkCj58uCVkxw/8qjXEnm5OoJAzSlJ +Goc4wh1N1rngYRbSGG+LxD0DeI/Hj3T5tWezMBLCE7J+hieUuVjOB9ELKip7aTj3 +KLNVudOATWYGXyvzux6ciu0= +-----END PRIVATE KEY----- diff --git a/src/fluent-bit/tests/runtime/data/wasm/append_tag.wasm b/src/fluent-bit/tests/runtime/data/wasm/append_tag.wasm Binary files differnew file mode 100755 index 000000000..437fee4ee --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/wasm/append_tag.wasm diff --git a/src/fluent-bit/tests/runtime/data/wasm/drop_record.wasm b/src/fluent-bit/tests/runtime/data/wasm/drop_record.wasm Binary files differnew file mode 100755 index 000000000..5b884f386 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/wasm/drop_record.wasm diff --git a/src/fluent-bit/tests/runtime/data/wasm/modify_record.wasm b/src/fluent-bit/tests/runtime/data/wasm/modify_record.wasm Binary files differnew file mode 100755 index 000000000..30db41571 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/wasm/modify_record.wasm diff --git a/src/fluent-bit/tests/runtime/data/wasm/numeric_records.wasm b/src/fluent-bit/tests/runtime/data/wasm/numeric_records.wasm Binary files differnew file mode 100755 index 000000000..8fc35f0d5 --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/wasm/numeric_records.wasm diff --git a/src/fluent-bit/tests/runtime/data/wasm/say_hello.wasm b/src/fluent-bit/tests/runtime/data/wasm/say_hello.wasm Binary files differnew file mode 100755 index 000000000..7e9d01d2f --- /dev/null +++ b/src/fluent-bit/tests/runtime/data/wasm/say_hello.wasm diff --git a/src/fluent-bit/tests/runtime/filter_aws.c b/src/fluent-bit/tests/runtime/filter_aws.c new file mode 100644 index 000000000..770915a01 --- /dev/null +++ b/src/fluent-bit/tests/runtime/filter_aws.c @@ -0,0 +1,846 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +#include <stdlib.h> +#include <fluent-bit.h> +#include <fluent-bit/flb_time.h> +#include "flb_tests_runtime.h" +#include "../../plugins/filter_aws/aws.h" + +#include "../include/aws_client_mock.h" +#include "../include/aws_client_mock.c" + +pthread_mutex_t result_mutex = PTHREAD_MUTEX_INITIALIZER; +char *output = NULL; + +void set_output(char *val) +{ + pthread_mutex_lock(&result_mutex); + if (output) { + free(output); + } + output = val; + pthread_mutex_unlock(&result_mutex); +} + +char *get_output(void) +{ + char *val; + + pthread_mutex_lock(&result_mutex); + val = output; + pthread_mutex_unlock(&result_mutex); + + return val; +} + +int callback_test(void* data, size_t size, void* cb_data) +{ + if (size > 0) { + flb_debug("[test_filter_aws] received message: %s", (char*)data); + set_output(data); /* success */ + } + return 0; +} + +void flb_test_aws_ec2_tags_present() { + int ret; + int bytes; + char *p = "[0, {\"log\": \"hello, from my ec2 instance\"}]"; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + int filter_ffd; + struct flb_lib_out_cb cb_data; + struct flb_aws_client_generator *client_generator; + struct flb_filter_aws_init_options *ops; + struct flb_aws_client_mock_request_chain *request_chain; + char *output = NULL; + char *result; + + request_chain = FLB_AWS_CLIENT_MOCK( + response( + expect(URI, "/latest/meta-data/tags/instance"), + expect(METHOD, FLB_HTTP_GET), + set(STATUS, 200), + set(PAYLOAD, "Name\nCUSTOMER_ID\nthis-would-be-my-very-long-tag-name-does-it-work"), + set(PAYLOAD_SIZE, 65) + ), + response( + expect(URI, "/latest/meta-data/tags/instance/Name"), + expect(METHOD, FLB_HTTP_GET), + set(STATUS, 200), + set(PAYLOAD, "my_ec2_instance"), + set(PAYLOAD_SIZE, 15) + ), + response( + expect(URI, "/latest/meta-data/tags/instance/CUSTOMER_ID"), + expect(METHOD, FLB_HTTP_GET), + set(STATUS, 200), + set(PAYLOAD, "70ec5c04-3a6e-11ed-a261-0242ac120002"), + set(PAYLOAD_SIZE, 36) + ), + response( + expect(URI, "/latest/meta-data/tags/instance/this-would-be-my-very-long-tag-name-does-it-work"), + expect(METHOD, FLB_HTTP_GET), + set(STATUS, 200), + set(PAYLOAD, "yes-it-does"), + set(PAYLOAD_SIZE, 11) + ) + ); + flb_aws_client_mock_configure_generator(request_chain); + + client_generator = flb_aws_client_get_mock_generator(); + ops = flb_calloc(1, sizeof(struct flb_filter_aws_init_options)); + if (ops == NULL) { + TEST_MSG("calloc for aws plugin options failed\n"); + TEST_CHECK(false); + return; + } + ops->client_generator = client_generator; + + ctx = flb_create(); + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + + /* Prepare output callback context*/ + cb_data.cb = callback_test; + cb_data.data = NULL; + + /* Lib output */ + out_ffd = flb_output(ctx, (char *) "lib", (void *)&cb_data); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, + "match", "*", + "format", "json", + NULL); + + filter_ffd = flb_filter(ctx, (char *) "aws", ops); + TEST_CHECK(filter_ffd >= 0); + ret = flb_filter_set(ctx, filter_ffd, "match", "*", NULL); + TEST_CHECK(ret == 0); + ret = flb_filter_set(ctx, filter_ffd, "ec2_instance_id", "false", NULL); + TEST_CHECK(ret == 0); + ret = flb_filter_set(ctx, filter_ffd, "az", "false", NULL); + TEST_CHECK(ret == 0); + ret = flb_filter_set(ctx, filter_ffd, "tags_enabled", "true", NULL); + TEST_CHECK(ret == 0); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + bytes = flb_lib_push(ctx, in_ffd, p, strlen(p)); + if (!TEST_CHECK(bytes > 0)) { + TEST_MSG("zero bytes were pushed\n"); + } + + flb_time_msleep(1500); /* waiting flush */ + + output = get_output(); + if (output) { + result = strstr(output, "\"Name\":\"my_ec2_instance\""); + if (!TEST_CHECK(result != NULL)) { + TEST_MSG("output:%s\n", output); + } + result = strstr(output, "\"CUSTOMER_ID\":\"70ec5c04-3a6e-11ed-a261-0242ac120002\""); + if (!TEST_CHECK(result != NULL)) { + TEST_MSG("output:%s\n", output); + } + result = strstr(output, "\"this-would-be-my-very-long-tag-name-does-it-work\":\"yes-it-does\""); + if (!TEST_CHECK(result != NULL)) { + TEST_MSG("output:%s\n", output); + } + result = strstr(output, "hello, from my ec2 instance"); + if (!TEST_CHECK(result != NULL)) { + TEST_MSG("output:%s\n", output); + } + } + else { + TEST_CHECK(false); + TEST_MSG("output is empty\n"); + } + + flb_stop(ctx); + flb_aws_client_mock_destroy_generator(); + flb_destroy(ctx); + flb_free(ops); + + set_output(NULL); +} + +void flb_test_aws_ec2_tags_404() { + int ret; + int bytes; + char *p = "[0, {\"log\": \"hello, from my ec2 instance\"}]"; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + int filter_ffd; + struct flb_lib_out_cb cb_data; + struct flb_aws_client_generator *client_generator; + struct flb_filter_aws_init_options *ops; + struct flb_aws_client_mock_request_chain *request_chain; + char *output = NULL; + char *result; + + request_chain = FLB_AWS_CLIENT_MOCK( + response( + expect(URI, "/latest/meta-data/tags/instance"), + expect(METHOD, FLB_HTTP_GET), + set(STATUS, 404) + ) + ); + flb_aws_client_mock_configure_generator(request_chain); + + client_generator = flb_aws_client_get_mock_generator(); + ops = flb_calloc(1, sizeof(struct flb_filter_aws_init_options)); + if (ops == NULL) { + TEST_MSG("calloc for aws plugin options failed\n"); + TEST_CHECK(false); + return; + } + ops->client_generator = client_generator; + + ctx = flb_create(); + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + + /* Prepare output callback context*/ + cb_data.cb = callback_test; + cb_data.data = NULL; + + /* Lib output */ + out_ffd = flb_output(ctx, (char *) "lib", (void *)&cb_data); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, + "match", "*", + "format", "json", + NULL); + + filter_ffd = flb_filter(ctx, (char *) "aws", ops); + TEST_CHECK(filter_ffd >= 0); + ret = flb_filter_set(ctx, filter_ffd, "match", "*", NULL); + TEST_CHECK(ret == 0); + ret = flb_filter_set(ctx, filter_ffd, "ec2_instance_id", "false", NULL); + TEST_CHECK(ret == 0); + ret = flb_filter_set(ctx, filter_ffd, "az", "false", NULL); + TEST_CHECK(ret == 0); + ret = flb_filter_set(ctx, filter_ffd, "tags_enabled", "true", NULL); + TEST_CHECK(ret == 0); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + bytes = flb_lib_push(ctx, in_ffd, p, strlen(p)); + if (!TEST_CHECK(bytes > 0)) { + TEST_MSG("zero bytes were pushed\n"); + } + + flb_time_msleep(1500); /* waiting flush */ + + output = get_output(); + if (output) { + result = strstr(output, "\"Name\":\"my_ec2_instance\""); + if (!TEST_CHECK(result == NULL)) { + TEST_MSG("output:%s\n", output); + } + result = strstr(output, "\"CUSTOMER_ID\":\"70ec5c04-3a6e-11ed-a261-0242ac120002\""); + if (!TEST_CHECK(result == NULL)) { + TEST_MSG("output:%s\n", output); + } + result = strstr(output, "hello, from my ec2 instance"); + if (!TEST_CHECK(result != NULL)) { + TEST_MSG("output:%s\n", output); + } + } + else { + TEST_CHECK(false); + TEST_MSG("output is empty"); + } + + flb_stop(ctx); + flb_aws_client_mock_destroy_generator(); + flb_destroy(ctx); + flb_free(ops); + + set_output(NULL); +} + +void flb_test_aws_ec2_tags_list_500() { + int ret; + int bytes; + char *p = "[0, {\"log\": \"hello, from my ec2 instance\"}]"; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + int filter_ffd; + struct flb_lib_out_cb cb_data; + struct flb_aws_client_generator *client_generator; + struct flb_filter_aws_init_options *ops; + struct flb_aws_client_mock_request_chain *request_chain; + char *output = NULL; + char *result; + + request_chain = FLB_AWS_CLIENT_MOCK( + response( + expect(URI, "/latest/meta-data/tags/instance"), + expect(METHOD, FLB_HTTP_GET), + set(STATUS, 500) + ), + response( + expect(URI, "/latest/meta-data/tags/instance"), + expect(METHOD, FLB_HTTP_GET), + set(STATUS, 500) + ) + ); + flb_aws_client_mock_configure_generator(request_chain); + + client_generator = flb_aws_client_get_mock_generator(); + ops = flb_calloc(1, sizeof(struct flb_filter_aws_init_options)); + if (ops == NULL) { + TEST_MSG("calloc for aws plugin options failed\n"); + TEST_CHECK(false); + return; + } + ops->client_generator = client_generator; + + ctx = flb_create(); + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + + /* Prepare output callback context*/ + cb_data.cb = callback_test; + cb_data.data = NULL; + + /* Lib output */ + out_ffd = flb_output(ctx, (char *) "lib", (void *)&cb_data); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, + "match", "*", + "format", "json", + NULL); + + filter_ffd = flb_filter(ctx, (char *) "aws", ops); + TEST_CHECK(filter_ffd >= 0); + ret = flb_filter_set(ctx, filter_ffd, "match", "*", NULL); + TEST_CHECK(ret == 0); + ret = flb_filter_set(ctx, filter_ffd, "ec2_instance_id", "false", NULL); + TEST_CHECK(ret == 0); + ret = flb_filter_set(ctx, filter_ffd, "az", "false", NULL); + TEST_CHECK(ret == 0); + ret = flb_filter_set(ctx, filter_ffd, "tags_enabled", "true", NULL); + TEST_CHECK(ret == 0); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + bytes = flb_lib_push(ctx, in_ffd, p, strlen(p)); + if (!TEST_CHECK(bytes > 0)) { + TEST_MSG("zero bytes were pushed\n"); + } + + flb_time_msleep(1500); /* waiting flush */ + + output = get_output(); + if (output) { + result = strstr(output, "hello, from my ec2 instance"); + if (!TEST_CHECK(result != NULL)) { + TEST_MSG("output:%s\n", output); + } + } + else { + TEST_CHECK(false); + TEST_MSG("output is empty"); + } + + flb_stop(ctx); + flb_aws_client_mock_destroy_generator(); + flb_destroy(ctx); + flb_free(ops); + + set_output(NULL); +} + +void flb_test_aws_ec2_tags_value_404() { + int ret; + int bytes; + char *p = "[0, {\"log\": \"hello, from my ec2 instance\"}]"; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + int filter_ffd; + struct flb_lib_out_cb cb_data; + struct flb_aws_client_generator *client_generator; + struct flb_filter_aws_init_options *ops; + struct flb_aws_client_mock_request_chain *request_chain; + char *output = NULL; + char *result; + + request_chain = FLB_AWS_CLIENT_MOCK( + response( + expect(URI, "/latest/meta-data/tags/instance"), + expect(METHOD, FLB_HTTP_GET), + set(STATUS, 200), + set(PAYLOAD, "Name"), + set(PAYLOAD_SIZE, 4) + ), + response( + expect(URI, "/latest/meta-data/tags/instance/Name"), + expect(METHOD, FLB_HTTP_GET), + set(STATUS, 404) + ), + response( + expect(URI, "/latest/meta-data/tags/instance"), + expect(METHOD, FLB_HTTP_GET), + set(STATUS, 200), + set(PAYLOAD, "Name"), + set(PAYLOAD_SIZE, 4) + ), + response( + expect(URI, "/latest/meta-data/tags/instance/Name"), + expect(METHOD, FLB_HTTP_GET), + set(STATUS, 404) + ) + ); + flb_aws_client_mock_configure_generator(request_chain); + + client_generator = flb_aws_client_get_mock_generator(); + ops = flb_calloc(1, sizeof(struct flb_filter_aws_init_options)); + if (ops == NULL) { + TEST_MSG("calloc for aws plugin options failed\n"); + TEST_CHECK(false); + return; + } + ops->client_generator = client_generator; + + ctx = flb_create(); + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + + /* Prepare output callback context*/ + cb_data.cb = callback_test; + cb_data.data = NULL; + + /* Lib output */ + out_ffd = flb_output(ctx, (char *) "lib", (void *)&cb_data); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, + "match", "*", + "format", "json", + NULL); + + filter_ffd = flb_filter(ctx, (char *) "aws", ops); + TEST_CHECK(filter_ffd >= 0); + ret = flb_filter_set(ctx, filter_ffd, "match", "*", NULL); + TEST_CHECK(ret == 0); + ret = flb_filter_set(ctx, filter_ffd, "ec2_instance_id", "false", NULL); + TEST_CHECK(ret == 0); + ret = flb_filter_set(ctx, filter_ffd, "az", "false", NULL); + TEST_CHECK(ret == 0); + ret = flb_filter_set(ctx, filter_ffd, "tags_enabled", "true", NULL); + TEST_CHECK(ret == 0); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + bytes = flb_lib_push(ctx, in_ffd, p, strlen(p)); + if (!TEST_CHECK(bytes > 0)) { + TEST_MSG("zero bytes were pushed\n"); + } + + flb_time_msleep(1500); /* waiting flush */ + + output = get_output(); + if (output) { + result = strstr(output, "hello, from my ec2 instance"); + if (!TEST_CHECK(result != NULL)) { + TEST_MSG("output:%s\n", output); + } + } + else { + TEST_CHECK(false); + TEST_MSG("output is empty"); + } + + flb_stop(ctx); + flb_aws_client_mock_destroy_generator(); + flb_destroy(ctx); + flb_free(ops); + + set_output(NULL); +} + +void flb_test_aws_ec2_tags_value_500() { + int ret; + int bytes; + char *p = "[0, {\"log\": \"hello, from my ec2 instance\"}]"; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + int filter_ffd; + struct flb_lib_out_cb cb_data; + struct flb_aws_client_generator *client_generator; + struct flb_filter_aws_init_options *ops; + struct flb_aws_client_mock_request_chain *request_chain; + char *output = NULL; + char *result; + + request_chain = FLB_AWS_CLIENT_MOCK( + response( + expect(URI, "/latest/meta-data/tags/instance"), + expect(METHOD, FLB_HTTP_GET), + set(STATUS, 200), + set(PAYLOAD, "Name"), + set(PAYLOAD_SIZE, 4) + ), + response( + expect(URI, "/latest/meta-data/tags/instance/Name"), + expect(METHOD, FLB_HTTP_GET), + set(STATUS, 500) + ), + response( + expect(URI, "/latest/meta-data/tags/instance"), + expect(METHOD, FLB_HTTP_GET), + set(STATUS, 200), + set(PAYLOAD, "Name"), + set(PAYLOAD_SIZE, 4) + ), + response( + expect(URI, "/latest/meta-data/tags/instance/Name"), + expect(METHOD, FLB_HTTP_GET), + set(STATUS, 500) + ) + ); + flb_aws_client_mock_configure_generator(request_chain); + + client_generator = flb_aws_client_get_mock_generator(); + ops = flb_calloc(1, sizeof(struct flb_filter_aws_init_options)); + if (ops == NULL) { + TEST_MSG("calloc for aws plugin options failed\n"); + TEST_CHECK(false); + return; + } + ops->client_generator = client_generator; + + ctx = flb_create(); + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + + /* Prepare output callback context*/ + cb_data.cb = callback_test; + cb_data.data = NULL; + + /* Lib output */ + out_ffd = flb_output(ctx, (char *) "lib", (void *)&cb_data); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, + "match", "*", + "format", "json", + NULL); + + filter_ffd = flb_filter(ctx, (char *) "aws", ops); + TEST_CHECK(filter_ffd >= 0); + ret = flb_filter_set(ctx, filter_ffd, "match", "*", NULL); + TEST_CHECK(ret == 0); + ret = flb_filter_set(ctx, filter_ffd, "ec2_instance_id", "false", NULL); + TEST_CHECK(ret == 0); + ret = flb_filter_set(ctx, filter_ffd, "az", "false", NULL); + TEST_CHECK(ret == 0); + ret = flb_filter_set(ctx, filter_ffd, "tags_enabled", "true", NULL); + TEST_CHECK(ret == 0); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + bytes = flb_lib_push(ctx, in_ffd, p, strlen(p)); + if (!TEST_CHECK(bytes > 0)) { + TEST_MSG("zero bytes were pushed\n"); + } + + flb_time_msleep(1500); /* waiting flush */ + + output = get_output(); + if (output) { + result = strstr(output, "hello, from my ec2 instance"); + if (!TEST_CHECK(result != NULL)) { + TEST_MSG("output:%s\n", output); + } + } + else { + TEST_CHECK(false); + TEST_MSG("output is empty"); + } + + flb_stop(ctx); + flb_aws_client_mock_destroy_generator(); + flb_destroy(ctx); + flb_free(ops); + + set_output(NULL); +} + +void flb_test_aws_ec2_tags_include() { + int ret; + int bytes; + char *p = "[0, {\"log\": \"hello, from my ec2 instance\"}]"; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + int filter_ffd; + struct flb_lib_out_cb cb_data; + struct flb_aws_client_generator *client_generator; + struct flb_filter_aws_init_options *ops; + struct flb_aws_client_mock_request_chain *request_chain; + char *output = NULL; + char *result; + + request_chain = FLB_AWS_CLIENT_MOCK( + response( + expect(URI, "/latest/meta-data/tags/instance"), + expect(METHOD, FLB_HTTP_GET), + set(STATUS, 200), + set(PAYLOAD, "Name\nCUSTOMER_ID"), + set(PAYLOAD_SIZE, 16) + ), + response( + expect(URI, "/latest/meta-data/tags/instance/Name"), + expect(METHOD, FLB_HTTP_GET), + set(STATUS, 200), + set(PAYLOAD, "my_ec2_instance"), + set(PAYLOAD_SIZE, 15) + ), + response( + expect(URI, "/latest/meta-data/tags/instance/CUSTOMER_ID"), + expect(METHOD, FLB_HTTP_GET), + set(STATUS, 200), + set(PAYLOAD, "70ec5c04-3a6e-11ed-a261-0242ac120002"), + set(PAYLOAD_SIZE, 36) + ) + ); + flb_aws_client_mock_configure_generator(request_chain); + + client_generator = flb_aws_client_get_mock_generator(); + ops = flb_calloc(1, sizeof(struct flb_filter_aws_init_options)); + if (ops == NULL) { + TEST_MSG("calloc for aws plugin options failed\n"); + TEST_CHECK(false); + return; + } + ops->client_generator = client_generator; + + ctx = flb_create(); + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + + /* Prepare output callback context*/ + cb_data.cb = callback_test; + cb_data.data = NULL; + + /* Lib output */ + out_ffd = flb_output(ctx, (char *) "lib", (void *)&cb_data); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, + "match", "*", + "format", "json", + NULL); + + filter_ffd = flb_filter(ctx, (char *) "aws", ops); + TEST_CHECK(filter_ffd >= 0); + ret = flb_filter_set(ctx, filter_ffd, "match", "*", NULL); + TEST_CHECK(ret == 0); + ret = flb_filter_set(ctx, filter_ffd, "ec2_instance_id", "false", NULL); + TEST_CHECK(ret == 0); + ret = flb_filter_set(ctx, filter_ffd, "az", "false", NULL); + TEST_CHECK(ret == 0); + ret = flb_filter_set(ctx, filter_ffd, "tags_enabled", "true", NULL); + TEST_CHECK(ret == 0); + ret = flb_filter_set(ctx, filter_ffd, "tags_include", "Namee,MyTag,Name", NULL); + TEST_CHECK(ret == 0); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + bytes = flb_lib_push(ctx, in_ffd, p, strlen(p)); + if (!TEST_CHECK(bytes > 0)) { + TEST_MSG("zero bytes were pushed\n"); + } + + flb_time_msleep(1500); /* waiting flush */ + + output = get_output(); + if (output) { + result = strstr(output, "\"Name\":\"my_ec2_instance\""); + if (!TEST_CHECK(result != NULL)) { + TEST_MSG("output:%s\n", output); + } + result = strstr(output, "hello, from my ec2 instance"); + if (!TEST_CHECK(result != NULL)) { + TEST_MSG("output:%s\n", output); + } + /* CUSTOMER_ID is not included, so we don't expect it in the log */ + result = strstr(output, "\"CUSTOMER_ID\":\"70ec5c04-3a6e-11ed-a261-0242ac120002\""); + if (!TEST_CHECK(result == NULL)) { + TEST_MSG("output:%s\n", output); + } + } + else { + TEST_CHECK(false); + TEST_MSG("output is empty\n"); + } + + flb_stop(ctx); + flb_aws_client_mock_destroy_generator(); + flb_destroy(ctx); + flb_free(ops); + + set_output(NULL); +} + +void flb_test_aws_ec2_tags_exclude() { + int ret; + int bytes; + char *p = "[0, {\"log\": \"hello, from my ec2 instance\"}]"; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + int filter_ffd; + struct flb_lib_out_cb cb_data; + struct flb_aws_client_generator *client_generator; + struct flb_filter_aws_init_options *ops; + struct flb_aws_client_mock_request_chain *request_chain; + char *output = NULL; + char *result; + + request_chain = FLB_AWS_CLIENT_MOCK( + response( + expect(URI, "/latest/meta-data/tags/instance"), + expect(METHOD, FLB_HTTP_GET), + set(STATUS, 200), + set(PAYLOAD, "Name\nCUSTOMER_ID"), + set(PAYLOAD_SIZE, 16) + ), + response( + expect(URI, "/latest/meta-data/tags/instance/Name"), + expect(METHOD, FLB_HTTP_GET), + set(STATUS, 200), + set(PAYLOAD, "my_ec2_instance"), + set(PAYLOAD_SIZE, 15) + ), + response( + expect(URI, "/latest/meta-data/tags/instance/CUSTOMER_ID"), + expect(METHOD, FLB_HTTP_GET), + set(STATUS, 200), + set(PAYLOAD, "70ec5c04-3a6e-11ed-a261-0242ac120002"), + set(PAYLOAD_SIZE, 36) + ) + ); + flb_aws_client_mock_configure_generator(request_chain); + + client_generator = flb_aws_client_get_mock_generator(); + ops = flb_calloc(1, sizeof(struct flb_filter_aws_init_options)); + if (ops == NULL) { + TEST_MSG("calloc for aws plugin options failed\n"); + TEST_CHECK(false); + return; + } + ops->client_generator = client_generator; + + ctx = flb_create(); + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + + /* Prepare output callback context*/ + cb_data.cb = callback_test; + cb_data.data = NULL; + + /* Lib output */ + out_ffd = flb_output(ctx, (char *) "lib", (void *)&cb_data); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, + "match", "*", + "format", "json", + NULL); + + filter_ffd = flb_filter(ctx, (char *) "aws", ops); + TEST_CHECK(filter_ffd >= 0); + ret = flb_filter_set(ctx, filter_ffd, "match", "*", NULL); + TEST_CHECK(ret == 0); + ret = flb_filter_set(ctx, filter_ffd, "ec2_instance_id", "false", NULL); + TEST_CHECK(ret == 0); + ret = flb_filter_set(ctx, filter_ffd, "az", "false", NULL); + TEST_CHECK(ret == 0); + ret = flb_filter_set(ctx, filter_ffd, "tags_enabled", "true", NULL); + TEST_CHECK(ret == 0); + ret = flb_filter_set(ctx, filter_ffd, "tags_exclude", "Name,Name2", NULL); + TEST_CHECK(ret == 0); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + bytes = flb_lib_push(ctx, in_ffd, p, strlen(p)); + if (!TEST_CHECK(bytes > 0)) { + TEST_MSG("zero bytes were pushed\n"); + } + + flb_time_msleep(1500); /* waiting flush */ + + output = get_output(); + if (output) { + /* Name is excluded, so we don't expect it in the log */ + result = strstr(output, "\"Name\":\"my_ec2_instance\""); + if (!TEST_CHECK(result == NULL)) { + TEST_MSG("output:%s\n", output); + } + result = strstr(output, "hello, from my ec2 instance"); + if (!TEST_CHECK(result != NULL)) { + TEST_MSG("output:%s\n", output); + } + result = strstr(output, "\"CUSTOMER_ID\":\"70ec5c04-3a6e-11ed-a261-0242ac120002\""); + if (!TEST_CHECK(result != NULL)) { + TEST_MSG("output:%s\n", output); + } + } + else { + TEST_CHECK(false); + TEST_MSG("output is empty\n"); + } + + flb_stop(ctx); + flb_aws_client_mock_destroy_generator(); + flb_destroy(ctx); + flb_free(ops); + + set_output(NULL); +} + + +TEST_LIST = { + {"aws_ec2_tags_present", flb_test_aws_ec2_tags_present}, + {"aws_ec2_tags_404", flb_test_aws_ec2_tags_404}, + {"aws_ec2_tags_list_500", flb_test_aws_ec2_tags_list_500}, + {"aws_ec2_tags_value_404", flb_test_aws_ec2_tags_value_404}, + {"aws_ec2_tags_value_500", flb_test_aws_ec2_tags_value_500}, + {"aws_ec2_tags_include", flb_test_aws_ec2_tags_include}, + {"aws_ec2_tags_exclude", flb_test_aws_ec2_tags_exclude}, + {NULL, NULL} +}; diff --git a/src/fluent-bit/tests/runtime/filter_checklist.c b/src/fluent-bit/tests/runtime/filter_checklist.c new file mode 100644 index 000000000..72d1e4815 --- /dev/null +++ b/src/fluent-bit/tests/runtime/filter_checklist.c @@ -0,0 +1,405 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2019-2022 The Fluent Bit Authors + * Copyright (C) 2015-2018 Treasure Data Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <fluent-bit.h> +#include <fluent-bit/flb_time.h> +#include <sys/types.h> +#include <sys/stat.h> +#include <fcntl.h> +#include "flb_tests_runtime.h" + +#define TMP_CHECKLIST_PATH "checklist.txt" + +struct test_ctx { + flb_ctx_t *flb; /* Fluent Bit library context */ + int i_ffd; /* Input fd */ + int f_ffd; /* Filter fd (unused) */ + int o_ffd; /* Output fd */ +}; + +static struct test_ctx *test_ctx_create(struct flb_lib_out_cb *data) +{ + int i_ffd; + int o_ffd; + int f_ffd; + struct test_ctx *ctx = NULL; + + ctx = flb_malloc(sizeof(struct test_ctx)); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("malloc failed"); + flb_errno(); + return NULL; + } + + /* Service config */ + ctx->flb = flb_create(); + flb_service_set(ctx->flb, + "Flush", "0.200000000", + "Grace", "1", + "Log_Level", "error", + NULL); + + /* Input */ + i_ffd = flb_input(ctx->flb, (char *) "lib", NULL); + TEST_CHECK(i_ffd >= 0); + flb_input_set(ctx->flb, i_ffd, "tag", "test", NULL); + ctx->i_ffd = i_ffd; + + /* Filter */ + f_ffd = flb_filter(ctx->flb, (char *) "checklist", NULL); + TEST_CHECK(f_ffd >= 0); + ctx->f_ffd = f_ffd; + + /* Output */ + o_ffd = flb_output(ctx->flb, (char *) "lib", (void *) data); + ctx->o_ffd = o_ffd; + TEST_CHECK(o_ffd >= 0); + flb_output_set(ctx->flb, o_ffd, + "match", "test", + NULL); + + return ctx; +} + +static void test_ctx_destroy(struct test_ctx *ctx) +{ + TEST_CHECK(ctx != NULL); + + sleep(1); + flb_stop(ctx->flb); + flb_destroy(ctx->flb); + flb_free(ctx); +} + +pthread_mutex_t result_mutex = PTHREAD_MUTEX_INITIALIZER; +char *output = NULL; + +void set_output(char *val) +{ + pthread_mutex_lock(&result_mutex); + output = val; + pthread_mutex_unlock(&result_mutex); +} + +char *get_output(void) +{ + char *val; + + pthread_mutex_lock(&result_mutex); + val = output; + pthread_mutex_unlock(&result_mutex); + + return val; +} + +/* Callback to check expected results */ +static int cb_check_result(void *record, size_t size, void *data) +{ + char *p; + char *expected; + char *result; + + expected = (char *) data; + result = (char *) record; + + p = strstr(result, expected); + TEST_CHECK(p != NULL); + + if (p==NULL) { + flb_error("Expected to find: '%s' in result '%s'", + expected, result); + } + /* + * If you want to debug your test + * + * printf("Expect: '%s' in result '%s'", expected, result); + */ + flb_free(record); + return 0; +} + +void delete_checklist() +{ + unlink(TMP_CHECKLIST_PATH); + flb_debug("remove checklist\n"); +} + + +int create_checklist(char *checklist_body, size_t body_size) +{ + FILE *fp = NULL; + fp = fopen(TMP_CHECKLIST_PATH, "w+"); + if (fp == NULL) { + TEST_MSG("fopen error\n"); + return -1; + } + fwrite(checklist_body, body_size, 1, fp); + fflush(fp); + fclose(fp); + return 0; +} + +void flb_test_lookup_key(void) +{ + int ret; + int bytes; + struct test_ctx *ctx; + struct flb_lib_out_cb cb_data; + char *checklist_body = "" + "malicious word\n"; + char *input = "[0, {\"secret\": \"malicious word\"}]"; + + cb_data.cb = cb_check_result; + cb_data.data = "\"secret\":\"----\""; + + ctx = test_ctx_create(&cb_data); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("test_ctx_create failed"); + exit(EXIT_FAILURE); + } + + ret = create_checklist(checklist_body, strlen(checklist_body)); + TEST_CHECK(ret == 0); + /* Filter */ + ret = flb_filter_set(ctx->flb, ctx->f_ffd, + "Match", "*", + "file", TMP_CHECKLIST_PATH, + "lookup_key", "secret", + "record", "secret ----", + NULL); + TEST_CHECK(ret==0); + + ret = flb_output_set(ctx->flb, ctx->o_ffd, + "format", "json", + NULL); + TEST_CHECK(ret==0); + + ret = flb_start(ctx->flb); + TEST_CHECK(ret==0); + + bytes = flb_lib_push(ctx->flb, ctx->i_ffd, input, strlen(input)); + TEST_CHECK(bytes == strlen(input)); + flb_time_msleep(1500); /* waiting flush */ + delete_checklist(); + + test_ctx_destroy(ctx); +} + +void flb_test_lookup_keys(void) +{ + int ret; + int bytes; + struct test_ctx *ctx; + struct flb_lib_out_cb cb_data; + char *checklist_body = "" + "malicious word\n" + "confidential\n"; + + char *input = "[0, {\"secret\": \"malicious word\"}]"; + char *input2 = "[0, {\"secret\": \"confidential\"}]"; + + cb_data.cb = cb_check_result; + cb_data.data = "\"secret\":\"----\""; + + ctx = test_ctx_create(&cb_data); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("test_ctx_create failed"); + exit(EXIT_FAILURE); + } + + ret = create_checklist(checklist_body, strlen(checklist_body)); + TEST_CHECK(ret == 0); + /* Filter */ + ret = flb_filter_set(ctx->flb, ctx->f_ffd, + "Match", "*", + "file", TMP_CHECKLIST_PATH, + "lookup_key", "secret", + "record", "secret ----", + NULL); + TEST_CHECK(ret==0); + + ret = flb_output_set(ctx->flb, ctx->o_ffd, + "format", "json", + NULL); + TEST_CHECK(ret==0); + + ret = flb_start(ctx->flb); + TEST_CHECK(ret==0); + + bytes = flb_lib_push(ctx->flb, ctx->i_ffd, input, strlen(input)); + TEST_CHECK(bytes == strlen(input)); + flb_time_msleep(1500); /* waiting flush */ + bytes = flb_lib_push(ctx->flb, ctx->i_ffd, input2, strlen(input2)); + TEST_CHECK(bytes == strlen(input2)); + flb_time_msleep(1500); /* waiting flush */ + delete_checklist(); + + test_ctx_destroy(ctx); +} + +void flb_test_records(void) +{ + int ret; + int bytes; + struct test_ctx *ctx; + struct flb_lib_out_cb cb_data; + char *checklist_body = "" + "malicious word\n"; + char *input = "[0, {\"secret\": \"malicious word\"}]"; + + cb_data.cb = cb_check_result; + cb_data.data = "\"secret\":\"----\",\"checklist\":true"; + + ctx = test_ctx_create(&cb_data); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("test_ctx_create failed"); + exit(EXIT_FAILURE); + } + + ret = create_checklist(checklist_body, strlen(checklist_body)); + TEST_CHECK(ret == 0); + /* Filter */ + ret = flb_filter_set(ctx->flb, ctx->f_ffd, + "Match", "*", + "file", TMP_CHECKLIST_PATH, + "lookup_key", "secret", + "record", "secret ----", + "record", "checklist true", + NULL); + TEST_CHECK(ret==0); + + ret = flb_output_set(ctx->flb, ctx->o_ffd, + "format", "json", + NULL); + TEST_CHECK(ret==0); + + ret = flb_start(ctx->flb); + TEST_CHECK(ret==0); + + bytes = flb_lib_push(ctx->flb, ctx->i_ffd, input, strlen(input)); + TEST_CHECK(bytes == strlen(input)); + flb_time_msleep(1500); /* waiting flush */ + delete_checklist(); + + test_ctx_destroy(ctx); +} + +void flb_test_ignore_case(void) +{ + int ret; + int bytes; + struct test_ctx *ctx; + struct flb_lib_out_cb cb_data; + char *checklist_body = "" + "MaliCioUs Word\n"; + char *input = "[0, {\"secret\": \"malicious word\"}]"; + + cb_data.cb = cb_check_result; + cb_data.data = "\"secret\":\"----\""; + + ctx = test_ctx_create(&cb_data); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("test_ctx_create failed"); + exit(EXIT_FAILURE); + } + + ret = create_checklist(checklist_body, strlen(checklist_body)); + TEST_CHECK(ret == 0); + /* Filter */ + ret = flb_filter_set(ctx->flb, ctx->f_ffd, + "Match", "*", + "file", TMP_CHECKLIST_PATH, + "lookup_key", "secret", + "record", "secret ----", + "ignore_case", "true", + NULL); + TEST_CHECK(ret==0); + + ret = flb_output_set(ctx->flb, ctx->o_ffd, + "format", "json", + NULL); + TEST_CHECK(ret==0); + + ret = flb_start(ctx->flb); + TEST_CHECK(ret==0); + + bytes = flb_lib_push(ctx->flb, ctx->i_ffd, input, strlen(input)); + TEST_CHECK(bytes == strlen(input)); + flb_time_msleep(1500); /* waiting flush */ + delete_checklist(); + + test_ctx_destroy(ctx); +} + +void flb_test_mode_partial(void) +{ + int ret; + int bytes; + struct test_ctx *ctx; + struct flb_lib_out_cb cb_data; + char *checklist_body = "" + "malicious\n"; + char *input = "[0, {\"secret\": \"malicious word\"}]"; + + cb_data.cb = cb_check_result; + cb_data.data = "\"secret\":\"----\""; + + ctx = test_ctx_create(&cb_data); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("test_ctx_create failed"); + exit(EXIT_FAILURE); + } + + ret = create_checklist(checklist_body, strlen(checklist_body)); + TEST_CHECK(ret == 0); + /* Filter */ + ret = flb_filter_set(ctx->flb, ctx->f_ffd, + "Match", "*", + "file", TMP_CHECKLIST_PATH, + "lookup_key", "secret", + "record", "secret ----", + "mode", "partial", + NULL); + TEST_CHECK(ret==0); + + ret = flb_output_set(ctx->flb, ctx->o_ffd, + "format", "json", + NULL); + TEST_CHECK(ret==0); + + ret = flb_start(ctx->flb); + TEST_CHECK(ret==0); + + bytes = flb_lib_push(ctx->flb, ctx->i_ffd, input, strlen(input)); + TEST_CHECK(bytes == strlen(input)); + flb_time_msleep(1500); /* waiting flush */ + delete_checklist(); + + test_ctx_destroy(ctx); +} + +TEST_LIST = { + {"lookup_key", flb_test_lookup_key}, + {"lookup_keys", flb_test_lookup_keys}, + {"records", flb_test_records}, + {"ignore_case", flb_test_ignore_case}, + {"mode_partial", flb_test_mode_partial}, + {NULL, NULL} +}; diff --git a/src/fluent-bit/tests/runtime/filter_ecs.c b/src/fluent-bit/tests/runtime/filter_ecs.c new file mode 100644 index 000000000..79868294b --- /dev/null +++ b/src/fluent-bit/tests/runtime/filter_ecs.c @@ -0,0 +1,445 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +#include <fluent-bit.h> +#include <fluent-bit/flb_sds.h> +#include "flb_tests_runtime.h" + +#define ERROR_RESPONSE "NOT FOUND" + + +struct filter_test { + flb_ctx_t *flb; /* Fluent Bit library context */ + int i_ffd; /* Input fd */ + int f_ffd; /* Filter fd */ +}; + +struct filter_test_result { + char *expected_pattern; /* string that must occur in output */ + int expected_pattern_index; /* which record to check for the pattern */ + int expected_records; /* expected number of outputted records */ + int actual_records; /* actual number of outputted records */ +}; + +/* Callback to check expected results */ +static int cb_check_result(void *record, size_t size, void *data) +{ + char *p; + struct filter_test_result *expected; + char *result; + + expected = (struct filter_test_result *) data; + result = (char *) record; + + if (expected->expected_pattern_index == expected->actual_records) { + p = strstr(result, expected->expected_pattern); + TEST_CHECK(p != NULL); + + if (!p) { + flb_error("Expected to find: '%s' in result '%s'", + expected->expected_pattern, result); + } + /* + * If you want to debug your test + * + * printf("Expect: '%s' in result '%s'\n", expected->expected_pattern, result); + */ + } + + expected->actual_records++; + + flb_free(record); + return 0; +} + + +struct str_list { + size_t size; /* size of lists */ + int ignore_min_line_num; /* ignore line if the length is less than this value */ + char **lists; /* string lists */ +}; + + +static struct filter_test *filter_test_create(struct flb_lib_out_cb *data, + char *tag) +{ + int i_ffd; + int f_ffd; + int o_ffd; + struct filter_test *ctx; + + ctx = flb_malloc(sizeof(struct filter_test)); + if (!ctx) { + flb_errno(); + return NULL; + } + + /* Service config */ + ctx->flb = flb_create(); + flb_service_set(ctx->flb, + "Flush", "0.200000000", + "Grace", "1", + NULL); + + /* Input */ + i_ffd = flb_input(ctx->flb, (char *) "lib", NULL); + TEST_CHECK(i_ffd >= 0); + /* filter relies on the tag container 12 char short container ID */ + flb_input_set(ctx->flb, i_ffd, "tag", tag, NULL); + ctx->i_ffd = i_ffd; + + /* Filter configuration */ + f_ffd = flb_filter(ctx->flb, (char *) "ecs", NULL); + TEST_CHECK(f_ffd >= 0); + flb_filter_set(ctx->flb, f_ffd, "match", "*", NULL); + ctx->f_ffd = f_ffd; + + /* Output */ + o_ffd = flb_output(ctx->flb, (char *) "lib", (void *) data); + TEST_CHECK(o_ffd >= 0); + flb_output_set(ctx->flb, o_ffd, + "match", "*", + "format", "json", + NULL); + + return ctx; +} + +static void filter_test_destroy(struct filter_test *ctx) +{ + flb_stop(ctx->flb); + flb_destroy(ctx->flb); + flb_free(ctx); +} + +static void flb_test_ecs_filter() +{ + int len; + int ret; + int bytes; + char *p; + struct flb_lib_out_cb cb_data; + struct filter_test *ctx; + struct filter_test_result expected = { 0 }; + + /* mocks calls- signals that we are in test mode */ + setenv("FLB_ECS_PLUGIN_UNDER_TEST", "true", 1); + + /* Create test context */ + ctx = filter_test_create((void *) &cb_data, "testprefix-79c796ed2a7f"); + if (!ctx) { + exit(EXIT_FAILURE); + } + + /* Configure filter */ + ret = flb_filter_set(ctx->flb, ctx->f_ffd, + "ecs_tag_prefix", "testprefix-", + "ADD", "resource $ClusterName.$TaskID.$ECSContainerName", + NULL); + TEST_CHECK(ret == 0); + + /* Prepare output callback with expected result */ + expected.expected_records = 1; /* 1 record with metadata added */ + expected.expected_pattern = "cluster_name.e01d58a8-151b-40e8-bc01-22647b9ecfec.nginx"; + expected.expected_pattern_index = 0; + cb_data.cb = cb_check_result; + cb_data.data = (void *) &expected; + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + /* Ingest data samples */ + p = "[0, {\"log\":\"error: my error\"}]"; + len = strlen(p); + bytes = flb_lib_push(ctx->flb, ctx->i_ffd, p, len); + TEST_CHECK(bytes == len); + + /* check number of outputted records */ + sleep(2); + TEST_CHECK(expected.actual_records == expected.expected_records); + filter_test_destroy(ctx); +} + +/* + * First release of ECS filter could crash + * when saving that it faild to get metadata for a tag + */ +static void flb_test_ecs_filter_mark_tag_failed() +{ + int len; + int ret; + int bytes; + char *p; + struct flb_lib_out_cb cb_data; + struct filter_test *ctx; + struct filter_test_result expected = { 0 }; + + /* mocks calls- signals that we are in test mode */ + setenv("FLB_ECS_PLUGIN_UNDER_TEST", "true", 1); + setenv("TEST_TASK_ERROR", ERROR_RESPONSE, 1); + + /* Create test context */ + ctx = filter_test_create((void *) &cb_data, "testprefix-79c796ed2a7f"); + if (!ctx) { + exit(EXIT_FAILURE); + } + + /* Configure filter */ + ret = flb_filter_set(ctx->flb, ctx->f_ffd, + "ecs_tag_prefix", "testprefix-", + "ADD", "resource $ClusterName.$TaskID.$ECSContainerName", + NULL); + TEST_CHECK(ret == 0); + + /* Prepare output callback with expected result */ + expected.expected_records = 4; /* 4 records with no metadata */ + expected.expected_pattern = ""; + expected.expected_pattern_index = 0; + cb_data.cb = cb_check_result; + cb_data.data = (void *) &expected; + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + /* Ingest data samples */ + p = "[0, {\"log\":\"error: my error\"}]"; + len = strlen(p); + bytes = flb_lib_push(ctx->flb, ctx->i_ffd, p, len); + TEST_CHECK(bytes == len); + sleep(1); + + p = "[0, {\"log\":\"error: my error\"}]"; + len = strlen(p); + bytes = flb_lib_push(ctx->flb, ctx->i_ffd, p, len); + TEST_CHECK(bytes == len); + sleep(1); + + p = "[0, {\"log\":\"error: my error\"}]"; + len = strlen(p); + bytes = flb_lib_push(ctx->flb, ctx->i_ffd, p, len); + TEST_CHECK(bytes == len); + sleep(1); + + p = "[0, {\"log\":\"error: my error\"}]"; + len = strlen(p); + bytes = flb_lib_push(ctx->flb, ctx->i_ffd, p, len); + TEST_CHECK(bytes == len); + sleep(2); + + /* check number of outputted records */ + TEST_CHECK(expected.actual_records == expected.expected_records); + filter_test_destroy(ctx); +} + +static void flb_test_ecs_filter_no_prefix() +{ + int len; + int ret; + int bytes; + char *p; + struct flb_lib_out_cb cb_data; + struct filter_test *ctx; + struct filter_test_result expected = { 0 }; + + /* mocks calls- signals that we are in test mode */ + setenv("FLB_ECS_PLUGIN_UNDER_TEST", "true", 1); + + /* Create test context */ + ctx = filter_test_create((void *) &cb_data, "79c796ed2a7f"); + if (!ctx) { + exit(EXIT_FAILURE); + } + + /* Configure filter */ + ret = flb_filter_set(ctx->flb, ctx->f_ffd, + "ecs_tag_prefix", "", + "ADD", "resource $ClusterName.$TaskID.$ECSContainerName", + NULL); + TEST_CHECK(ret == 0); + + /* Prepare output callback with expected result */ + expected.expected_records = 1; /* 1 record with metadata added */ + expected.expected_pattern = "cluster_name.e01d58a8-151b-40e8-bc01-22647b9ecfec.nginx"; + expected.expected_pattern_index = 0; + cb_data.cb = cb_check_result; + cb_data.data = (void *) &expected; + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + /* Ingest data samples */ + p = "[0, {\"log\":\"error: my error\"}]"; + len = strlen(p); + bytes = flb_lib_push(ctx->flb, ctx->i_ffd, p, len); + TEST_CHECK(bytes == len); + + /* check number of outputted records */ + sleep(2); + TEST_CHECK(expected.actual_records == expected.expected_records); + filter_test_destroy(ctx); +} + +static void flb_test_ecs_filter_cluster_metadata_only() +{ + int len; + int ret; + int bytes; + char *p; + struct flb_lib_out_cb cb_data; + struct filter_test *ctx; + struct filter_test_result expected = { 0 }; + + /* mocks calls- signals that we are in test mode */ + setenv("FLB_ECS_PLUGIN_UNDER_TEST", "true", 1); + + /* Create test context */ + ctx = filter_test_create((void *) &cb_data, "var.lib.ecs.79c796ed2a7f"); + if (!ctx) { + exit(EXIT_FAILURE); + } + + /* Configure filter */ + ret = flb_filter_set(ctx->flb, ctx->f_ffd, + "ecs_tag_prefix", "", + "cluster_metadata_only", "on", + /* only cluster value will be populated */ + "ADD", "resource $ClusterName.$TaskID.$ECSContainerName", + NULL); + TEST_CHECK(ret == 0); + + /* Prepare output callback with expected result */ + expected.expected_records = 1; /* 1 record with only cluster metadata values added */ + expected.expected_pattern = "cluster_name.."; + expected.expected_pattern_index = 0; + cb_data.cb = cb_check_result; + cb_data.data = (void *) &expected; + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + /* Ingest data samples */ + p = "[0, {\"log\":\"error: my error\"}]"; + len = strlen(p); + bytes = flb_lib_push(ctx->flb, ctx->i_ffd, p, len); + TEST_CHECK(bytes == len); + + /* check number of outputted records */ + sleep(2); + TEST_CHECK(expected.actual_records == expected.expected_records); + filter_test_destroy(ctx); +} + +static void flb_test_ecs_filter_cluster_error() +{ + int len; + int ret; + int bytes; + char *p; + struct flb_lib_out_cb cb_data; + struct filter_test *ctx; + struct filter_test_result expected = { 0 }; + + /* mocks calls- signals that we are in test mode */ + setenv("FLB_ECS_PLUGIN_UNDER_TEST", "true", 1); + setenv("TEST_CLUSTER_ERROR", ERROR_RESPONSE, 1); + + /* Create test context */ + ctx = filter_test_create((void *) &cb_data, "79c796ed2a7f"); + if (!ctx) { + exit(EXIT_FAILURE); + } + + /* Configure filter */ + ret = flb_filter_set(ctx->flb, ctx->f_ffd, + "ecs_tag_prefix", "", + "ADD", "resource $ClusterName.$TaskID.$ECSContainerName", + NULL); + TEST_CHECK(ret == 0); + + /* this test is mainly for leak checking on error, not for checking result record */ + expected.expected_records = 1; /* 1 record with no metadata */ + expected.expected_pattern = ""; + expected.expected_pattern_index = 0; + cb_data.cb = cb_check_result; + cb_data.data = (void *) &expected; + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + /* Ingest data samples */ + p = "[0, {\"log\":\"error: my error\"}]"; + len = strlen(p); + bytes = flb_lib_push(ctx->flb, ctx->i_ffd, p, len); + TEST_CHECK(bytes == len); + + /* check number of outputted records */ + sleep(2); + TEST_CHECK(expected.actual_records == expected.expected_records); + filter_test_destroy(ctx); +} + +static void flb_test_ecs_filter_task_error() +{ + int len; + int ret; + int bytes; + char *p; + struct flb_lib_out_cb cb_data; + struct filter_test *ctx; + struct filter_test_result expected = { 0 }; + + /* mocks calls- signals that we are in test mode */ + setenv("FLB_ECS_PLUGIN_UNDER_TEST", "true", 1); + setenv("TEST_TASK_ERROR", ERROR_RESPONSE, 1); + + /* Create test context */ + ctx = filter_test_create((void *) &cb_data, "79c796ed2a7f"); + if (!ctx) { + exit(EXIT_FAILURE); + } + + /* Configure filter */ + ret = flb_filter_set(ctx->flb, ctx->f_ffd, + "ecs_tag_prefix", "", + "ADD", "resource $ClusterName.$TaskID.$ECSContainerName", + NULL); + TEST_CHECK(ret == 0); + + /* this test is mainly for leak checking on error, not for checking result record */ + expected.expected_records = 1; /* 1 record with no metadata */ + expected.expected_pattern = ""; + expected.expected_pattern_index = 0; + cb_data.cb = cb_check_result; + cb_data.data = (void *) &expected; + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + /* Ingest data samples */ + p = "[0, {\"log\":\"error: my error\"}]"; + len = strlen(p); + bytes = flb_lib_push(ctx->flb, ctx->i_ffd, p, len); + TEST_CHECK(bytes == len); + + /* check number of outputted records */ + sleep(2); + TEST_CHECK(expected.actual_records == expected.expected_records); + filter_test_destroy(ctx); +} + +TEST_LIST = { + + {"flb_test_ecs_filter_mark_tag_failed" , flb_test_ecs_filter_mark_tag_failed }, + {"flb_test_ecs_filter" , flb_test_ecs_filter }, + {"flb_test_ecs_filter_no_prefix" , flb_test_ecs_filter_no_prefix }, + {"flb_test_ecs_filter_cluster_metadata_only" , flb_test_ecs_filter_cluster_metadata_only }, + {"flb_test_ecs_filter_cluster_error" , flb_test_ecs_filter_cluster_error }, + {"flb_test_ecs_filter_task_error" , flb_test_ecs_filter_task_error }, + + {NULL, NULL} +}; diff --git a/src/fluent-bit/tests/runtime/filter_expect.c b/src/fluent-bit/tests/runtime/filter_expect.c new file mode 100644 index 000000000..dad1b6375 --- /dev/null +++ b/src/fluent-bit/tests/runtime/filter_expect.c @@ -0,0 +1,555 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +#include <fluent-bit.h> +#include <fluent-bit/flb_time.h> +#include <fluent-bit/flb_sds.h> +#include "flb_tests_runtime.h" + +struct test_ctx { + flb_ctx_t *flb; /* Fluent Bit library context */ + int i_ffd; /* Input fd */ + int f_ffd; /* Filter fd */ + int o_ffd; /* Output fd */ +}; + +pthread_mutex_t result_mutex = PTHREAD_MUTEX_INITIALIZER; +int num_output = 0; +static int get_output_num() +{ + int ret; + pthread_mutex_lock(&result_mutex); + ret = num_output; + pthread_mutex_unlock(&result_mutex); + + return ret; +} + +static void set_output_num(int num) +{ + pthread_mutex_lock(&result_mutex); + num_output = num; + pthread_mutex_unlock(&result_mutex); +} + +static void clear_output_num() +{ + set_output_num(0); +} + +/* Callback to check expected results */ +static int cb_check_result_json(void *record, size_t size, void *data) +{ + char *p; + char *expected; + char *result; + int num = get_output_num(); + + set_output_num(num+1); + + expected = (char *) data; + result = (char *) record; + + p = strstr(result, expected); + if (!TEST_CHECK(p != NULL)) { + TEST_MSG("Expected to find: '%s' in result '%s'", + expected, result); + } + /* + * If you want to debug your test + * + * printf("Expect: '%s' in result '%s'", expected, result); + */ + flb_free(record); + return 0; +} + + +static struct test_ctx *test_ctx_create(struct flb_lib_out_cb *data) +{ + int i_ffd; + int f_ffd; + int o_ffd; + struct test_ctx *ctx; + + ctx = flb_malloc(sizeof(struct test_ctx)); + if (!ctx) { + flb_errno(); + return NULL; + } + + /* Service config */ + ctx->flb = flb_create(); + flb_service_set(ctx->flb, + "Flush", "0.200000000", + "Grace", "1", + "Log_Level", "Error", + NULL); + + /* Input */ + i_ffd = flb_input(ctx->flb, (char *) "lib", NULL); + TEST_CHECK(i_ffd >= 0); + flb_input_set(ctx->flb, i_ffd, "tag", "test", NULL); + ctx->i_ffd = i_ffd; + + /* Filter configuration */ + f_ffd = flb_filter(ctx->flb, (char *) "expect", NULL); + TEST_CHECK(f_ffd >= 0); + flb_filter_set(ctx->flb, f_ffd, + "match", "*", "action", "result_key", + "result_key", "result", + NULL); + ctx->f_ffd = f_ffd; + + /* Output */ + o_ffd = flb_output(ctx->flb, (char *) "lib", (void *) data); + TEST_CHECK(o_ffd >= 0); + flb_output_set(ctx->flb, o_ffd, + "match", "test", + "format", "json", + NULL); + ctx->o_ffd = o_ffd; + + return ctx; +} + +static void test_ctx_destroy(struct test_ctx *ctx) +{ + flb_stop(ctx->flb); + flb_destroy(ctx->flb); + flb_free(ctx); +} + +void flb_test_key_exists_matched() +{ + struct flb_lib_out_cb cb_data; + struct test_ctx *ctx; + int len; + int ret; + int bytes; + char *input = "[0, {\"key\":\"val\"}]"; + + clear_output_num(); + cb_data.cb = cb_check_result_json; + cb_data.data = "\"result\":true"; + + ctx = test_ctx_create(&cb_data); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("test_ctx_create failed"); + exit(EXIT_FAILURE); + } + + ret = flb_filter_set(ctx->flb, ctx->f_ffd, + "key_exists", "key", + NULL); + TEST_CHECK(ret == 0); + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + /* Ingest data samples */ + len = strlen(input); + bytes = flb_lib_push(ctx->flb, ctx->i_ffd, input, len); + TEST_CHECK(bytes == len); + flb_time_msleep(500); + + ret = get_output_num(); + if (!TEST_CHECK(ret > 0)) { + TEST_MSG("no output"); + } + + test_ctx_destroy(ctx); +} + +void flb_test_key_exists_not_matched() +{ + struct flb_lib_out_cb cb_data; + struct test_ctx *ctx; + int len; + int ret; + int bytes; + char *input = "[0, {\"key\":\"val\"}]"; + + clear_output_num(); + cb_data.cb = cb_check_result_json; + cb_data.data = "\"result\":false"; + + ctx = test_ctx_create(&cb_data); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("test_ctx_create failed"); + exit(EXIT_FAILURE); + } + + ret = flb_filter_set(ctx->flb, ctx->f_ffd, + "key_exists", "not_key", + NULL); + TEST_CHECK(ret == 0); + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + /* Ingest data samples */ + len = strlen(input); + bytes = flb_lib_push(ctx->flb, ctx->i_ffd, input, len); + TEST_CHECK(bytes == len); + flb_time_msleep(500); + + ret = get_output_num(); + if (!TEST_CHECK(ret > 0)) { + TEST_MSG("no output"); + } + + test_ctx_destroy(ctx); +} + +void flb_test_key_not_exists_matched() +{ + struct flb_lib_out_cb cb_data; + struct test_ctx *ctx; + int len; + int ret; + int bytes; + char *input = "[0, {\"key\":\"val\"}]"; + + clear_output_num(); + cb_data.cb = cb_check_result_json; + cb_data.data = "\"result\":true"; + + ctx = test_ctx_create(&cb_data); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("test_ctx_create failed"); + exit(EXIT_FAILURE); + } + + ret = flb_filter_set(ctx->flb, ctx->f_ffd, + "key_not_exists", "not_key", + NULL); + TEST_CHECK(ret == 0); + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + /* Ingest data samples */ + len = strlen(input); + bytes = flb_lib_push(ctx->flb, ctx->i_ffd, input, len); + TEST_CHECK(bytes == len); + flb_time_msleep(500); + + ret = get_output_num(); + if (!TEST_CHECK(ret > 0)) { + TEST_MSG("no output"); + } + + test_ctx_destroy(ctx); +} + +void flb_test_key_not_exists_not_matched() +{ + struct flb_lib_out_cb cb_data; + struct test_ctx *ctx; + int len; + int ret; + int bytes; + char *input = "[0, {\"key\":\"val\"}]"; + + clear_output_num(); + cb_data.cb = cb_check_result_json; + cb_data.data = "\"result\":false"; + + ctx = test_ctx_create(&cb_data); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("test_ctx_create failed"); + exit(EXIT_FAILURE); + } + + ret = flb_filter_set(ctx->flb, ctx->f_ffd, + "key_not_exists", "key", + NULL); + TEST_CHECK(ret == 0); + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + /* Ingest data samples */ + len = strlen(input); + bytes = flb_lib_push(ctx->flb, ctx->i_ffd, input, len); + TEST_CHECK(bytes == len); + flb_time_msleep(500); + + ret = get_output_num(); + if (!TEST_CHECK(ret > 0)) { + TEST_MSG("no output"); + } + + test_ctx_destroy(ctx); +} + +void flb_test_key_val_is_null_matched() +{ + struct flb_lib_out_cb cb_data; + struct test_ctx *ctx; + int len; + int ret; + int bytes; + char *input = "[0, {\"key\":null}]"; + + clear_output_num(); + cb_data.cb = cb_check_result_json; + cb_data.data = "\"result\":true"; + + ctx = test_ctx_create(&cb_data); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("test_ctx_create failed"); + exit(EXIT_FAILURE); + } + + ret = flb_filter_set(ctx->flb, ctx->f_ffd, + "key_val_is_null", "key", + NULL); + TEST_CHECK(ret == 0); + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + /* Ingest data samples */ + len = strlen(input); + bytes = flb_lib_push(ctx->flb, ctx->i_ffd, input, len); + TEST_CHECK(bytes == len); + flb_time_msleep(500); + + ret = get_output_num(); + if (!TEST_CHECK(ret > 0)) { + TEST_MSG("no output"); + } + + test_ctx_destroy(ctx); +} + +void flb_test_key_val_is_null_not_matched() +{ + struct flb_lib_out_cb cb_data; + struct test_ctx *ctx; + int len; + int ret; + int bytes; + char *input = "[0, {\"key\":\"val\"}]"; + + clear_output_num(); + cb_data.cb = cb_check_result_json; + cb_data.data = "\"result\":false"; + + ctx = test_ctx_create(&cb_data); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("test_ctx_create failed"); + exit(EXIT_FAILURE); + } + + ret = flb_filter_set(ctx->flb, ctx->f_ffd, + "key_val_is_null", "key", + NULL); + TEST_CHECK(ret == 0); + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + /* Ingest data samples */ + len = strlen(input); + bytes = flb_lib_push(ctx->flb, ctx->i_ffd, input, len); + TEST_CHECK(bytes == len); + flb_time_msleep(500); + + ret = get_output_num(); + if (!TEST_CHECK(ret > 0)) { + TEST_MSG("no output"); + } + + test_ctx_destroy(ctx); +} + +void flb_test_key_val_is_not_null_matched() +{ + struct flb_lib_out_cb cb_data; + struct test_ctx *ctx; + int len; + int ret; + int bytes; + char *input = "[0, {\"key\":\"val\"}]"; + + clear_output_num(); + cb_data.cb = cb_check_result_json; + cb_data.data = "\"result\":true"; + + ctx = test_ctx_create(&cb_data); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("test_ctx_create failed"); + exit(EXIT_FAILURE); + } + + ret = flb_filter_set(ctx->flb, ctx->f_ffd, + "key_val_is_not_null", "key", + NULL); + TEST_CHECK(ret == 0); + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + /* Ingest data samples */ + len = strlen(input); + bytes = flb_lib_push(ctx->flb, ctx->i_ffd, input, len); + TEST_CHECK(bytes == len); + flb_time_msleep(500); + + ret = get_output_num(); + if (!TEST_CHECK(ret > 0)) { + TEST_MSG("no output"); + } + + test_ctx_destroy(ctx); +} + +void flb_test_key_val_is_not_null_not_matched() +{ + struct flb_lib_out_cb cb_data; + struct test_ctx *ctx; + int len; + int ret; + int bytes; + char *input = "[0, {\"key\":null}]"; + + clear_output_num(); + cb_data.cb = cb_check_result_json; + cb_data.data = "\"result\":false"; + + ctx = test_ctx_create(&cb_data); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("test_ctx_create failed"); + exit(EXIT_FAILURE); + } + + ret = flb_filter_set(ctx->flb, ctx->f_ffd, + "key_val_is_not_null", "key", + NULL); + TEST_CHECK(ret == 0); + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + /* Ingest data samples */ + len = strlen(input); + bytes = flb_lib_push(ctx->flb, ctx->i_ffd, input, len); + TEST_CHECK(bytes == len); + flb_time_msleep(500); + + ret = get_output_num(); + if (!TEST_CHECK(ret > 0)) { + TEST_MSG("no output"); + } + + test_ctx_destroy(ctx); +} + +void flb_test_key_val_eq_matched() +{ + struct flb_lib_out_cb cb_data; + struct test_ctx *ctx; + int len; + int ret; + int bytes; + char *input = "[0, {\"key\":\"val\"}]"; + + clear_output_num(); + cb_data.cb = cb_check_result_json; + cb_data.data = "\"result\":true"; + + ctx = test_ctx_create(&cb_data); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("test_ctx_create failed"); + exit(EXIT_FAILURE); + } + + ret = flb_filter_set(ctx->flb, ctx->f_ffd, + "key_val_eq", "key val", + NULL); + TEST_CHECK(ret == 0); + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + /* Ingest data samples */ + len = strlen(input); + bytes = flb_lib_push(ctx->flb, ctx->i_ffd, input, len); + TEST_CHECK(bytes == len); + flb_time_msleep(500); + + ret = get_output_num(); + if (!TEST_CHECK(ret > 0)) { + TEST_MSG("no output"); + } + + test_ctx_destroy(ctx); +} + +void flb_test_key_val_eq_not_matched() +{ + struct flb_lib_out_cb cb_data; + struct test_ctx *ctx; + int len; + int ret; + int bytes; + char *input = "[0, {\"key\":\"val\"}]"; + + clear_output_num(); + cb_data.cb = cb_check_result_json; + cb_data.data = "\"result\":false"; + + ctx = test_ctx_create(&cb_data); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("test_ctx_create failed"); + exit(EXIT_FAILURE); + } + + ret = flb_filter_set(ctx->flb, ctx->f_ffd, + "key_val_eq", "not_key val", + NULL); + TEST_CHECK(ret == 0); + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + /* Ingest data samples */ + len = strlen(input); + bytes = flb_lib_push(ctx->flb, ctx->i_ffd, input, len); + TEST_CHECK(bytes == len); + flb_time_msleep(500); + + ret = get_output_num(); + if (!TEST_CHECK(ret > 0)) { + TEST_MSG("no output"); + } + + test_ctx_destroy(ctx); +} + + +TEST_LIST = { + {"key_exists_matched", flb_test_key_exists_matched}, + {"key_exists_not_matched", flb_test_key_exists_not_matched}, + {"key_not_exists_matched", flb_test_key_not_exists_matched}, + {"key_not_exists_not_matched", flb_test_key_not_exists_not_matched}, + {"key_val_is_null_matched", flb_test_key_val_is_null_matched}, + {"key_val_is_null_not_matched", flb_test_key_val_is_null_not_matched}, + {"key_val_is_not_null_matched", flb_test_key_val_is_not_null_matched}, + {"key_val_is_not_null_not_matched", flb_test_key_val_is_not_null_not_matched}, + {"key_val_eq_matched", flb_test_key_val_eq_matched}, + {"key_val_eq_not_matched", flb_test_key_val_eq_not_matched}, + {NULL, NULL} +}; diff --git a/src/fluent-bit/tests/runtime/filter_grep.c b/src/fluent-bit/tests/runtime/filter_grep.c new file mode 100644 index 000000000..6ae5721fc --- /dev/null +++ b/src/fluent-bit/tests/runtime/filter_grep.c @@ -0,0 +1,840 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +#include <fluent-bit.h> +#include <fluent-bit/flb_time.h> +#include "flb_tests_runtime.h" + +/* Test data */ + +/* Test functions */ +void flb_test_filter_grep_regex(void); +void flb_test_filter_grep_exclude(void); +void flb_test_filter_grep_invalid(void); + +pthread_mutex_t result_mutex = PTHREAD_MUTEX_INITIALIZER; +int num_output = 0; + +static int cb_count_msgpack(void *record, size_t size, void *data) +{ + msgpack_unpacked result; + size_t off = 0; + + if (!TEST_CHECK(data != NULL)) { + flb_error("data is NULL"); + } + + /* Iterate each item array and apply rules */ + msgpack_unpacked_init(&result); + while (msgpack_unpack_next(&result, record, size, &off) == MSGPACK_UNPACK_SUCCESS) { + pthread_mutex_lock(&result_mutex); + num_output++; + pthread_mutex_unlock(&result_mutex); + } + msgpack_unpacked_destroy(&result); + + flb_free(record); + return 0; +} + +static void clear_output_num() +{ + pthread_mutex_lock(&result_mutex); + num_output = 0; + pthread_mutex_unlock(&result_mutex); +} + +static int get_output_num() +{ + int ret; + pthread_mutex_lock(&result_mutex); + ret = num_output; + pthread_mutex_unlock(&result_mutex); + + return ret; +} + +void flb_test_filter_grep_regex(void) +{ + int i; + int ret; + int bytes; + char p[100]; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + int filter_ffd; + + ctx = flb_create(); + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + out_ffd = flb_output(ctx, (char *) "stdout", NULL); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, "match", "test", NULL); + + filter_ffd = flb_filter(ctx, (char *) "grep", NULL); + TEST_CHECK(filter_ffd >= 0); + ret = flb_filter_set(ctx, filter_ffd, "match", "*", NULL); + TEST_CHECK(ret == 0); + ret = flb_filter_set(ctx, filter_ffd, "Regex", "val 1", NULL); + TEST_CHECK(ret == 0); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + for (i = 0; i < 256; i++) { + memset(p, '\0', sizeof(p)); + snprintf(p, sizeof(p), "[%d, {\"val\": \"%d\",\"END_KEY\": \"JSON_END\"}]", i, (i * i)); + bytes = flb_lib_push(ctx, in_ffd, p, strlen(p)); + TEST_CHECK(bytes == strlen(p)); + } + + flb_time_msleep(1500); /* waiting flush */ + + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_filter_grep_exclude(void) +{ + int i; + int ret; + int bytes; + char p[100]; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + int filter_ffd; + + ctx = flb_create(); + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + out_ffd = flb_output(ctx, (char *) "stdout", NULL); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, "match", "test", NULL); + + filter_ffd = flb_filter(ctx, (char *) "grep", NULL); + TEST_CHECK(filter_ffd >= 0); + ret = flb_filter_set(ctx, filter_ffd, "match", "*", NULL); + TEST_CHECK(ret == 0); + ret = flb_filter_set(ctx, filter_ffd, "Exclude", "val 1", NULL); + TEST_CHECK(ret == 0); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + for (i = 0; i < 256; i++) { + memset(p, '\0', sizeof(p)); + snprintf(p, sizeof(p), "[%d, {\"val\": \"%d\",\"END_KEY\": \"JSON_END\"}]", i, (i * i)); + bytes = flb_lib_push(ctx, in_ffd, p, strlen(p)); + TEST_CHECK(bytes == strlen(p)); + } + + flb_time_msleep(1500); /* waiting flush */ + + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_filter_grep_invalid(void) +{ + int i; + int ret; + int bytes; + char p[100]; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + int filter_ffd; + + ctx = flb_create(); + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + out_ffd = flb_output(ctx, (char *) "stdout", NULL); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, "match", "test", NULL); + + filter_ffd = flb_filter(ctx, (char *) "grep", NULL); + TEST_CHECK(filter_ffd >= 0); + ret = flb_filter_set(ctx, filter_ffd, "match", "*", NULL); + TEST_CHECK(ret == 0); + ret = flb_filter_set(ctx, filter_ffd, "Regex", "val", NULL); + TEST_CHECK(ret == 0); + ret = flb_filter_set(ctx, filter_ffd, "Exclude", "val", NULL); + TEST_CHECK(ret == 0); + + ret = flb_start(ctx); + TEST_CHECK(ret == -1); + + for (i = 0; i < 256; i++) { + memset(p, '\0', sizeof(p)); + snprintf(p, sizeof(p), "[%d, {\"val\": \"%d\",\"END_KEY\": \"JSON_END\"}]", i, (i * i)); + bytes = flb_lib_push(ctx, in_ffd, p, strlen(p)); + TEST_CHECK(bytes == -1); + } + + flb_stop(ctx); + flb_destroy(ctx); +} + +/* filter_grep supports multiple 'Exclude's. + * If user sets multiple 'Exclude's, fluent-bit uses as OR conditions. + */ +void flb_test_filter_grep_multi_exclude(void) +{ + int i; + int ret; + int bytes; + char p[512]; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + int filter_ffd; + int got; + int n_loop = 256; + int not_used = 0; + struct flb_lib_out_cb cb_data; + + /* Prepare output callback with expected result */ + cb_data.cb = cb_count_msgpack; + cb_data.data = ¬_used; + + ctx = flb_create(); + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + out_ffd = flb_output(ctx, (char *) "lib", &cb_data); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, "match", "test", NULL); + + filter_ffd = flb_filter(ctx, (char *) "grep", NULL); + TEST_CHECK(filter_ffd >= 0); + ret = flb_filter_set(ctx, filter_ffd, "match", "*", NULL); + TEST_CHECK(ret == 0); + ret = flb_filter_set(ctx, filter_ffd, + "Exclude", "log deprecated", + "Exclude", "log hoge", + NULL); + TEST_CHECK(ret == 0); + + clear_output_num(); + + ret = flb_start(ctx); + if(!TEST_CHECK(ret == 0)) { + TEST_MSG("flb_start failed"); + exit(EXIT_FAILURE); + } + + /* Ingest 2 records per loop. One of them should be excluded. */ + for (i = 0; i < n_loop; i++) { + memset(p, '\0', sizeof(p)); + snprintf(p, sizeof(p), "[%d, {\"val\": \"%d\",\"log\": \"Using deprecated option\"}]", i, (i * i)); + bytes = flb_lib_push(ctx, in_ffd, p, strlen(p)); + TEST_CHECK(bytes == strlen(p)); + + /* Below record will be included */ + memset(p, '\0', sizeof(p)); + snprintf(p, sizeof(p), "[%d, {\"val\": \"%d\",\"log\": \"Using option\"}]", i, (i * i)); + bytes = flb_lib_push(ctx, in_ffd, p, strlen(p)); + TEST_CHECK(bytes == strlen(p)); + } + + flb_time_msleep(1500); /* waiting flush */ + + got = get_output_num(); + if (!TEST_CHECK(got == n_loop)) { + TEST_MSG("expect: %d got: %d", n_loop, got); + } + + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_filter_grep_unknown_property(void) +{ + int ret; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + int filter_ffd; + int not_used = 0; + struct flb_lib_out_cb cb_data; + + /* Prepare output callback with expected result */ + cb_data.cb = cb_count_msgpack; + cb_data.data = ¬_used; + + ctx = flb_create(); + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + out_ffd = flb_output(ctx, (char *) "lib", &cb_data); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, "match", "test", NULL); + + filter_ffd = flb_filter(ctx, (char *) "grep", NULL); + TEST_CHECK(filter_ffd >= 0); + ret = flb_filter_set(ctx, filter_ffd, "match", "*", NULL); + TEST_CHECK(ret == 0); + ret = flb_filter_set(ctx, filter_ffd, "UNKNOWN_PROPERTY", "aaaaaa", NULL); + TEST_CHECK(ret == 0); + + clear_output_num(); + + ret = flb_start(ctx); + if(!TEST_CHECK(ret != 0)) { + TEST_MSG("flb_start should be failed"); + exit(EXIT_FAILURE); + } + + flb_destroy(ctx); +} + +/* + * https://github.com/fluent/fluent-bit/issues/5209 + * To support /REGEX/ style. + */ +void flb_test_issue_5209(void) +{ + int i; + int ret; + int bytes; + char p[512]; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + int filter_ffd; + int got; + int n_loop = 256; + int not_used = 0; + struct flb_lib_out_cb cb_data; + + /* Prepare output callback with expected result */ + cb_data.cb = cb_count_msgpack; + cb_data.data = ¬_used; + + ctx = flb_create(); + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + out_ffd = flb_output(ctx, (char *) "lib", &cb_data); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, "match", "test", NULL); + + filter_ffd = flb_filter(ctx, (char *) "grep", NULL); + TEST_CHECK(filter_ffd >= 0); + ret = flb_filter_set(ctx, filter_ffd, "match", "*", NULL); + TEST_CHECK(ret == 0); + ret = flb_filter_set(ctx, filter_ffd, "Exclude", "log /Using deprecated option/", NULL); + TEST_CHECK(ret == 0); + + clear_output_num(); + + ret = flb_start(ctx); + if(!TEST_CHECK(ret == 0)) { + TEST_MSG("flb_start failed"); + exit(EXIT_FAILURE); + } + + /* Ingest 2 records per loop. One of them should be excluded. */ + for (i = 0; i < n_loop; i++) { + memset(p, '\0', sizeof(p)); + snprintf(p, sizeof(p), "[%d, {\"val\": \"%d\",\"END_KEY\": \"JSON_END\"}]", i, (i * i)); + bytes = flb_lib_push(ctx, in_ffd, p, strlen(p)); + TEST_CHECK(bytes == strlen(p)); + + /* Below record will be excluded */ + memset(p, '\0', sizeof(p)); + snprintf(p, sizeof(p), "[%d, {\"val\": \"%d\",\"log\": \"Using deprecated option\"}]", i, (i * i)); + bytes = flb_lib_push(ctx, in_ffd, p, strlen(p)); + TEST_CHECK(bytes == strlen(p)); + } + + flb_time_msleep(1500); /* waiting flush */ + + got = get_output_num(); + if (!TEST_CHECK(got == n_loop)) { + TEST_MSG("expect: %d got: %d", n_loop, got); + } + + flb_stop(ctx); + flb_destroy(ctx); +} + + +/* filter_grep supports multiple 'Regex's. + * If user sets multiple 'Regex's, fluent-bit uses as AND conditions. + */ +void flb_test_filter_grep_multi_regex(void) +{ + int i; + int ret; + int bytes; + char p[512]; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + int filter_ffd; + int got; + int n_loop = 256; + int not_used = 0; + struct flb_lib_out_cb cb_data; + + /* Prepare output callback with expected result */ + cb_data.cb = cb_count_msgpack; + cb_data.data = ¬_used; + + ctx = flb_create(); + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + out_ffd = flb_output(ctx, (char *) "lib", &cb_data); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, "match", "test", NULL); + + filter_ffd = flb_filter(ctx, (char *) "grep", NULL); + TEST_CHECK(filter_ffd >= 0); + ret = flb_filter_set(ctx, filter_ffd, "match", "*", NULL); + TEST_CHECK(ret == 0); + ret = flb_filter_set(ctx, filter_ffd, + "Regex", "log deprecated", + "Regex", "log option", + NULL); + TEST_CHECK(ret == 0); + + clear_output_num(); + + ret = flb_start(ctx); + if(!TEST_CHECK(ret == 0)) { + TEST_MSG("flb_start failed"); + exit(EXIT_FAILURE); + } + + /* Ingest 2 records per loop. One of them should be excluded. */ + for (i = 0; i < n_loop; i++) { + memset(p, '\0', sizeof(p)); + /* Below record will be included */ + snprintf(p, sizeof(p), "[%d, {\"val\": \"%d\",\"log\": \"Using deprecated option\"}]", i, (i * i)); + bytes = flb_lib_push(ctx, in_ffd, p, strlen(p)); + TEST_CHECK(bytes == strlen(p)); + + /* Below record will be excluded */ + memset(p, '\0', sizeof(p)); + snprintf(p, sizeof(p), "[%d, {\"val\": \"%d\",\"log\": \"Using option\"}]", i, (i * i)); + bytes = flb_lib_push(ctx, in_ffd, p, strlen(p)); + TEST_CHECK(bytes == strlen(p)); + } + + flb_time_msleep(1500); /* waiting flush */ + + got = get_output_num(); + if (!TEST_CHECK(got == n_loop)) { + TEST_MSG("expect: %d got: %d", n_loop, got); + } + + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_error_AND_regex_exclude(void) +{ + int ret; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + int filter_ffd; + + ctx = flb_create(); + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + out_ffd = flb_output(ctx, (char *) "stdout", NULL); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, "match", "test", NULL); + + filter_ffd = flb_filter(ctx, (char *) "grep", NULL); + TEST_CHECK(filter_ffd >= 0); + ret = flb_filter_set(ctx, filter_ffd, "match", "*", NULL); + TEST_CHECK(ret == 0); + ret = flb_filter_set(ctx, filter_ffd, + "Regex", "val 1", + "Exclude", "val2 3", + "Logical_Op", "AND", + NULL); + TEST_CHECK(ret == 0); + + ret = flb_start(ctx); + TEST_CHECK(ret != 0); + + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_error_OR_regex_exclude(void) +{ + int ret; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + int filter_ffd; + + ctx = flb_create(); + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + out_ffd = flb_output(ctx, (char *) "stdout", NULL); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, "match", "test", NULL); + + filter_ffd = flb_filter(ctx, (char *) "grep", NULL); + TEST_CHECK(filter_ffd >= 0); + ret = flb_filter_set(ctx, filter_ffd, "match", "*", NULL); + TEST_CHECK(ret == 0); + ret = flb_filter_set(ctx, filter_ffd, + "Regex", "val 1", + "Exclude", "val2 3", + "Logical_Op", "OR", + NULL); + TEST_CHECK(ret == 0); + + ret = flb_start(ctx); + TEST_CHECK(ret != 0); + + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_AND_regex(void) +{ + int i; + int ret; + int bytes; + char p[512]; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + int filter_ffd; + int got; + int n_loop = 256; + int not_used = 0; + struct flb_lib_out_cb cb_data; + + /* Prepare output callback with expected result */ + cb_data.cb = cb_count_msgpack; + cb_data.data = ¬_used; + + ctx = flb_create(); + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + out_ffd = flb_output(ctx, (char *) "lib", &cb_data); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, "match", "test", NULL); + + filter_ffd = flb_filter(ctx, (char *) "grep", NULL); + TEST_CHECK(filter_ffd >= 0); + ret = flb_filter_set(ctx, filter_ffd, "match", "*", NULL); + TEST_CHECK(ret == 0); + ret = flb_filter_set(ctx, filter_ffd, + "Regex", "log deprecated", + "Regex", "log option", + "Logical_Op", "AND", + NULL); + TEST_CHECK(ret == 0); + + clear_output_num(); + + ret = flb_start(ctx); + if(!TEST_CHECK(ret == 0)) { + TEST_MSG("flb_start failed"); + exit(EXIT_FAILURE); + } + + /* Ingest 2 records per loop. One of them should be excluded. */ + for (i = 0; i < n_loop; i++) { + memset(p, '\0', sizeof(p)); + /* Below record will be included */ + snprintf(p, sizeof(p), "[%d, {\"val\": \"%d\",\"log\": \"Using deprecated option\"}]", i, (i * i)); + bytes = flb_lib_push(ctx, in_ffd, p, strlen(p)); + TEST_CHECK(bytes == strlen(p)); + + /* Below record will be excluded */ + memset(p, '\0', sizeof(p)); + snprintf(p, sizeof(p), "[%d, {\"val\": \"%d\",\"log\": \"Using option\"}]", i, (i * i)); + bytes = flb_lib_push(ctx, in_ffd, p, strlen(p)); + TEST_CHECK(bytes == strlen(p)); + } + + flb_time_msleep(1500); /* waiting flush */ + + got = get_output_num(); + if (!TEST_CHECK(got == n_loop)) { + TEST_MSG("expect: %d got: %d", n_loop, got); + } + + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_OR_regex(void) +{ + int i; + int ret; + int bytes; + char p[512]; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + int filter_ffd; + int got; + int n_loop = 256; + int not_used = 0; + struct flb_lib_out_cb cb_data; + + /* Prepare output callback with expected result */ + cb_data.cb = cb_count_msgpack; + cb_data.data = ¬_used; + + ctx = flb_create(); + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + out_ffd = flb_output(ctx, (char *) "lib", &cb_data); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, "match", "test", NULL); + + filter_ffd = flb_filter(ctx, (char *) "grep", NULL); + TEST_CHECK(filter_ffd >= 0); + ret = flb_filter_set(ctx, filter_ffd, "match", "*", NULL); + TEST_CHECK(ret == 0); + ret = flb_filter_set(ctx, filter_ffd, + "Regex", "log deprecated", + "Regex", "log option", + "Logical_Op", "OR", + NULL); + TEST_CHECK(ret == 0); + + clear_output_num(); + + ret = flb_start(ctx); + if(!TEST_CHECK(ret == 0)) { + TEST_MSG("flb_start failed"); + exit(EXIT_FAILURE); + } + + /* Ingest 2 records per loop. One of them should be excluded. */ + for (i = 0; i < n_loop; i++) { + memset(p, '\0', sizeof(p)); + /* Below record will be included */ + snprintf(p, sizeof(p), "[%d, {\"val\": \"%d\",\"log\": \"Using deprecated option\"}]", i, (i * i)); + bytes = flb_lib_push(ctx, in_ffd, p, strlen(p)); + TEST_CHECK(bytes == strlen(p)); + + /* Below record will be excluded */ + memset(p, '\0', sizeof(p)); + snprintf(p, sizeof(p), "[%d, {\"val\": \"%d\",\"log\": \"Using option\"}]", i, (i * i)); + bytes = flb_lib_push(ctx, in_ffd, p, strlen(p)); + TEST_CHECK(bytes == strlen(p)); + } + + flb_time_msleep(1500); /* waiting flush */ + + got = get_output_num(); + if (!TEST_CHECK(got == n_loop * 2)) { + TEST_MSG("expect: %d got: %d", n_loop * 2, got); + } + + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_AND_exclude(void) +{ + int i; + int ret; + int bytes; + char p[512]; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + int filter_ffd; + int got; + int n_loop = 256; + int not_used = 0; + struct flb_lib_out_cb cb_data; + + /* Prepare output callback with expected result */ + cb_data.cb = cb_count_msgpack; + cb_data.data = ¬_used; + + ctx = flb_create(); + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + out_ffd = flb_output(ctx, (char *) "lib", &cb_data); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, "match", "test", NULL); + + filter_ffd = flb_filter(ctx, (char *) "grep", NULL); + TEST_CHECK(filter_ffd >= 0); + ret = flb_filter_set(ctx, filter_ffd, "match", "*", NULL); + TEST_CHECK(ret == 0); + ret = flb_filter_set(ctx, filter_ffd, + "Exclude", "log deprecated", + "Exclude", "log option", + "Logical_Op", "AND", + NULL); + TEST_CHECK(ret == 0); + + clear_output_num(); + + ret = flb_start(ctx); + if(!TEST_CHECK(ret == 0)) { + TEST_MSG("flb_start failed"); + exit(EXIT_FAILURE); + } + + /* Ingest 2 records per loop. One of them should be excluded. */ + for (i = 0; i < n_loop; i++) { + memset(p, '\0', sizeof(p)); + /* Below record will be included */ + snprintf(p, sizeof(p), "[%d, {\"val\": \"%d\",\"log\": \"Using deprecated option\"}]", i, (i * i)); + bytes = flb_lib_push(ctx, in_ffd, p, strlen(p)); + TEST_CHECK(bytes == strlen(p)); + + /* Below record will be excluded */ + memset(p, '\0', sizeof(p)); + snprintf(p, sizeof(p), "[%d, {\"val\": \"%d\",\"log\": \"Using option\"}]", i, (i * i)); + bytes = flb_lib_push(ctx, in_ffd, p, strlen(p)); + TEST_CHECK(bytes == strlen(p)); + } + + flb_time_msleep(1500); /* waiting flush */ + + got = get_output_num(); + if (!TEST_CHECK(got == n_loop)) { + TEST_MSG("expect: %d got: %d", n_loop, got); + } + + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_OR_exclude(void) +{ + int i; + int ret; + int bytes; + char p[512]; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + int filter_ffd; + int got; + int n_loop = 256; + int not_used = 0; + struct flb_lib_out_cb cb_data; + + /* Prepare output callback with expected result */ + cb_data.cb = cb_count_msgpack; + cb_data.data = ¬_used; + + ctx = flb_create(); + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + out_ffd = flb_output(ctx, (char *) "lib", &cb_data); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, "match", "test", NULL); + + filter_ffd = flb_filter(ctx, (char *) "grep", NULL); + TEST_CHECK(filter_ffd >= 0); + ret = flb_filter_set(ctx, filter_ffd, "match", "*", NULL); + TEST_CHECK(ret == 0); + ret = flb_filter_set(ctx, filter_ffd, + "Exclude", "log deprecated", + "Exclude", "log other", + "Logical_Op", "OR", + NULL); + TEST_CHECK(ret == 0); + + clear_output_num(); + + ret = flb_start(ctx); + if(!TEST_CHECK(ret == 0)) { + TEST_MSG("flb_start failed"); + exit(EXIT_FAILURE); + } + + /* Ingest 2 records per loop. One of them should be excluded. */ + for (i = 0; i < n_loop; i++) { + memset(p, '\0', sizeof(p)); + /* Below record will be included */ + snprintf(p, sizeof(p), "[%d, {\"val\": \"%d\",\"log\": \"Using deprecated option\"}]", i, (i * i)); + bytes = flb_lib_push(ctx, in_ffd, p, strlen(p)); + TEST_CHECK(bytes == strlen(p)); + + /* Below record will be excluded */ + memset(p, '\0', sizeof(p)); + snprintf(p, sizeof(p), "[%d, {\"val\": \"%d\",\"log\": \"Using option\"}]", i, (i * i)); + bytes = flb_lib_push(ctx, in_ffd, p, strlen(p)); + TEST_CHECK(bytes == strlen(p)); + } + + flb_time_msleep(1500); /* waiting flush */ + + got = get_output_num(); + if (!TEST_CHECK(got == n_loop)) { + TEST_MSG("expect: %d got: %d", n_loop, got); + } + + flb_stop(ctx); + flb_destroy(ctx); +} + +/* Test list */ +TEST_LIST = { + {"regex", flb_test_filter_grep_regex }, + {"exclude", flb_test_filter_grep_exclude }, + {"invalid", flb_test_filter_grep_invalid }, + {"multi_regex", flb_test_filter_grep_multi_regex }, + {"multi_exclude", flb_test_filter_grep_multi_exclude }, + {"unknown_property", flb_test_filter_grep_unknown_property }, + {"AND_regex", flb_test_AND_regex}, + {"OR_regex", flb_test_OR_regex}, + {"AND_exclude", flb_test_AND_exclude}, + {"OR_exclude", flb_test_OR_exclude}, + {"error_OR_regex_exclude", flb_test_error_OR_regex_exclude}, + {"error_AND_regex_exclude", flb_test_error_AND_regex_exclude}, + {"error_OR_regex_exclude", flb_test_error_OR_regex_exclude}, + {"issue_5209", flb_test_issue_5209 }, + {NULL, NULL} +}; diff --git a/src/fluent-bit/tests/runtime/filter_kubernetes.c b/src/fluent-bit/tests/runtime/filter_kubernetes.c new file mode 100644 index 000000000..46c9a966d --- /dev/null +++ b/src/fluent-bit/tests/runtime/filter_kubernetes.c @@ -0,0 +1,1086 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +#define _GNU_SOURCE /* for accept4 */ +#include <fluent-bit.h> +#include <fluent-bit/flb_time.h> +#include <fluent-bit/flb_info.h> +#include "flb_tests_runtime.h" + +#include <sys/types.h> +#include <sys/stat.h> +#include <unistd.h> +#ifdef _WIN32 + #define TIME_EPSILON_MS 30 +#else + #define TIME_EPSILON_MS 10 +#endif + +struct kube_test { + flb_ctx_t *flb; +}; + +struct kube_test_result { + const char *target; + const char *suffix; + int type; + int nMatched; +}; + +void wait_with_timeout(uint32_t timeout_ms, struct kube_test_result *result, int nExpected) +{ + struct flb_time start_time; + struct flb_time end_time; + struct flb_time diff_time; + uint64_t elapsed_time_flb = 0; + + flb_time_get(&start_time); + + while (true) { + if (result->nMatched == nExpected) { + break; + } + + flb_time_msleep(100); + flb_time_get(&end_time); + flb_time_diff(&end_time, &start_time, &diff_time); + elapsed_time_flb = flb_time_to_nanosec(&diff_time) / 1000000; + + if (elapsed_time_flb > timeout_ms - TIME_EPSILON_MS) { + flb_warn("[timeout] elapsed_time: %ld", elapsed_time_flb); + // Reached timeout. + break; + } + } +} + +/* Test target mode */ +#define KUBE_TAIL 0 +#define KUBE_SYSTEMD 1 + +#ifdef FLB_HAVE_SYSTEMD +int flb_test_systemd_send(void); +char kube_test_id[64]; +#endif + +/* Constants */ +#define KUBE_IP "127.0.0.1" +#define KUBE_PORT "8002" +#define KUBE_URL "http://" KUBE_IP ":" KUBE_PORT +#define DPATH FLB_TESTS_DATA_PATH "/data/kubernetes" + +static int file_to_buf(const char *path, char **out_buf, size_t *out_size) +{ + int ret; + long bytes; + char *buf; + FILE *fp; + struct stat st; + + ret = stat(path, &st); + if (ret == -1) { + return -1; + } + + fp = fopen(path, "r"); + if (!fp) { + return -1; + } + + buf = flb_malloc(st.st_size); + if (!buf) { + flb_errno(); + fclose(fp); + return -1; + } + + bytes = fread(buf, st.st_size, 1, fp); + if (bytes != 1) { + flb_errno(); + flb_free(buf); + fclose(fp); + return -1; + } + + fclose(fp); + *out_buf = buf; + *out_size = st.st_size; + + return 0; +} + +/* Given a target, lookup the .out file and return it content in a new buffer */ +static char *get_out_file_content(const char *target, const char *suffix) +{ + int ret; + char file[PATH_MAX]; + char *p; + char *out_buf; + size_t out_size; + + if (suffix) { + snprintf(file, sizeof(file) - 1, DPATH "/out/%s_%s.out", target, suffix); + } + else { + snprintf(file, sizeof(file) - 1, DPATH "/out/%s.out", target); + } + + ret = file_to_buf(file, &out_buf, &out_size); + TEST_CHECK_(ret == 0, "getting output file content: %s", file); + if (ret != 0) { + return NULL; + } + + /* Sanitize content, get rid of ending \n */ + p = out_buf + (out_size - 1); + while (*p == '\n' || *p == '\r') p--; + *++p = '\0'; + + return out_buf; +} + +static int cb_check_result(void *record, size_t size, void *data) +{ + struct kube_test_result *result; + char *out = NULL; + + result = (struct kube_test_result *) data; + + if (result->type == KUBE_SYSTEMD) { + char *skip_record, *skip_out; + int check; + + out = get_out_file_content(result->target, result->suffix); + if (!out) { + goto exit; + } + /* Skip the other records since some are created by systemd, + only check the kubernetes annotations + */ + skip_out = strstr(out, "\"kubernetes\":"); + skip_record = strstr(record, "\"kubernetes\":"); + if (skip_out && skip_record) { + check = strcmp(skip_record, skip_out); + TEST_CHECK(check == 0); + if (check != 0) { + printf("skip_record: %s\nskip_out: %s\n", + skip_record, skip_out); + } + result->nMatched++; + } + } else { + char *check; + char streamfilter[64] = {'\0'}; + + if (result->suffix && *result->suffix) { + sprintf(streamfilter, "\"stream\":\"%s\"", result->suffix); + } + if (!*streamfilter || + strstr(record, streamfilter)) { + out = get_out_file_content(result->target, result->suffix); + if (!out) { + goto exit; + } + /* + * Our validation is: check that the content of out file is found + * in the output record. + */ + check = strstr(record, out); + TEST_CHECK_(check != NULL, + "comparing expected record with actual record"); + if (check == NULL) { + if (result->suffix) { + printf("Target: %s, suffix: %s\n", + result->target, result->suffix); + } + else + { + printf("Target: %s\n", + result->target); + } + printf("Expected record:\n%s\n" + "Actual record:\n%s\n", + out, (char *)record); + } + result->nMatched++; + } + } + +exit: + if (size > 0) { + flb_free(record); + } + if (out) { + flb_free(out); + } + return 0; +} + +static void kube_test(const char *target, int type, const char *suffix, int nExpected, ...) +{ + int ret; + int in_ffd; + int filter_ffd; + int out_ffd; + char *key; + char *value; + char path[PATH_MAX]; + va_list va; + struct kube_test ctx; + struct flb_lib_out_cb cb_data; + struct kube_test_result result = {0}; + + result.nMatched = 0; + result.target = target; + result.suffix = suffix; + result.type = type; + + ctx.flb = flb_create(); + TEST_CHECK_(ctx.flb != NULL, "initialising service"); + if (!ctx.flb) { + goto exit; + } + + ret = flb_service_set(ctx.flb, + "Flush", "1", + "Grace", "1", + "Log_Level", "error", + "Parsers_File", DPATH "/parsers.conf", + NULL); + TEST_CHECK_(ret == 0, "setting service options"); + + if (type == KUBE_TAIL) { + /* Compose path based on target */ + snprintf(path, sizeof(path) - 1, DPATH "/log/%s.log", target); + TEST_CHECK_(access(path, R_OK) == 0, "accessing log file: %s", path); + in_ffd = flb_input(ctx.flb, "tail", NULL); + TEST_CHECK_(in_ffd >= 0, "initialising input"); + ret = flb_input_set(ctx.flb, in_ffd, + "Tag", "kube.<namespace>.<pod>.<container>", + "Tag_Regex", "^" DPATH "/log/(?:[^/]+/)?(?<namespace>.+)_(?<pod>.+)_(?<container>.+)\\.log$", + "Path", path, + "Parser", "docker", + "Docker_Mode", "On", + "read_from_head", "on", + NULL); + TEST_CHECK_(ret == 0, "setting input options"); + } +#ifdef FLB_HAVE_SYSTEMD + else if (type == KUBE_SYSTEMD) { + sprintf(kube_test_id, "KUBE_TEST=%u%lu", getpid(), random()); + in_ffd = flb_input(ctx.flb, "systemd", NULL); + TEST_CHECK_(in_ffd >= 0, "initialising input"); + ret = flb_input_set(ctx.flb, in_ffd, + "Tag", "kube.*", + "Systemd_Filter", kube_test_id, + NULL); + TEST_CHECK_(ret == 0, "setting input options"); + } +#endif + + filter_ffd = flb_filter(ctx.flb, "kubernetes", NULL); + TEST_CHECK_(filter_ffd >= 0, "initialising filter"); + ret = flb_filter_set(ctx.flb, filter_ffd, + "Match", "kube.*", + "Kube_Url", KUBE_URL, + "Kube_Meta_Preload_Cache_Dir", DPATH "/meta", + NULL); + TEST_CHECK_(ret == 0, "setting filter options"); + + /* Iterate number of arguments for filter_kubernetes additional options */ + va_start(va, nExpected); + while ((key = va_arg(va, char *))) { + value = va_arg(va, char *); + if (!value) { + /* Wrong parameter */ + break; + } + ret = flb_filter_set(ctx.flb, filter_ffd, key, value, NULL); + TEST_CHECK_(ret == 0, "setting filter additional options"); + } + va_end(va); + + if (type == KUBE_TAIL) { + ret = flb_filter_set(ctx.flb, filter_ffd, + "Regex_Parser", "kubernetes-tag", + "Kube_Tag_Prefix", "kube.", + NULL); + TEST_CHECK_(ret == 0, "setting filter specific options"); + } +#ifdef FLB_HAVE_SYSTEMD + else if (type == KUBE_SYSTEMD) { + ret = flb_filter_set(ctx.flb, filter_ffd, + "Use_Journal", "On", + NULL); + TEST_CHECK_(ret == 0, "setting filter specific options"); + } +#endif + + /* Prepare output callback context*/ + cb_data.cb = cb_check_result; + cb_data.data = &result; + + /* Output */ + out_ffd = flb_output(ctx.flb, "lib", (void *) &cb_data); + TEST_CHECK_(out_ffd >= 0, "initialising output"); + flb_output_set(ctx.flb, out_ffd, + "Match", "kube.*", + "format", "json", + NULL); + TEST_CHECK_(ret == 0, "setting output options"); + +#ifdef FLB_HAVE_SYSTEMD + /* + * If the source of data is Systemd, just let the output lib plugin + * to process one record only, otherwise when the test case stop after + * the first callback it destroy the contexts, but out_lib still have + * pending data to flush. This option solves the problem. + */ + if (type == KUBE_SYSTEMD) { + flb_output_set(ctx.flb, out_ffd, + "Max_Records", "1", + NULL); + TEST_CHECK_(ret == 0, "setting output specific options"); + } +#endif + + /* Start the engine */ + ret = flb_start(ctx.flb); + TEST_CHECK_(ret == 0, "starting engine"); + if (ret == -1) { + goto exit; + } +#ifdef FLB_HAVE_SYSTEMD + if (type == KUBE_SYSTEMD) { + TEST_CHECK_(flb_test_systemd_send() >= 0, + "sending sample message to journal"); + } +#endif + + /* Poll for up to 2 seconds or until we got a match */ + for (ret = 0; ret < 2000 && result.nMatched == 0; ret++) { + usleep(1000); + } + + /* Wait until matching nExpected results */ + wait_with_timeout(5000, &result, nExpected); + + TEST_CHECK(result.nMatched == nExpected); + TEST_MSG("result.nMatched: %i\nnExpected: %i", result.nMatched, nExpected); + + ret = flb_stop(ctx.flb); + TEST_CHECK_(ret == 0, "stopping engine"); + +exit: + if (ctx.flb) { + flb_destroy(ctx.flb); + } +} + + +#define flb_test_core(target, suffix, nExpected) \ + kube_test("core/" target, KUBE_TAIL, suffix, nExpected, NULL); + +static void flb_test_core_base() +{ + flb_test_core("core_base_fluent-bit", NULL, 1); +} + +static void flb_test_core_no_meta() +{ + flb_test_core("core_no-meta_text", NULL, 1); +} + +static void flb_test_core_unescaping_text() +{ + flb_test_core("core_unescaping_text", NULL, 1); +} + +static void flb_test_core_unescaping_json() +{ + flb_test_core("core_unescaping_json", NULL, 1); +} + + +#define flb_test_options_use_kubelet_enabled(target, suffix, nExpected) \ + kube_test("options/" target, KUBE_TAIL, suffix, nExpected, \ + "use_kubelet", "true", \ + "kubelet_port", "8002", \ + NULL); \ + +#define flb_test_options_use_kubelet_disabled(target, suffix, nExpected) \ + kube_test("options/" target, KUBE_TAIL, suffix, nExpected, \ + "use_kubelet", "false", \ + "kubelet_port", "8002", \ + NULL); \ + + +static void flb_test_options_use_kubelet_enabled_json() +{ + flb_test_options_use_kubelet_enabled("options_use-kubelet-enabled_fluent-bit", NULL, 1); +} + +static void flb_test_options_use_kubelet_disabled_json() +{ + flb_test_options_use_kubelet_disabled("options_use-kubelet-disabled_fluent-bit", NULL, 1); +} + +#define flb_test_options_merge_log_enabled(target, suffix, nExpected) \ + kube_test("options/" target, KUBE_TAIL, suffix, nExpected, \ + "Merge_Log", "On", \ + NULL); \ + +#define flb_test_options_merge_log_disabled(target, suffix, nExpected) \ + kube_test("options/" target, KUBE_TAIL, suffix, nExpected, \ + NULL); \ + +static void flb_test_options_merge_log_enabled_text() +{ + flb_test_options_merge_log_enabled("options_merge-log-enabled_text", NULL, 1); +} + +static void flb_test_options_merge_log_enabled_json() +{ + flb_test_options_merge_log_enabled("options_merge-log-enabled_json", NULL, 1); +} + +static void flb_test_options_merge_log_enabled_invalid_json() +{ + flb_test_options_merge_log_enabled("options_merge-log-enabled_invalid-json", NULL, 1); +} + +static void flb_test_options_merge_log_disabled_json() +{ + flb_test_options_merge_log_disabled("options_merge-log-disabled_json", NULL, 1); +} + +#define flb_test_options_merge_log_trim_enabled(target, suffix, nExpected) \ + kube_test("options/" target, KUBE_TAIL, suffix, nExpected, \ + "Merge_Log", "On", \ + NULL); \ + +#define flb_test_options_merge_log_trim_disabled(target, suffix, nExpected) \ + kube_test("options/" target, KUBE_TAIL, suffix, nExpected, \ + "Merge_Log", "On", \ + "Merge_Log_Trim", "Off", \ + NULL); \ + +static void flb_test_options_merge_log_trim_enabled_json() +{ + flb_test_options_merge_log_trim_enabled("options_merge-log-trim-enabled_json", NULL, 1); +} + +static void flb_test_options_merge_log_trim_disabled_json() +{ + flb_test_options_merge_log_trim_disabled("options_merge-log-trim-disabled_json", NULL, 1); +} + +#define flb_test_options_merge_log_key(target, suffix, nExpected) \ + kube_test("options/" target, KUBE_TAIL, suffix, nExpected, \ + "Merge_Log", "On", \ + "Merge_Log_Key", "merge-log-key", \ + NULL); \ + +static void flb_test_options_merge_log_key_json() +{ + flb_test_options_merge_log_key("options_merge-log-key_json", NULL, 1); +} + +#define flb_test_options_keep_log_enabled(target, suffix, nExpected) \ + kube_test("options/" target, KUBE_TAIL, suffix, nExpected, \ + "Merge_Log", "On", \ + NULL); \ + +#define flb_test_options_keep_log_disabled(target, suffix, nExpected) \ + kube_test("options/" target, KUBE_TAIL, suffix, nExpected, \ + "Merge_Log", "On", \ + "Keep_Log", "Off", \ + NULL); \ + +static void flb_test_options_keep_log_enabled_json() +{ + flb_test_options_keep_log_enabled("options_keep-log-enabled_json", NULL, 1); +} + +static void flb_test_options_keep_log_disabled_json() +{ + flb_test_options_keep_log_disabled("options_keep-log-disabled_json", NULL, 1); +} + +#define flb_test_options_k8s_logging_parser_disabled(target, suffix, nExpected) \ + kube_test("options/" target, KUBE_TAIL, suffix, nExpected, \ + "Merge_Log", "On", \ + NULL); \ + +static void flb_test_options_k8s_logging_parser_disabled_text_stdout() +{ + flb_test_options_k8s_logging_parser_disabled("options_k8s-logging-parser-disabled_text", "stdout", 1); +} + +static void flb_test_options_k8s_logging_parser_disabled_text_stderr() +{ + flb_test_options_k8s_logging_parser_disabled("options_k8s-logging-parser-disabled_text", "stderr", 1); +} + +#define flb_test_options_k8s_logging_exclude_disabled(target, suffix, nExpected) \ + kube_test("options/" target, KUBE_TAIL, suffix, nExpected, \ + "Merge_Log", "On", \ + NULL); \ + +static void flb_test_options_k8s_logging_exclude_disabled_text_stdout() +{ + flb_test_options_k8s_logging_exclude_disabled("options_k8s-logging-exclude-disabled_text", "stdout", 1); +} + +static void flb_test_options_k8s_logging_exclude_disabled_text_stderr() +{ + flb_test_options_k8s_logging_exclude_disabled("options_k8s-logging-exclude-disabled_text", "stderr", 1); +} + +#define flb_test_annotations(target, suffix, nExpected) \ + kube_test("annotations/" target, KUBE_TAIL, suffix, nExpected, \ + "K8s-Logging.Parser", "On", \ + "K8s-Logging.Exclude", "On", \ + NULL); \ + +static void flb_test_annotations_invalid_text() +{ + flb_test_annotations("annotations_invalid_text", NULL, 1); +} + +#define flb_test_annotations_parser(target, suffix, nExpected) \ + kube_test("annotations-parser/" target, KUBE_TAIL, suffix, nExpected, \ + "K8s-Logging.Parser", "On", \ + "Merge_Log", "On", \ + "Keep_Log", "Off", \ + NULL); \ + +static void flb_test_annotations_parser_regex_with_time_text() +{ + flb_test_annotations_parser("annotations-parser_regex-with-time_text", NULL, 1); +} + +static void flb_test_annotations_parser_regex_with_time_invalid_text_1() +{ + flb_test_annotations_parser("annotations-parser_regex-with-time_invalid-text-1", NULL, 1); +} + +static void flb_test_annotations_parser_json_with_time_json() +{ + flb_test_annotations_parser("annotations-parser_json-with-time_json", NULL, 1); +} + +static void flb_test_annotations_parser_json_with_time_invalid_json_1() +{ + flb_test_annotations_parser("annotations-parser_json-with-time_invalid-json-1", NULL, 1); +} + +static void flb_test_annotations_parser_invalid_text_stdout() +{ + flb_test_annotations_parser("annotations-parser_invalid_text", "stdout", 1); +} + +static void flb_test_annotations_parser_invalid_text_stderr() +{ + flb_test_annotations_parser("annotations-parser_invalid_text", "stderr", 1); +} + +static void flb_test_annotations_parser_stdout_text_stdout() +{ + flb_test_annotations_parser("annotations-parser_stdout_text", "stdout", 1); +} + +static void flb_test_annotations_parser_stdout_text_stderr() +{ + flb_test_annotations_parser("annotations-parser_stdout_text", "stderr", 1); +} + +static void flb_test_annotations_parser_stderr_text_stdout() +{ + flb_test_annotations_parser("annotations-parser_stderr_text", "stdout", 1); +} + +static void flb_test_annotations_parser_stderr_text_stderr() +{ + flb_test_annotations_parser("annotations-parser_stderr_text", "stderr", 1); +} + +static void flb_test_annotations_parser_multiple_1_container_1_stdout() +{ + flb_test_annotations_parser("annotations-parser_multiple-1_container-1", "stdout", 1); +} + +static void flb_test_annotations_parser_multiple_1_container_1_stderr() +{ + flb_test_annotations_parser("annotations-parser_multiple-1_container-1", "stderr", 1); +} + +static void flb_test_annotations_parser_multiple_1_container_2_stdout() +{ + flb_test_annotations_parser("annotations-parser_multiple-1_container-2", "stdout", 1); +} + +static void flb_test_annotations_parser_multiple_1_container_2_stderr() +{ + flb_test_annotations_parser("annotations-parser_multiple-1_container-2", "stderr", 1); +} + +static void flb_test_annotations_parser_multiple_1_container_3_stdout() +{ + flb_test_annotations_parser("annotations-parser_multiple-1_container-3", "stdout", 1); +} + +static void flb_test_annotations_parser_multiple_1_container_3_stderr() +{ + flb_test_annotations_parser("annotations-parser_multiple-1_container-3", "stderr", 1); +} + +static void flb_test_annotations_parser_multiple_1_container_4_stdout() +{ + flb_test_annotations_parser("annotations-parser_multiple-1_container-4", "stdout", 1); +} + +static void flb_test_annotations_parser_multiple_1_container_4_stderr() +{ + flb_test_annotations_parser("annotations-parser_multiple-1_container-4", "stderr", 1); +} + +static void flb_test_annotations_parser_multiple_1_container_5_stdout() +{ + flb_test_annotations_parser("annotations-parser_multiple-1_container-5", "stdout", 1); +} + +static void flb_test_annotations_parser_multiple_1_container_5_stderr() +{ + flb_test_annotations_parser("annotations-parser_multiple-1_container-5", "stderr", 1); +} + +static void flb_test_annotations_parser_multiple_2_container_1_stdout() +{ + flb_test_annotations_parser("annotations-parser_multiple-2_container-1", "stdout", 1); +} + +static void flb_test_annotations_parser_multiple_2_container_1_stderr() +{ + flb_test_annotations_parser("annotations-parser_multiple-2_container-1", "stderr", 1); +} + +static void flb_test_annotations_parser_multiple_2_container_2_stdout() +{ + flb_test_annotations_parser("annotations-parser_multiple-2_container-2", "stdout", 1); +} + +static void flb_test_annotations_parser_multiple_2_container_2_stderr() +{ + flb_test_annotations_parser("annotations-parser_multiple-2_container-2", "stderr", 1); +} + +static void flb_test_annotations_parser_multiple_2_container_3_stdout() +{ + flb_test_annotations_parser("annotations-parser_multiple-2_container-3", "stdout", 1); +} + +static void flb_test_annotations_parser_multiple_2_container_3_stderr() +{ + flb_test_annotations_parser("annotations-parser_multiple-2_container-3", "stderr", 1); +} + +static void flb_test_annotations_parser_multiple_2_container_4_stdout() +{ + flb_test_annotations_parser("annotations-parser_multiple-2_container-4", "stdout", 1); +} + +static void flb_test_annotations_parser_multiple_2_container_4_stderr() +{ + flb_test_annotations_parser("annotations-parser_multiple-2_container-4", "stderr", 1); +} + +static void flb_test_annotations_parser_multiple_2_container_5_stdout() +{ + flb_test_annotations_parser("annotations-parser_multiple-2_container-5", "stdout", 1); +} + +static void flb_test_annotations_parser_multiple_2_container_5_stderr() +{ + flb_test_annotations_parser("annotations-parser_multiple-2_container-5", "stderr", 1); +} + +#define flb_test_annotations_exclude(target, suffix, nExpected) \ + kube_test("annotations-exclude/" target, KUBE_TAIL, suffix, nExpected, \ + "K8s-Logging.Exclude", "On", \ + NULL); \ + +static void flb_test_annotations_exclude_default_text() +{ + flb_test_annotations_exclude("annotations-exclude_default_text", NULL, 0); +} + +static void flb_test_annotations_exclude_invalid_text_stdout() +{ + flb_test_annotations_exclude("annotations-exclude_invalid_text", "stdout", 1); +} + +static void flb_test_annotations_exclude_invalid_text_stderr() +{ + flb_test_annotations_exclude("annotations-exclude_invalid_text", "stderr", 1); +} + +static void flb_test_annotations_exclude_stdout_text_stdout() +{ + flb_test_annotations_exclude("annotations-exclude_stdout_text", "stdout", 0); +} + +static void flb_test_annotations_exclude_stdout_text_stderr() +{ + flb_test_annotations_exclude("annotations-exclude_stdout_text", "stderr", 1); +} + +static void flb_test_annotations_exclude_stderr_text_stdout() +{ + flb_test_annotations_exclude("annotations-exclude_stderr_text", "stdout", 1); +} + +static void flb_test_annotations_exclude_stderr_text_stderr() +{ + flb_test_annotations_exclude("annotations-exclude_stderr_text", "stderr", 0); +} + +static void flb_test_annotations_exclude_multiple_1_container_1_stdout() +{ + flb_test_annotations_exclude("annotations-exclude_multiple-1_container-1", "stdout", 0); +} + +static void flb_test_annotations_exclude_multiple_1_container_1_stderr() +{ + flb_test_annotations_exclude("annotations-exclude_multiple-1_container-1", "stderr", 0); +} + +static void flb_test_annotations_exclude_multiple_1_container_2_stdout() +{ + flb_test_annotations_exclude("annotations-exclude_multiple-1_container-2", "stdout", 0); +} + +static void flb_test_annotations_exclude_multiple_1_container_2_stderr() +{ + flb_test_annotations_exclude("annotations-exclude_multiple-1_container-2", "stderr", 1); +} + +static void flb_test_annotations_exclude_multiple_1_container_3_stdout() +{ + flb_test_annotations_exclude("annotations-exclude_multiple-1_container-3", "stdout", 1); +} + +static void flb_test_annotations_exclude_multiple_1_container_3_stderr() +{ + flb_test_annotations_exclude("annotations-exclude_multiple-1_container-3", "stderr", 0); +} + +static void flb_test_annotations_exclude_multiple_1_container_4_stdout() +{ + flb_test_annotations_exclude("annotations-exclude_multiple-1_container-4", "stdout", 1); +} + +static void flb_test_annotations_exclude_multiple_1_container_4_stderr() +{ + flb_test_annotations_exclude("annotations-exclude_multiple-1_container-4", "stderr", 1); +} + +static void flb_test_annotations_exclude_multiple_2_container_1_stdout() +{ + flb_test_annotations_exclude("annotations-exclude_multiple-2_container-1", "stdout", 0); +} + +static void flb_test_annotations_exclude_multiple_2_container_1_stderr() +{ + flb_test_annotations_exclude("annotations-exclude_multiple-2_container-1", "stderr", 0); +} + +static void flb_test_annotations_exclude_multiple_2_container_2_stdout() +{ + flb_test_annotations_exclude("annotations-exclude_multiple-2_container-2", "stdout", 0); +} + +static void flb_test_annotations_exclude_multiple_2_container_2_stderr() +{ + flb_test_annotations_exclude("annotations-exclude_multiple-2_container-2", "stderr", 1); +} + +static void flb_test_annotations_exclude_multiple_2_container_3_stdout() +{ + flb_test_annotations_exclude("annotations-exclude_multiple-2_container-3", "stdout", 1); +} + +static void flb_test_annotations_exclude_multiple_2_container_3_stderr() +{ + flb_test_annotations_exclude("annotations-exclude_multiple-2_container-3", "stderr", 0); +} + +static void flb_test_annotations_exclude_multiple_2_container_4_stdout() +{ + flb_test_annotations_exclude("annotations-exclude_multiple-2_container-4", "stdout", 1); +} + +static void flb_test_annotations_exclude_multiple_2_container_4_stderr() +{ + flb_test_annotations_exclude("annotations-exclude_multiple-2_container-4", "stderr", 1); +} + +static void flb_test_annotations_exclude_multiple_3_container_1_stdout() +{ + flb_test_annotations_exclude("annotations-exclude_multiple-3_container-1", "stdout", 0); +} + +static void flb_test_annotations_exclude_multiple_3_container_1_stderr() +{ + flb_test_annotations_exclude("annotations-exclude_multiple-3_container-1", "stderr", 0); +} + +static void flb_test_annotations_exclude_multiple_3_container_2_stdout() +{ + flb_test_annotations_exclude("annotations-exclude_multiple-3_container-2", "stdout", 0); +} + +static void flb_test_annotations_exclude_multiple_3_container_2_stderr() +{ + flb_test_annotations_exclude("annotations-exclude_multiple-3_container-2", "stderr", 1); +} + +static void flb_test_annotations_exclude_multiple_3_container_3_stdout() +{ + flb_test_annotations_exclude("annotations-exclude_multiple-3_container-3", "stdout", 1); +} + +static void flb_test_annotations_exclude_multiple_3_container_3_stderr() +{ + flb_test_annotations_exclude("annotations-exclude_multiple-3_container-3", "stderr", 0); +} + +static void flb_test_annotations_exclude_multiple_3_container_4_stdout() +{ + flb_test_annotations_exclude("annotations-exclude_multiple-3_container-4", "stdout", 1); +} + +static void flb_test_annotations_exclude_multiple_3_container_4_stderr() +{ + flb_test_annotations_exclude("annotations-exclude_multiple-3_container-4", "stderr", 1); +} + +static void flb_test_annotations_exclude_multiple_4_container_1_stdout() +{ + flb_test_annotations_exclude("annotations-exclude_multiple-4_container-1", "stdout", 0); +} + +static void flb_test_annotations_exclude_multiple_4_container_1_stderr() +{ + flb_test_annotations_exclude("annotations-exclude_multiple-4_container-1", "stderr", 0); +} + +static void flb_test_annotations_exclude_multiple_4_container_2_stdout() +{ + flb_test_annotations_exclude("annotations-exclude_multiple-4_container-2", "stdout", 0); +} + +static void flb_test_annotations_exclude_multiple_4_container_2_stderr() +{ + flb_test_annotations_exclude("annotations-exclude_multiple-4_container-2", "stderr", 1); +} + +static void flb_test_annotations_exclude_multiple_4_container_3_stdout() +{ + flb_test_annotations_exclude("annotations-exclude_multiple-4_container-3", "stdout", 1); +} + +static void flb_test_annotations_exclude_multiple_4_container_3_stderr() +{ + flb_test_annotations_exclude("annotations-exclude_multiple-4_container-3", "stderr", 0); +} + +static void flb_test_annotations_exclude_multiple_4_container_4_stdout() +{ + flb_test_annotations_exclude("annotations-exclude_multiple-4_container-4", "stdout", 1); +} + +static void flb_test_annotations_exclude_multiple_4_container_4_stderr() +{ + flb_test_annotations_exclude("annotations-exclude_multiple-4_container-4", "stderr", 1); +} + +#ifdef FLB_HAVE_SYSTEMD +#define CONTAINER_NAME "CONTAINER_NAME=k8s_kairosdb_kairosdb-914055854-b63vq_default_d6c53deb-05a4-11e8-a8c4-080027435fb7_23" +#include <systemd/sd-journal.h> + +int flb_test_systemd_send() +{ + return sd_journal_send( + "@timestamp=2018-02-23T08:58:45.0Z", + "PRIORITY=6", + CONTAINER_NAME, + "CONTAINER_TAG=", + "CONTAINER_ID=56e257661383", + "CONTAINER_ID_FULL=56e257661383836fac4cd90a23ee8a7a02ee1538c8f35657d1a90f3de1065a22", + "MESSAGE=08:58:45.839 [qtp151442075-47] DEBUG [HttpParser.java:281] - filled 157/157", + kube_test_id, + NULL); +} + +static void flb_test_systemd_logs() +{ + struct stat statb; + /* We want to avoid possibly getting a log from a previous run, + so create a unique-id for each 'send' */ + sprintf(kube_test_id, "KUBE_TEST=%u%lu", getpid(), random()); + + if (stat("/run/systemd/journal/socket", &statb) == 0 && + statb.st_mode & S_IFSOCK) { + + int r; + sd_journal *journal; + r = sd_journal_open(&journal, 0); + if (r < 0) { + flb_error("Skip test: journal error: %s", strerror(-r)); + return; + } + + r = sd_journal_get_fd(journal); + if (r < 0) { + flb_error("Skip test: journal fd error: %s", strerror(-r)); + sd_journal_close(journal); + return; + } + sd_journal_add_match(journal, CONTAINER_NAME, 0); + sd_journal_seek_tail(journal); + + /* + * Send test message to Journal. If this fails (e.g. journal is + * not running then skip the test. + */ + if (flb_test_systemd_send() < 0) { + + flb_error("Skip test: journal send error: %s", strerror(-r)); + sd_journal_close(journal); + return; + } + + r = sd_journal_previous(journal); + if (r < 0) { + flb_error("Skip test: journal previous error: %s", strerror(-r)); + sd_journal_close(journal); + return; + } + + r = sd_journal_next(journal); + if (r < 0) { + flb_error("Skip test: journal next error: %s", strerror(-r)); + sd_journal_close(journal); + return; + } + + r = sd_journal_wait(journal, 2000); + if (r < 0) { + flb_error("Skip test: journal wait error: %s", strerror(-r)); + sd_journal_close(journal); + return; + } + sd_journal_close(journal); + + kube_test("kairosdb-914055854-b63vq", KUBE_SYSTEMD, NULL, 1, + "Merge_Log", "On", + NULL); + } +} +#endif + +TEST_LIST = { + {"kube_core_base", flb_test_core_base}, + {"kube_core_no_meta", flb_test_core_no_meta}, + {"kube_core_unescaping_text", flb_test_core_unescaping_text}, + {"kube_core_unescaping_json", flb_test_core_unescaping_json}, + {"kube_options_use-kubelet_enabled_json", flb_test_options_use_kubelet_enabled_json}, + {"kube_options_use-kubelet_disabled_json", flb_test_options_use_kubelet_disabled_json}, + {"kube_options_merge_log_enabled_text", flb_test_options_merge_log_enabled_text}, + {"kube_options_merge_log_enabled_json", flb_test_options_merge_log_enabled_json}, + {"kube_options_merge_log_enabled_invalid_json", flb_test_options_merge_log_enabled_invalid_json}, + {"kube_options_merge_log_disabled_json", flb_test_options_merge_log_disabled_json}, + {"kube_options_merge_log_trim_enabled_json", flb_test_options_merge_log_trim_enabled_json}, + {"kube_options_merge_log_trim_disabled_json", flb_test_options_merge_log_trim_disabled_json}, + {"kube_options_merge_log_key_json", flb_test_options_merge_log_key_json}, + {"kube_options_keep_log_enabled_json", flb_test_options_keep_log_enabled_json}, + {"kube_options_keep_log_disabled_json", flb_test_options_keep_log_disabled_json}, + {"kube_options_k8s_logging_parser_disabled_text_stdout", flb_test_options_k8s_logging_parser_disabled_text_stdout}, + {"kube_options_k8s_logging_parser_disabled_text_stderr", flb_test_options_k8s_logging_parser_disabled_text_stderr}, + {"kube_options_k8s_logging_exclude_disabled_text_stdout", flb_test_options_k8s_logging_exclude_disabled_text_stdout}, + {"kube_options_k8s_logging_exclude_disabled_text_stderr", flb_test_options_k8s_logging_exclude_disabled_text_stderr}, + {"kube_annotations_invalid_text", flb_test_annotations_invalid_text}, + {"kube_annotations_parser_regex_with_time_text", flb_test_annotations_parser_regex_with_time_text}, + {"kube_annotations_parser_regex_with_time_invalid_text_1", flb_test_annotations_parser_regex_with_time_invalid_text_1}, + {"kube_annotations_parser_json_with_time_json", flb_test_annotations_parser_json_with_time_json}, + {"kube_annotations_parser_json_with_time_invalid_json_1", flb_test_annotations_parser_json_with_time_invalid_json_1}, + {"kube_annotations_parser_invalid_text_stdout", flb_test_annotations_parser_invalid_text_stdout}, + {"kube_annotations_parser_invalid_text_stderr", flb_test_annotations_parser_invalid_text_stderr}, + {"kube_annotations_parser_stdout_text_stdout", flb_test_annotations_parser_stdout_text_stdout}, + {"kube_annotations_parser_stdout_text_stderr", flb_test_annotations_parser_stdout_text_stderr}, + {"kube_annotations_parser_stderr_text_stdout", flb_test_annotations_parser_stderr_text_stdout}, + {"kube_annotations_parser_stderr_text_stderr", flb_test_annotations_parser_stderr_text_stderr}, + {"kube_annotations_parser_multiple_1_container_1_stdout", flb_test_annotations_parser_multiple_1_container_1_stdout}, + {"kube_annotations_parser_multiple_1_container_1_stderr", flb_test_annotations_parser_multiple_1_container_1_stderr}, + {"kube_annotations_parser_multiple_1_container_2_stdout", flb_test_annotations_parser_multiple_1_container_2_stdout}, + {"kube_annotations_parser_multiple_1_container_2_stderr", flb_test_annotations_parser_multiple_1_container_2_stderr}, + {"kube_annotations_parser_multiple_1_container_3_stdout", flb_test_annotations_parser_multiple_1_container_3_stdout}, + {"kube_annotations_parser_multiple_1_container_3_stderr", flb_test_annotations_parser_multiple_1_container_3_stderr}, + {"kube_annotations_parser_multiple_1_container_4_stdout", flb_test_annotations_parser_multiple_1_container_4_stdout}, + {"kube_annotations_parser_multiple_1_container_4_stderr", flb_test_annotations_parser_multiple_1_container_4_stderr}, + {"kube_annotations_parser_multiple_1_container_5_stdout", flb_test_annotations_parser_multiple_1_container_5_stdout}, + {"kube_annotations_parser_multiple_1_container_5_stderr", flb_test_annotations_parser_multiple_1_container_5_stderr}, + {"kube_annotations_parser_multiple_2_container_1_stdout", flb_test_annotations_parser_multiple_2_container_1_stdout}, + {"kube_annotations_parser_multiple_2_container_1_stderr", flb_test_annotations_parser_multiple_2_container_1_stderr}, + {"kube_annotations_parser_multiple_2_container_2_stdout", flb_test_annotations_parser_multiple_2_container_2_stdout}, + {"kube_annotations_parser_multiple_2_container_2_stderr", flb_test_annotations_parser_multiple_2_container_2_stderr}, + {"kube_annotations_parser_multiple_2_container_3_stdout", flb_test_annotations_parser_multiple_2_container_3_stdout}, + {"kube_annotations_parser_multiple_2_container_3_stderr", flb_test_annotations_parser_multiple_2_container_3_stderr}, + {"kube_annotations_parser_multiple_2_container_4_stdout", flb_test_annotations_parser_multiple_2_container_4_stdout}, + {"kube_annotations_parser_multiple_2_container_4_stderr", flb_test_annotations_parser_multiple_2_container_4_stderr}, + {"kube_annotations_parser_multiple_2_container_5_stdout", flb_test_annotations_parser_multiple_2_container_5_stdout}, + {"kube_annotations_parser_multiple_2_container_5_stderr", flb_test_annotations_parser_multiple_2_container_5_stderr}, + {"kube_annotations_exclude_default_text", flb_test_annotations_exclude_default_text}, + {"kube_annotations_exclude_invalid_text_stdout", flb_test_annotations_exclude_invalid_text_stdout}, + {"kube_annotations_exclude_invalid_text_stderr", flb_test_annotations_exclude_invalid_text_stderr}, + {"kube_annotations_exclude_stdout_text_stdout", flb_test_annotations_exclude_stdout_text_stdout}, + {"kube_annotations_exclude_stdout_text_stderr", flb_test_annotations_exclude_stdout_text_stderr}, + {"kube_annotations_exclude_stderr_text_stdout", flb_test_annotations_exclude_stderr_text_stdout}, + {"kube_annotations_exclude_stderr_text_stderr", flb_test_annotations_exclude_stderr_text_stderr}, + {"kube_annotations_exclude_multiple_1_container_1_stdout", flb_test_annotations_exclude_multiple_1_container_1_stdout}, + {"kube_annotations_exclude_multiple_1_container_1_stderr", flb_test_annotations_exclude_multiple_1_container_1_stderr}, + {"kube_annotations_exclude_multiple_1_container_2_stdout", flb_test_annotations_exclude_multiple_1_container_2_stdout}, + {"kube_annotations_exclude_multiple_1_container_2_stderr", flb_test_annotations_exclude_multiple_1_container_2_stderr}, + {"kube_annotations_exclude_multiple_1_container_3_stdout", flb_test_annotations_exclude_multiple_1_container_3_stdout}, + {"kube_annotations_exclude_multiple_1_container_3_stderr", flb_test_annotations_exclude_multiple_1_container_3_stderr}, + {"kube_annotations_exclude_multiple_1_container_4_stdout", flb_test_annotations_exclude_multiple_1_container_4_stdout}, + {"kube_annotations_exclude_multiple_1_container_4_stderr", flb_test_annotations_exclude_multiple_1_container_4_stderr}, + {"kube_annotations_exclude_multiple_2_container_1_stdout", flb_test_annotations_exclude_multiple_2_container_1_stdout}, + {"kube_annotations_exclude_multiple_2_container_1_stderr", flb_test_annotations_exclude_multiple_2_container_1_stderr}, + {"kube_annotations_exclude_multiple_2_container_2_stdout", flb_test_annotations_exclude_multiple_2_container_2_stdout}, + {"kube_annotations_exclude_multiple_2_container_2_stderr", flb_test_annotations_exclude_multiple_2_container_2_stderr}, + {"kube_annotations_exclude_multiple_2_container_3_stdout", flb_test_annotations_exclude_multiple_2_container_3_stdout}, + {"kube_annotations_exclude_multiple_2_container_3_stderr", flb_test_annotations_exclude_multiple_2_container_3_stderr}, + {"kube_annotations_exclude_multiple_2_container_4_stdout", flb_test_annotations_exclude_multiple_2_container_4_stdout}, + {"kube_annotations_exclude_multiple_2_container_4_stderr", flb_test_annotations_exclude_multiple_2_container_4_stderr}, + {"kube_annotations_exclude_multiple_3_container_1_stdout", flb_test_annotations_exclude_multiple_3_container_1_stdout}, + {"kube_annotations_exclude_multiple_3_container_1_stderr", flb_test_annotations_exclude_multiple_3_container_1_stderr}, + {"kube_annotations_exclude_multiple_3_container_2_stdout", flb_test_annotations_exclude_multiple_3_container_2_stdout}, + {"kube_annotations_exclude_multiple_3_container_2_stderr", flb_test_annotations_exclude_multiple_3_container_2_stderr}, + {"kube_annotations_exclude_multiple_3_container_3_stdout", flb_test_annotations_exclude_multiple_3_container_3_stdout}, + {"kube_annotations_exclude_multiple_3_container_3_stderr", flb_test_annotations_exclude_multiple_3_container_3_stderr}, + {"kube_annotations_exclude_multiple_3_container_4_stdout", flb_test_annotations_exclude_multiple_3_container_4_stdout}, + {"kube_annotations_exclude_multiple_3_container_4_stderr", flb_test_annotations_exclude_multiple_3_container_4_stderr}, + {"kube_annotations_exclude_multiple_4_container_1_stdout", flb_test_annotations_exclude_multiple_4_container_1_stdout}, + {"kube_annotations_exclude_multiple_4_container_1_stderr", flb_test_annotations_exclude_multiple_4_container_1_stderr}, + {"kube_annotations_exclude_multiple_4_container_2_stdout", flb_test_annotations_exclude_multiple_4_container_2_stdout}, + {"kube_annotations_exclude_multiple_4_container_2_stderr", flb_test_annotations_exclude_multiple_4_container_2_stderr}, + {"kube_annotations_exclude_multiple_4_container_3_stdout", flb_test_annotations_exclude_multiple_4_container_3_stdout}, + {"kube_annotations_exclude_multiple_4_container_3_stderr", flb_test_annotations_exclude_multiple_4_container_3_stderr}, + {"kube_annotations_exclude_multiple_4_container_4_stdout", flb_test_annotations_exclude_multiple_4_container_4_stdout}, + {"kube_annotations_exclude_multiple_4_container_4_stderr", flb_test_annotations_exclude_multiple_4_container_4_stderr}, +#ifdef FLB_HAVE_SYSTEMD + {"kube_systemd_logs", flb_test_systemd_logs}, +#endif + {NULL, NULL} +}; diff --git a/src/fluent-bit/tests/runtime/filter_kubernetes.md b/src/fluent-bit/tests/runtime/filter_kubernetes.md new file mode 100644 index 000000000..8f213abc4 --- /dev/null +++ b/src/fluent-bit/tests/runtime/filter_kubernetes.md @@ -0,0 +1,69 @@ +# Filter Kubernetes Tests + +The tests implemented on [filter_kubernets.c]() file aims to take different Kubernetes Pod log files (formats) and play with different setups. These files resides in the [data/kubernetes]() directory and each .log file have it corresponding .meta which contains the metadata associated with the Pod. + +The unit test implements a fake Kubernetes API server to expose .meta files when the filter query for such data. This make things easier to test without to spawn a real cluster. + +If you find that a specific test is missing, don't hesitate to open an issue on our Github repository. + +## Tests Available + +All log files used in these tests have been generated in a single Kubernetes cluster with Minikube. + +- [Apache Logs](#apache-logs) + +- [Apache Logs Annotated](#apache-logs-annotated) + +- [Apache Logs Annotated Invalid](#apache-logs-annotated-invalid) + +- [JSON Stringify](#json-stringify) + +- [JSON Invalid](#json-invalid) + +- No Log + + ​ + +#### Apache Logs + +||| +|-|-| +| Description | Simple Apache access log line | +| Log File | apache-logs_default_apache-logs-ac6095b6c715d823d732dcc9067f75b1299de5cc69a012b08d616a6058bdc0ad.log | +| Command | $ kubectl run apache-logs --rm --attach --restart=Never --image=edsiper/apache_logs | + +#### Apache Logs Annotated + +| | | +| ----------- | ------------------------------------------------------------ | +| Description | Simple Apache access log line with annotation suggesting a registered Parser | +| Log File | apache-logs-annotated_default_apache-logs-annotated-5c79b78d458d86fff56127cc8657058c10b837d0f2c147b61afea4c8bc65fad7.log | +| Command | $ kubectl run apache-logs-annotated --rm --attach --restart=Never --image=edsiper/apache_logs | +| Command | $ kubectl annotate pods apache-logs-annotated logging.parser='apache' | + + + +#### Apache Logs Annotated Invalid + +| | | +| ----------- | ------------------------------------------------------------ | +| Description | Simple Apache access log line with annotation suggesting an invalid Parser | +| Log File | apache-logs-annotated-invalid_default_apache-logs-annotated-invalid-b8aab41f6104d7d7ea121852cd00276d8fe42d2a3192b3ae8f949477a272b91b.log | +| Command | $ kubectl run apache-logs-annotated-invalid --rm --attach --restart=Never --image=edsiper/apache_logs | +| Command | $ kubectl annotate pods apache-logs-annotated-invalid logging.parser='404' | + +#### JSON Stringify + +| | | +| ----------- | ------------------------------------------------------------ | +| Description | Application writes a JSON message, it become stringified by Docker. | +| Log File | json-logs_default_json-logs-c053db7370be9c33d64677f9759863d850ebe35104069bec241cd1bb4674bd19.log | +| Command | $ kubectl run json-logs --rm --attach --restart=Never --image=edsiper/json_logs | + +#### JSON Invalid + +| | | +| ----------- | ------------------------------------------------------------ | +| Description | Application writes an invalid JSON message. | +| Log File | json-logs-invalid_default_json-logs-invalid-054e8bb83c2cc890bae4a184e7a2f96f18dfb121f83e4c5c5541dd452fa4e58e.log | +| Command | $ kubectl run json-logs-invalid --rm --attach --restart=Never --image=edsiper/json_logs_invalid |
\ No newline at end of file diff --git a/src/fluent-bit/tests/runtime/filter_log_to_metrics.c b/src/fluent-bit/tests/runtime/filter_log_to_metrics.c new file mode 100644 index 000000000..eb38af61c --- /dev/null +++ b/src/fluent-bit/tests/runtime/filter_log_to_metrics.c @@ -0,0 +1,698 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2023 The Fluent Bit Authors + * Copyright (C) 2023 SAP SE + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <fluent-bit.h> +#include <fluent-bit/flb_time.h> +#include <sys/types.h> +#include <sys/stat.h> +#include <fcntl.h> +#include "flb_tests_runtime.h" + +/* +* +-------------------------------+-------------------------------+ +* | | | +* | +--------------------+ | +--------------------+ | +* | | INPUTS | | ->| INPUT (IN_EMITTER | | +* | | | |/ | METRICS)| | +* | +--------------------+ / +--------------------+ | +* | | /| | | +* | v / | | | +* | +--------------------+ | | | | +* | | FILTER (KUBERNETES)| | | | | +* | | (NOT USED IN TEST)| | | | | +* | +--------------------+ | | | | +* | | | | | | +* | v / | | | +* | +--------------------+ / | | | +* | | FILTER (LOG METRIC)| / | | | +* | | |-/ | | | +* | +--------------------+ | | | +* | | | | | +* | v | v | +* | +--------------------+ | +--------------------+ | +* | | OUTPUT | | | OUTPUT (METRICS) | | +* | | (NONE IN THIS TEST)| | | | | +* | +--------------------+ | +--------------------+ | +* | | | +* +-------------------------------+-------------------------------+ +*/ + +/* Test functions */ +void flb_test_log_to_metrics_counter_k8s(void); +void flb_test_log_to_metrics_counter(void); +void flb_test_log_to_metrics_counter_k8s_two_tuples(void); +void flb_test_log_to_metrics_gauge(void); +void flb_test_log_to_metrics_histogram(void); +void flb_test_log_to_metrics_reg(void); +void flb_test_log_to_metrics_empty_label_keys_regex(void); +void flb_test_log_to_metrics_label(void); + + +/* Test data */ +#define JSON_MSG1 "[" \ + "1448403340," \ + "{" \ + "\"message\": \"dummy\"," \ + "\"kubernetes\":{" \ + "\"container_name\": \"mycontainer\"," \ + "\"namespace_name\": \"k8s-dummy\"," \ + "\"docker_id\": \"abc123\"," \ + "\"pod_name\": \"testpod\"," \ + "\"pod_id\": \"def456\"," \ + "}," \ + "\"duration\": \"20\"," \ + "\"color\": \"red\"," \ + "\"direction\": \"right\"" \ + "}]" + +#define JSON_MSG2 "[" \ + "1448403341," \ + "{" \ + "\"message\": \"dummy\"," \ + "\"kubernetes\":{" \ + "\"container_name\": \"mycontainer\"," \ + "\"namespace_name\": \"k8s-dummy\"," \ + "\"docker_id\": \"abc123\"," \ + "\"pod_name\": \"testpod\"," \ + "\"pod_id\": \"def456\"," \ + "}," \ + "\"duration\": \"20\"," \ + "\"color\": \"red\"," \ + "\"direction\": \"left\"" \ + "}]" + +#define JSON_MSG3 "[" \ + "1448403341," \ + "{" \ + "\"message\": \"hello\"," \ + "\"kubernetes\":{" \ + "\"container_name\": \"mycontainer\"," \ + "\"namespace_name\": \"k8s-dummy\"," \ + "\"docker_id\": \"abc123\"," \ + "\"pod_name\": \"testpod\"," \ + "\"pod_id\": \"def456\"," \ + "}," \ + "\"duration\": \"20\"," \ + "\"color\": \"red\"," \ + "\"direction\": \"left\"" \ + "}]" + +/* Test list */ +TEST_LIST = { + {"counter_k8s", flb_test_log_to_metrics_counter_k8s }, + {"counter", flb_test_log_to_metrics_counter }, + {"counter_k8s_two_tuples", flb_test_log_to_metrics_counter_k8s_two_tuples }, + {"gauge", flb_test_log_to_metrics_gauge }, + {"histogram", flb_test_log_to_metrics_histogram }, + {"counter_regex", flb_test_log_to_metrics_reg }, + {"regex_empty_label_keys", flb_test_log_to_metrics_empty_label_keys_regex }, + {"label", flb_test_log_to_metrics_label }, + {NULL, NULL} +}; + +pthread_mutex_t result_mutex = PTHREAD_MUTEX_INITIALIZER; +int data_size = 0; +bool new_data = false; +char output[32768]; + + +int callback_test(void* data, size_t size, void* cb_data) +{ + if (size > 0) { + new_data = true; + flb_debug("[test_filter_log_to_metrics] received message: %s", (char*)data); + pthread_mutex_lock(&result_mutex); + strncat(output, data, size); + data_size = size; + pthread_mutex_unlock(&result_mutex); + } + flb_free(data); + return 0; +} + +static void filter_test_destroy(flb_ctx_t *ctx) +{ + sleep(1); + flb_stop(ctx); + flb_destroy(ctx); +} + +void wait_with_timeout(uint32_t timeout_ms, char *out_result) +{ + struct flb_time start_time; + struct flb_time end_time; + struct flb_time diff_time; + uint64_t elapsed_time_flb = 0; + flb_time_get(&start_time); + + while (true) { + if(new_data){ + pthread_mutex_lock(&result_mutex); + new_data = false; + strcat(out_result, output); + pthread_mutex_unlock(&result_mutex); + + } + + flb_time_msleep(100); + flb_time_get(&end_time); + flb_time_diff(&end_time, &start_time, &diff_time); + elapsed_time_flb = flb_time_to_nanosec(&diff_time) / 1000000; + + if (elapsed_time_flb > timeout_ms) { + // Reached timeout. + break; + } + } +} + +void flb_test_log_to_metrics_counter_k8s(void) +{ + int ret; + int i; + flb_ctx_t *ctx; + int in_ffd; + int filter_ffd; + int out_ffd; + char *result = NULL; + struct flb_lib_out_cb cb_data; + char *input = JSON_MSG1; + char finalString[32768] = ""; + + const char *expected = "\"value\":5.0,\"labels\":[\"k8s-dummy\"," + "\"testpod\",\"mycontainer\",\"abc123\"," + "\"def456\",\"red\",\"right\"]"; + + ctx = flb_create(); + flb_service_set(ctx, "Flush", "0.200000000", "Grace", "1", "Log_Level", + "error", NULL); + + cb_data.cb = callback_test; + cb_data.data = NULL; + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + filter_ffd = flb_filter(ctx, (char *) "log_to_metrics", NULL); + TEST_CHECK(filter_ffd >= 0); + ret = flb_filter_set(ctx, filter_ffd, + "Match", "*", + "Tag", "test_metric", + "metric_mode", "counter", + "metric_name", "test", + "metric_description", "Counts messages", + "kubernetes_mode", "on", + "label_field", "color", + "label_field", "direction", + NULL); + + out_ffd = flb_output(ctx, (char *) "lib", (void *)&cb_data); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, + "match", "*", + "format", "json", + NULL); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + for (i = 0; i < 5; i++){ + flb_lib_push(ctx, in_ffd, input, strlen(input)); + } + wait_with_timeout(2000, finalString); + result = strstr(finalString, expected); + if (!TEST_CHECK(result != NULL)) { + TEST_MSG("expected substring:\n%s\ngot:\n%s\n", expected, finalString); + } + + filter_test_destroy(ctx); + +} + +void flb_test_log_to_metrics_counter(void) +{ + int ret; + int i; + flb_ctx_t *ctx; + int in_ffd; + int filter_ffd; + int out_ffd; + char *result = NULL; + struct flb_lib_out_cb cb_data; + char *input = JSON_MSG1; + char finalString[32768] = ""; + const char *expected = "\"value\":5.0,\"labels\":[\"red\",\"right\"]"; + + ctx = flb_create(); + flb_service_set(ctx, "Flush", "0.200000000", "Grace", "1", "Log_Level", + "error", NULL); + + cb_data.cb = callback_test; + cb_data.data = NULL; + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + filter_ffd = flb_filter(ctx, (char *) "log_to_metrics", NULL); + TEST_CHECK(filter_ffd >= 0); + ret = flb_filter_set(ctx, filter_ffd, + "Match", "*", + "Tag", "test_metric", + "metric_mode", "counter", + "metric_name", "test", + "metric_description", "Counts messages", + "kubernetes_mode", "off", + "label_field", "color", + "label_field", "direction", + NULL); + + out_ffd = flb_output(ctx, (char *) "lib", (void *)&cb_data); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, + "match", "*", + "format", "json", + NULL); + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + for (i = 0; i < 5; i++){ + flb_lib_push(ctx, in_ffd, input, strlen(input)); + } + wait_with_timeout(2000, finalString); + result = strstr(finalString, expected); + if (!TEST_CHECK(result != NULL)) { + TEST_MSG("expected substring:\n%s\ngot:\n%s\n", expected, finalString); + } + filter_test_destroy(ctx); + +} + +void flb_test_log_to_metrics_counter_k8s_two_tuples(void) +{ + int ret; + int i; + flb_ctx_t *ctx; + int in_ffd; + int filter_ffd; + int out_ffd; + char *result = NULL; + struct flb_lib_out_cb cb_data; + char *input1 = JSON_MSG1; + char *input2 = JSON_MSG2; + char finalString[32768] = ""; + const char *expected1 = "\"value\":5.0,\"labels\":[\"k8s-dummy\"," + "\"testpod\",\"mycontainer\",\"abc123\"," + "\"def456\",\"red\",\"right\"]"; + const char *expected2 = "\"value\":3.0,\"labels\":[\"k8s-dummy\"," + "\"testpod\",\"mycontainer\",\"abc123\"," + "\"def456\",\"red\",\"left\"]"; + + + ctx = flb_create(); + flb_service_set(ctx, "Flush", "0.200000000", "Grace", "1", "Log_Level", + "error", NULL); + + cb_data.cb = callback_test; + cb_data.data = NULL; + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + filter_ffd = flb_filter(ctx, (char *) "log_to_metrics", NULL); + TEST_CHECK(filter_ffd >= 0); + ret = flb_filter_set(ctx, filter_ffd, + "Match", "*", + "Tag", "test_metric", + "metric_mode", "counter", + "metric_name", "test", + "metric_description", "Counts two different messages", + "kubernetes_mode", "on", + "label_field", "color", + "label_field", "direction", + NULL); + + out_ffd = flb_output(ctx, (char *) "lib", (void *)&cb_data); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, + "match", "*", + "format", "json", + NULL); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + for (i = 0; i < 5; i++){ + flb_lib_push(ctx, in_ffd, input1, strlen(input1)); + } + for (i = 0; i < 3; i++){ + flb_lib_push(ctx, in_ffd, input2, strlen(input2)); + } + wait_with_timeout(2000, finalString); + result = strstr(finalString, expected1); + if (!TEST_CHECK(result != NULL)) { + TEST_MSG("expected substring:\n%s\ngot:\n%s\n", expected1, finalString); + } + + result = strstr(finalString, expected2); + if (!TEST_CHECK(result != NULL)) { + TEST_MSG("expected substring:\n%s\ngot:\n%s\n", expected2, finalString); + } + + filter_test_destroy(ctx); + +} + +void flb_test_log_to_metrics_gauge(void) +{ + int ret; + flb_ctx_t *ctx; + int in_ffd; + int filter_ffd; + int out_ffd; + char *result = NULL; + struct flb_lib_out_cb cb_data; + char *input = JSON_MSG1; + char finalString[32768] = ""; + const char *expected = "\"value\":20.0,\"labels\":[\"red\",\"right\"]"; + + ctx = flb_create(); + flb_service_set(ctx, "Flush", "0.200000000", "Grace", "1", "Log_Level", + "error", NULL); + + cb_data.cb = callback_test; + cb_data.data = NULL; + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + filter_ffd = flb_filter(ctx, (char *) "log_to_metrics", NULL); + TEST_CHECK(filter_ffd >= 0); + ret = flb_filter_set(ctx, filter_ffd, + "Match", "*", + "Tag", "test_metric", + "metric_mode", "gauge", + "metric_name", "test", + "metric_description", "Reports gauge from messages", + "kubernetes_mode", "off", + "value_field", "duration", + "label_field", "color", + "label_field", "direction", + NULL); + + out_ffd = flb_output(ctx, (char *) "lib", (void *)&cb_data); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, + "match", "*", + "format", "json", + NULL); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + flb_lib_push(ctx, in_ffd, input, strlen(input)); + + wait_with_timeout(2000, finalString); + result = strstr(finalString, expected); + if (!TEST_CHECK(result != NULL)) { + TEST_MSG("expected substring:\n%s\ngot:\n%s\n", expected, finalString); + } + + filter_test_destroy(ctx); + +} + + +void flb_test_log_to_metrics_histogram(void) +{ + int ret; + int i; + flb_ctx_t *ctx; + int in_ffd; + int filter_ffd; + int out_ffd; + char *result = NULL; + struct flb_lib_out_cb cb_data; + char *input = JSON_MSG1; + char finalString[32768] = ""; + const char *expected = "\"histogram\":{\"buckets\":" \ + "[0,0,0,0,0,0,0,0,0,0,0,5],\"" \ + "sum\":100.0,\"count\":5},\"" \ + "labels\":[\"red\",\"right\"]"; + ctx = flb_create(); + flb_service_set(ctx, "Flush", "0.200000000", "Grace", "1", "Log_Level", + "error", NULL); + + cb_data.cb = callback_test; + cb_data.data = NULL; + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + filter_ffd = flb_filter(ctx, (char *) "log_to_metrics", NULL); + TEST_CHECK(filter_ffd >= 0); + ret = flb_filter_set(ctx, filter_ffd, + "Match", "*", + "Tag", "test_metric", + "metric_mode", "histogram", + "metric_name", "test", + "metric_description", "Histogram of duration", + "kubernetes_mode", "off", + "value_field", "duration", + "label_field", "color", + "label_field", "direction", + NULL); + + out_ffd = flb_output(ctx, (char *) "lib", (void *)&cb_data); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, + "match", "*", + "format", "json", + NULL); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + for(i = 0; i < 5; i++){ + flb_lib_push(ctx, in_ffd, input, strlen(input)); + } + + wait_with_timeout(2000, finalString); + result = strstr(finalString, expected); + if (!TEST_CHECK(result != NULL)) { + TEST_MSG("expected substring:\n%s\ngot:\n%s\n", expected, finalString); + } + filter_test_destroy(ctx); + +} + +void flb_test_log_to_metrics_reg(void) +{ + int ret; + int i; + flb_ctx_t *ctx; + int in_ffd; + int filter_ffd; + int out_ffd; + char *result = NULL; + struct flb_lib_out_cb cb_data; + char *input1 = JSON_MSG1; + char *input2 = JSON_MSG3; + char finalString[32768] = ""; + const char *expected = "\"value\":3.0,\"labels\":[\"red\",\"left\"]"; + + + ctx = flb_create(); + flb_service_set(ctx, "Flush", "0.200000000", "Grace", "1", "Log_Level", + "error", NULL); + + cb_data.cb = callback_test; + cb_data.data = NULL; + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + filter_ffd = flb_filter(ctx, (char *) "log_to_metrics", NULL); + TEST_CHECK(filter_ffd >= 0); + ret = flb_filter_set(ctx, filter_ffd, + "Match", "*", + "Tag", "test_metric", + "metric_mode", "counter", + "metric_name", "test", + "metric_description", "Counts messages with regex", + "kubernetes_mode", "off", + "label_field", "color", + "label_field", "direction", + "regex", "message .*el.*", + NULL); + + out_ffd = flb_output(ctx, (char *) "lib", (void *)&cb_data); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, + "match", "*", + "format", "json", + NULL); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + + for (i = 0; i < 3; i++){ + flb_lib_push(ctx, in_ffd, input1, strlen(input1)); + flb_lib_push(ctx, in_ffd, input2, strlen(input2)); + } + wait_with_timeout(2000, finalString); + result = strstr(finalString, expected); + if (!TEST_CHECK(result != NULL)) { + TEST_MSG("expected substring:\n%s\ngot:\n%s\n", expected, finalString); + } + + filter_test_destroy(ctx); + +} + +void flb_test_log_to_metrics_empty_label_keys_regex(void) +{ + int ret; + int i; + flb_ctx_t *ctx; + int in_ffd; + int filter_ffd; + int out_ffd; + char *result = NULL; + struct flb_lib_out_cb cb_data; + char *input = JSON_MSG3; + char finalString[32768] = ""; + const char *expected = "\"value\":3.0,"; + + + ctx = flb_create(); + flb_service_set(ctx, "Flush", "0.200000000", "Grace", "1", "Log_Level", + "error", NULL); + + cb_data.cb = callback_test; + cb_data.data = NULL; + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + filter_ffd = flb_filter(ctx, (char *) "log_to_metrics", NULL); + TEST_CHECK(filter_ffd >= 0); + ret = flb_filter_set(ctx, filter_ffd, + "Match", "*", + "Tag", "test_metric", + "metric_mode", "counter", + "metric_name", "test", + "metric_description", "Counts messages with regex", + "kubernetes_mode", "off", + "regex", "message .*el.*", + NULL); + + out_ffd = flb_output(ctx, (char *) "lib", (void *)&cb_data); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, + "match", "*", + "format", "json", + NULL); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + + for (i = 0; i < 3; i++){ + flb_lib_push(ctx, in_ffd, input, strlen(input)); + } + wait_with_timeout(2000, finalString); + result = strstr(finalString, expected); + if (!TEST_CHECK(result != NULL)) { + TEST_MSG("expected substring:\n%s\ngot:\n%s\n", expected, finalString); + } + + filter_test_destroy(ctx); +} + +void flb_test_log_to_metrics_label(void) +{ + int ret; + int i; + flb_ctx_t *ctx; + int in_ffd; + int filter_ffd; + int out_ffd; + char *result = NULL; + struct flb_lib_out_cb cb_data; + char *input = JSON_MSG1; + char finalString[32768] = ""; + const char *expected_label_name = ",\"labels\":[\"pod_name\"],"; + const char *expected_label_value = "\"value\":2.0,\"labels\":[\"testpod\"]"; + + ctx = flb_create(); + flb_service_set(ctx, "Flush", "0.200000000", "Grace", "1", "Log_Level", + "error", NULL); + + cb_data.cb = callback_test; + cb_data.data = NULL; + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + filter_ffd = flb_filter(ctx, (char *) "log_to_metrics", NULL); + TEST_CHECK(filter_ffd >= 0); + ret = flb_filter_set(ctx, filter_ffd, + "Match", "*", + "Tag", "test_metric", + "metric_mode", "counter", + "metric_name", "test", + "metric_description", "Counts messages", + "kubernetes_mode", "off", + "add_label", "pod_name $kubernetes['pod_name']", + NULL); + + out_ffd = flb_output(ctx, (char *) "lib", (void *)&cb_data); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, + "match", "*", + "format", "json", + NULL); + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + for (i = 0; i < 2; i++){ + flb_lib_push(ctx, in_ffd, input, strlen(input)); + } + wait_with_timeout(500, finalString); + result = strstr(finalString, expected_label_name); + if (!TEST_CHECK(result != NULL)) { + TEST_MSG("expected substring:\n%s\ngot:\n%s\n", expected_label_name, finalString); + } + result = strstr(finalString, expected_label_value); + if (!TEST_CHECK(result != NULL)) { + TEST_MSG("expected substring:\n%s\ngot:\n%s\n", expected_label_value, finalString); + } + filter_test_destroy(ctx); + +} diff --git a/src/fluent-bit/tests/runtime/filter_lua.c b/src/fluent-bit/tests/runtime/filter_lua.c new file mode 100644 index 000000000..1e3111266 --- /dev/null +++ b/src/fluent-bit/tests/runtime/filter_lua.c @@ -0,0 +1,984 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2019-2021 The Fluent Bit Authors + * Copyright (C) 2015-2018 Treasure Data Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <fluent-bit.h> +#include <fluent-bit/flb_time.h> +#include <sys/types.h> +#include <sys/stat.h> +#include <fcntl.h> +#include "flb_tests_runtime.h" + +#define TMP_LUA_PATH "a.lua" +#define FLUSH_INTERVAL "1.0" + +pthread_mutex_t result_mutex = PTHREAD_MUTEX_INITIALIZER; +char *output = NULL; +int num_output = 0; + +void set_output(char *val) +{ + pthread_mutex_lock(&result_mutex); + output = val; + pthread_mutex_unlock(&result_mutex); +} + +void clear_output() +{ + pthread_mutex_lock(&result_mutex); + output = NULL; + pthread_mutex_unlock(&result_mutex); +} + +char *get_output(void) +{ + char *val; + + pthread_mutex_lock(&result_mutex); + val = output; + pthread_mutex_unlock(&result_mutex); + + return val; +} + +static void clear_output_num() +{ + pthread_mutex_lock(&result_mutex); + num_output = 0; + pthread_mutex_unlock(&result_mutex); +} + +static int get_output_num() +{ + int ret; + pthread_mutex_lock(&result_mutex); + ret = num_output; + pthread_mutex_unlock(&result_mutex); + + return ret; +} + +static int cb_count_msgpack_events(void *record, size_t size, void *data) +{ + msgpack_unpacked result; + size_t off = 0; + + if (!TEST_CHECK(data != NULL)) { + flb_error("data is NULL"); + } + + /* Iterate each item array and apply rules */ + msgpack_unpacked_init(&result); + while (msgpack_unpack_next(&result, record, size, &off) == MSGPACK_UNPACK_SUCCESS) { + pthread_mutex_lock(&result_mutex); + num_output++; + pthread_mutex_unlock(&result_mutex); + } + msgpack_unpacked_destroy(&result); + + flb_free(record); + return 0; +} + +int callback_test(void* data, size_t size, void* cb_data) +{ + if (size > 0) { + flb_debug("[test_filter_lua] received message: %s", (char*)data); + set_output(data); /* success */ + } + return 0; +} + +int callback_cat(void* data, size_t size, void* cb_data) +{ + flb_sds_t *outbuf = cb_data; + if (size > 0) { + flb_debug("[test_filter_lua] received message: %s", (char*)data); + pthread_mutex_lock(&result_mutex); + flb_sds_cat_safe(outbuf, data, size); + flb_free(data); + pthread_mutex_unlock(&result_mutex); + } + return 0; +} + +void delete_script() +{ + unlink(TMP_LUA_PATH); + flb_debug("remove script\n"); +} + + +int create_script(char *script_body, size_t body_size) +{ + FILE *fp = NULL; + fp = fopen(TMP_LUA_PATH, "w+"); + if (fp == NULL) { + TEST_MSG("fopen error\n"); + return -1; + } + fwrite(script_body, body_size, 1, fp); + fflush(fp); + fclose(fp); + return 0; +} + +void wait_with_timeout(uint32_t timeout_ms, char **out_result) +{ + struct flb_time start_time; + struct flb_time end_time; + struct flb_time diff_time; + uint64_t elapsed_time_flb = 0; + char *output = NULL; + + flb_time_get(&start_time); + + while (true) { + output = get_output(); + + if (output != NULL) { + *out_result = output; + break; + } + + flb_time_msleep(100); + flb_time_get(&end_time); + flb_time_diff(&end_time, &start_time, &diff_time); + elapsed_time_flb = flb_time_to_nanosec(&diff_time) / 1000000; + + if (elapsed_time_flb > timeout_ms) { + flb_warn("[timeout] elapsed_time: %ld", elapsed_time_flb); + // Reached timeout. + break; + } + } +} + + +void flb_test_type_int_key(void) +{ + int ret; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + int filter_ffd; + char *output = NULL; + char *input = "[0, {\"key\":\"val\"}]"; + char *result; + struct flb_lib_out_cb cb_data; + + char *script_body = "" + "function lua_main(tag, timestamp, record)\n" + " new_record = record\n" + " new_record[\"lua_int\"] = 10.2\n" + " return 1, timestamp, new_record\n" + "end\n"; + + clear_output(); + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", FLUSH_INTERVAL, "grace", "1", NULL); + + /* Prepare output callback context*/ + cb_data.cb = callback_test; + cb_data.data = NULL; + + ret = create_script(script_body, strlen(script_body)); + TEST_CHECK(ret == 0); + /* Filter */ + filter_ffd = flb_filter(ctx, (char *) "lua", NULL); + TEST_CHECK(filter_ffd >= 0); + ret = flb_filter_set(ctx, filter_ffd, + "Match", "*", + "call", "lua_main", + "type_int_key", "lua_int", + "script", TMP_LUA_PATH, + NULL); + + /* Input */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + TEST_CHECK(in_ffd >= 0); + + /* Lib output */ + out_ffd = flb_output(ctx, (char *) "lib", (void *)&cb_data); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, + "match", "test", + "format", "json", + NULL); + + ret = flb_start(ctx); + TEST_CHECK(ret==0); + + flb_lib_push(ctx, in_ffd, input, strlen(input)); + wait_with_timeout(2000, &output); + result = strstr(output, "\"lua_int\":10."); + if(!TEST_CHECK(result == NULL)) { + TEST_MSG("output:%s\n", output); + } + result = strstr(output, "\"lua_int\":10"); + if(!TEST_CHECK(result != NULL)) { + TEST_MSG("output:%s\n", output); + } + + /* clean up */ + flb_lib_free(output); + delete_script(); + + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_type_int_key_multi(void) +{ + int ret; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + int filter_ffd; + char *output = NULL; + char *input = "[0, {\"key\":\"val\"}]"; + char *result; + struct flb_lib_out_cb cb_data; + + char *script_body = "" + "function lua_main(tag, timestamp, record)\n" + " new_record = record\n" + " new_record[\"lua_int_1\"] = 10.1\n" + " new_record[\"lua_int_2\"] = 100.2\n" + " return 1, timestamp, new_record\n" + "end\n"; + + clear_output(); + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", FLUSH_INTERVAL, "grace", "1", NULL); + + /* Prepare output callback context*/ + cb_data.cb = callback_test; + cb_data.data = NULL; + + ret = create_script(script_body, strlen(script_body)); + TEST_CHECK(ret == 0); + /* Filter */ + filter_ffd = flb_filter(ctx, (char *) "lua", NULL); + TEST_CHECK(filter_ffd >= 0); + ret = flb_filter_set(ctx, filter_ffd, + "Match", "*", + "call", "lua_main", + "type_int_key", "lua_int_1 lua_int_2", + "script", TMP_LUA_PATH, + NULL); + + /* Input */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + TEST_CHECK(in_ffd >= 0); + + /* Lib output */ + out_ffd = flb_output(ctx, (char *) "lib", (void *)&cb_data); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, + "match", "test", + "format", "json", + NULL); + + ret = flb_start(ctx); + TEST_CHECK(ret==0); + + flb_lib_push(ctx, in_ffd, input, strlen(input)); + wait_with_timeout(2000, &output); + + /* check if float */ + result = strstr(output, "\"lua_int_1\":10."); + if (!TEST_CHECK(result == NULL)) { + TEST_MSG("output:%s\n", output); + } + result = strstr(output, "\"lua_int_2\":100."); + if (!TEST_CHECK(result == NULL)) { + TEST_MSG("output:%s\n", output); + } + + /* check if int */ + result = strstr(output, "\"lua_int_1\":10"); + if (!TEST_CHECK(result != NULL)) { + TEST_MSG("output:%s\n", output); + } + result = strstr(output, "\"lua_int_2\":100"); + if (!TEST_CHECK(result != NULL)) { + TEST_MSG("output:%s\n", output); + } + + /* clean up */ + flb_lib_free(output); + delete_script(); + + flb_stop(ctx); + flb_destroy(ctx); +} + + +void flb_test_append_tag(void) +{ + int ret; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + int filter_ffd; + char *output = NULL; + char *input = "[0, {\"key\":\"val\"}]"; + char *result; + struct flb_lib_out_cb cb_data; + + char *script_body = "" + "function lua_main(tag, timestamp, record)\n" + " new_record = record\n" + " new_record[\"tag\"] = tag\n" + " return 1, timestamp, new_record\n" + "end\n"; + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", FLUSH_INTERVAL, "grace", "1", NULL); + + /* Prepare output callback context*/ + cb_data.cb = callback_test; + cb_data.data = NULL; + + ret = create_script(script_body, strlen(script_body)); + TEST_CHECK(ret == 0); + /* Filter */ + filter_ffd = flb_filter(ctx, (char *) "lua", NULL); + TEST_CHECK(filter_ffd >= 0); + ret = flb_filter_set(ctx, filter_ffd, + "Match", "*", + "call", "lua_main", + "script", TMP_LUA_PATH, + NULL); + + /* Input */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + TEST_CHECK(in_ffd >= 0); + + /* Lib output */ + out_ffd = flb_output(ctx, (char *) "lib", (void *)&cb_data); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, + "match", "test", + "format", "json", + NULL); + + ret = flb_start(ctx); + TEST_CHECK(ret==0); + + flb_lib_push(ctx, in_ffd, input, strlen(input)); + wait_with_timeout(2000, &output); + result = strstr(output, "\"tag\":\"test\""); + TEST_CHECK(result != NULL); + + /* clean up */ + flb_lib_free(output); + delete_script(); + + flb_stop(ctx); + flb_destroy(ctx); +} + + +void flb_test_helloworld(void) +{ + int ret; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + int filter_ffd; + + char *script_body = "" + "function lua_main(tag, timestamp, record)\n" + " print(\"hello world\")\n" + " return 0, timestamp, record\n" + "end\n"; + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", FLUSH_INTERVAL, "grace", "1", NULL); + + ret = create_script(script_body, strlen(script_body)); + TEST_CHECK(ret == 0); + /* Filter */ + filter_ffd = flb_filter(ctx, (char *) "lua", NULL); + TEST_CHECK(filter_ffd >= 0); + ret = flb_filter_set(ctx, filter_ffd, + "Match", "*", + "call", "lua_main", + "script", TMP_LUA_PATH, + NULL); + + /* Input */ + in_ffd = flb_input(ctx, (char *) "dummy", NULL); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + TEST_CHECK(in_ffd >= 0); + + /* Lib output */ + out_ffd = flb_output(ctx, (char *) "stdout", NULL); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, + "match", "test", + NULL); + + ret = flb_start(ctx); + TEST_CHECK(ret==0); + + delete_script(); + flb_stop(ctx); + flb_destroy(ctx); +} + +// https://github.com/fluent/fluent-bit/issues/3343 +void flb_test_type_array_key(void) +{ + int ret; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + int filter_ffd; + char *output = NULL; + char *input = "[0, {\"key\":\"val\"}]"; + char *result; + struct flb_lib_out_cb cb_data; + + char *script_body = "" + "function lua_main(tag, timestamp, record)\n" + " new_record = record\n" + " new_record[\"lua_array\"] = {};\n" + " new_record[\"lua_array2\"] = {1,2,3};\n" + " return 1, timestamp, new_record\n" + "end\n"; + + clear_output(); + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", FLUSH_INTERVAL, "grace", "1", NULL); + + /* Prepare output callback context*/ + cb_data.cb = callback_test; + cb_data.data = NULL; + + ret = create_script(script_body, strlen(script_body)); + TEST_CHECK(ret == 0); + /* Filter */ + filter_ffd = flb_filter(ctx, (char *) "lua", NULL); + TEST_CHECK(filter_ffd >= 0); + ret = flb_filter_set(ctx, filter_ffd, + "Match", "*", + "call", "lua_main", + "type_array_key", "lua_array lua_array2", + "script", TMP_LUA_PATH, + NULL); + + /* Input */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + TEST_CHECK(in_ffd >= 0); + + /* Lib output */ + out_ffd = flb_output(ctx, (char *) "lib", (void *)&cb_data); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, + "match", "test", + "format", "json", + NULL); + + ret = flb_start(ctx); + TEST_CHECK(ret==0); + + flb_lib_push(ctx, in_ffd, input, strlen(input)); + wait_with_timeout(2000, &output); + result = strstr(output, "\"lua_array\":[]"); + if(!TEST_CHECK(result != NULL)) { + TEST_MSG("output:%s\n", output); + } + result = strstr(output, "\"lua_array2\":[1,2,3]"); + if(!TEST_CHECK(result != NULL)) { + TEST_MSG("output:%s\n", output); + } + + /* clean up */ + flb_lib_free(output); + delete_script(); + + flb_stop(ctx); + flb_destroy(ctx); +} + +/* https://github.com/fluent/fluent-bit/issues/3433 */ +void flb_test_array_contains_null(void) +{ + int ret; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + int filter_ffd; + char *output = NULL; + char *input = "[0, {\"hello\": [1, null, \"world\"]}]"; + char *result; + struct flb_lib_out_cb cb_data; + + char *script_body = "" + "function lua_main(tag, timestamp, record)\n" + " new_record = record\n" + " new_record[\"modify\"] = \"yes\"\n" + " return 1, timestamp, new_record\n" + "end\n"; + + clear_output(); + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", FLUSH_INTERVAL, "grace", "1", NULL); + + /* Prepare output callback context*/ + cb_data.cb = callback_test; + cb_data.data = NULL; + + ret = create_script(script_body, strlen(script_body)); + TEST_CHECK(ret == 0); + /* Filter */ + filter_ffd = flb_filter(ctx, (char *) "lua", NULL); + TEST_CHECK(filter_ffd >= 0); + ret = flb_filter_set(ctx, filter_ffd, + "Match", "*", + "call", "lua_main", + "script", TMP_LUA_PATH, + NULL); + + /* Input */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + TEST_CHECK(in_ffd >= 0); + + /* Lib output */ + out_ffd = flb_output(ctx, (char *) "lib", (void *)&cb_data); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, + "match", "test", + "format", "json", + NULL); + + ret = flb_start(ctx); + TEST_CHECK(ret==0); + + flb_lib_push(ctx, in_ffd, input, strlen(input)); + wait_with_timeout(2000, &output); + result = strstr(output, "[1,null,\"world\"]"); + if(!TEST_CHECK(result != NULL)) { + TEST_MSG("output:%s\n", output); + } + result = strstr(output, "\"modify\":\"yes\""); + if(!TEST_CHECK(result != NULL)) { + TEST_MSG("output:%s\n", output); + } + + /* clean up */ + flb_lib_free(output); + delete_script(); + + flb_stop(ctx); + flb_destroy(ctx); +} + +/* https://github.com/fluent/fluent-bit/issues/5251 */ +void flb_test_drop_all_records(void) +{ + int ret; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + int filter_ffd; + char *output = NULL; + char *input = "[0, {\"key\":\"val\"}]"; + struct flb_lib_out_cb cb_data; + + char *script_body = "" + "function lua_main(tag, timestamp, record)\n" + " return -1, 0, 0\n" + "end\n"; + + clear_output_num(); + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", FLUSH_INTERVAL, "grace", "1", NULL); + + /* Prepare output callback context*/ + cb_data.cb = cb_count_msgpack_events; + cb_data.data = NULL; + + ret = create_script(script_body, strlen(script_body)); + TEST_CHECK(ret == 0); + /* Filter */ + filter_ffd = flb_filter(ctx, (char *) "lua", NULL); + TEST_CHECK(filter_ffd >= 0); + ret = flb_filter_set(ctx, filter_ffd, + "Match", "*", + "call", "lua_main", + "script", TMP_LUA_PATH, + NULL); + + /* Input */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + TEST_CHECK(in_ffd >= 0); + + /* Lib output */ + out_ffd = flb_output(ctx, (char *) "lib", (void *)&cb_data); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, + "match", "test", + NULL); + + ret = flb_start(ctx); + TEST_CHECK(ret==0); + + flb_lib_push(ctx, in_ffd, input, strlen(input)); + flb_time_msleep(1500); /* waiting flush */ + + ret = get_output_num(); + if (!TEST_CHECK(ret == 0)) { + TEST_MSG("error. got %d expect 0", ret); + } + + /* clean up */ + flb_lib_free(output); + delete_script(); + + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_enable_flb_null(void) +{ + int ret; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + int filter_ffd; + char *output = NULL; + char *input = "[0, {\"hello\":null}]"; + char *result; + struct flb_lib_out_cb cb_data; + + char *script_body = "" + "function lua_main(tag, timestamp, record)\n" + " return 1, timestamp, record\n" + "end\n"; + + clear_output(); + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", FLUSH_INTERVAL, "grace", "1", NULL); + + /* Prepare output callback context*/ + cb_data.cb = callback_test; + cb_data.data = NULL; + + ret = create_script(script_body, strlen(script_body)); + TEST_CHECK(ret == 0); + /* Filter */ + filter_ffd = flb_filter(ctx, (char *) "lua", NULL); + TEST_CHECK(filter_ffd >= 0); + ret = flb_filter_set(ctx, filter_ffd, + "Match", "*", + "call", "lua_main", + "script", TMP_LUA_PATH, + "enable_flb_null", "true", + NULL); + + /* Input */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + TEST_CHECK(in_ffd >= 0); + + /* Lib output */ + out_ffd = flb_output(ctx, (char *) "lib", (void *)&cb_data); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, + "match", "test", + "format", "json", + NULL); + + ret = flb_start(ctx); + TEST_CHECK(ret==0); + + flb_lib_push(ctx, in_ffd, input, strlen(input)); + wait_with_timeout(2000, &output); + result = strstr(output, "\"hello\":null"); + if(!TEST_CHECK(result != NULL)) { + TEST_MSG("output:%s\n", output); + } + + /* clean up */ + flb_lib_free(output); + delete_script(); + + flb_stop(ctx); + flb_destroy(ctx); +} + +/* https://github.com/fluent/fluent-bit/issues/5496 */ +void flb_test_split_record(void) +{ + int ret; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + int filter_ffd; + struct flb_lib_out_cb cb_data; + char *output = NULL; + flb_sds_t outbuf = flb_sds_create(""); + char *input = "[0, {\"x\": [ " + "{\"a1\":\"aa\"}, " + "{\"b1\":\"bb\"}, " + "{\"c1\":\"cc\"} ]}]"; + const char *expected = + "[5.000000,{\"a1\":\"aa\"}]" + "[5.000000,{\"b1\":\"bb\"}]" + "[5.000000,{\"c1\":\"cc\"}]"; + char *script_body = "" + "function lua_main(tag, timestamp, record)\n" + " return 1, 5, record.x\n" + "end\n"; + + clear_output(); + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", FLUSH_INTERVAL, "grace", "1", NULL); + + /* Prepare output callback context*/ + cb_data.cb = callback_cat; + cb_data.data = &outbuf; + + ret = create_script(script_body, strlen(script_body)); + TEST_CHECK(ret == 0); + /* Filter */ + filter_ffd = flb_filter(ctx, (char *) "lua", NULL); + TEST_CHECK(filter_ffd >= 0); + ret = flb_filter_set(ctx, filter_ffd, + "Match", "*", + "call", "lua_main", + "script", TMP_LUA_PATH, + NULL); + + /* Input */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + TEST_CHECK(in_ffd >= 0); + + /* Lib output */ + out_ffd = flb_output(ctx, (char *) "lib", (void *)&cb_data); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, + "match", "test", + "format", "json", + NULL); + + ret = flb_start(ctx); + TEST_CHECK(ret==0); + + flb_lib_push(ctx, in_ffd, input, strlen(input)); + wait_with_timeout(2000, &output); + if (!TEST_CHECK(!strcmp(outbuf, expected))) { + TEST_MSG("expected:\n%s\ngot:\n%s\n", expected, outbuf); + } + + /* clean up */ + flb_lib_free(output); + delete_script(); + + flb_stop(ctx); + flb_destroy(ctx); + flb_sds_destroy(outbuf); +} + +void flb_test_empty_array(void) +{ + int ret; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + int filter_ffd; + flb_sds_t outbuf = flb_sds_create(""); + char *input = "[0, {\"key\":[]}]"; + struct flb_lib_out_cb cb_data; + + const char *expected = + "[5.000000,{\"key\":[]}]"; + + char *script_body = "" + "function lua_main(tag, timestamp, record)\n" + " return 1, 5, record\n" + "end\n"; + + clear_output_num(); + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", FLUSH_INTERVAL, "grace", "1", NULL); + + /* Prepare output callback context*/ + cb_data.cb = callback_cat; + cb_data.data = &outbuf; + + ret = create_script(script_body, strlen(script_body)); + TEST_CHECK(ret == 0); + /* Filter */ + filter_ffd = flb_filter(ctx, (char *) "lua", NULL); + TEST_CHECK(filter_ffd >= 0); + ret = flb_filter_set(ctx, filter_ffd, + "Match", "*", + "call", "lua_main", + "script", TMP_LUA_PATH, + NULL); + + /* Input */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + TEST_CHECK(in_ffd >= 0); + + /* Lib output */ + out_ffd = flb_output(ctx, (char *) "lib", (void *)&cb_data); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, + "match", "test", + "format", "json", + NULL); + + ret = flb_start(ctx); + TEST_CHECK(ret==0); + + flb_lib_push(ctx, in_ffd, input, strlen(input)); + wait_with_timeout(2000, &output); + if (!TEST_CHECK(!strcmp(outbuf, expected))) { + TEST_MSG("expected:\n%s\ngot:\n%s\n", expected, outbuf); + } + + /* clean up */ + flb_lib_free(output); + delete_script(); + + flb_stop(ctx); + flb_destroy(ctx); + flb_sds_destroy(outbuf); +} + +void flb_test_invalid_metatable(void) +{ + int ret; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + int unused = 0; + int filter_ffd; + char *output = NULL; + char *input = "[0, {\"key\":\"val\"}]"; + struct flb_lib_out_cb cb_data; + + char *script_body = "" + "function lua_main(tag, timestamp, record)\n" + " meta = getmetatable(record)\n" + " meta[10] = \"hoge\"\n" + " return 1, timestamp, record\n" + "end\n"; + + clear_output_num(); + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", FLUSH_INTERVAL, "grace", "1", NULL); + + /* Prepare output callback context*/ + cb_data.cb = cb_count_msgpack_events; + cb_data.data = &unused; + + ret = create_script(script_body, strlen(script_body)); + TEST_CHECK(ret == 0); + /* Filter */ + filter_ffd = flb_filter(ctx, (char *) "lua", NULL); + TEST_CHECK(filter_ffd >= 0); + ret = flb_filter_set(ctx, filter_ffd, + "Match", "*", + "call", "lua_main", + "script", TMP_LUA_PATH, + NULL); + + /* Input */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + TEST_CHECK(in_ffd >= 0); + + /* Lib output */ + out_ffd = flb_output(ctx, (char *) "lib", (void *)&cb_data); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, + "match", "test", + NULL); + + ret = flb_start(ctx); + TEST_CHECK(ret==0); + + ret = flb_lib_push(ctx, in_ffd, input, strlen(input)); + if (!TEST_CHECK(ret != -1)) { + TEST_MSG("flb_lib_push error"); + } + flb_time_msleep(1500); /* waiting flush */ + + ret = get_output_num(); + if (!TEST_CHECK(ret > 0)) { + TEST_MSG("error. no output"); + } + + /* clean up */ + flb_lib_free(output); + delete_script(); + + flb_stop(ctx); + flb_destroy(ctx); +} + +TEST_LIST = { + {"hello_world", flb_test_helloworld}, + {"append_tag", flb_test_append_tag}, + {"type_int_key", flb_test_type_int_key}, + {"type_int_key_multi", flb_test_type_int_key_multi}, + {"type_array_key", flb_test_type_array_key}, + {"array_contains_null", flb_test_array_contains_null}, + {"drop_all_records", flb_test_drop_all_records}, + {"enable_flb_null", flb_test_enable_flb_null}, + {"split_record", flb_test_split_record}, + {"empty_array", flb_test_empty_array}, + {"invalid_metatable", flb_test_invalid_metatable}, + {NULL, NULL} +}; diff --git a/src/fluent-bit/tests/runtime/filter_modify.c b/src/fluent-bit/tests/runtime/filter_modify.c new file mode 100644 index 000000000..ac5a166be --- /dev/null +++ b/src/fluent-bit/tests/runtime/filter_modify.c @@ -0,0 +1,1674 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +#include <fluent-bit.h> +#include "flb_tests_runtime.h" + +struct filter_test { + flb_ctx_t *flb; /* Fluent Bit library context */ + int i_ffd; /* Input fd */ + int f_ffd; /* Filter fd */ +}; + +/* Callback to check expected results */ +static int cb_check_result(void *record, size_t size, void *data) +{ + char *p; + char *expected; + char *result; + + expected = (char *) data; + result = (char *) record; + + p = strstr(result, expected); + TEST_CHECK(p != NULL); + + if (!p) { + flb_error("Expected to find: '%s' in result '%s'", + expected, result); + } + /* + * If you want to debug your test + * + * printf("Expect: '%s' in result '%s'", expected, result); + */ + + flb_free(record); + return 0; +} + + + +static struct filter_test *filter_test_create(struct flb_lib_out_cb *data) +{ + int i_ffd; + int f_ffd; + int o_ffd; + struct filter_test *ctx; + + ctx = flb_malloc(sizeof(struct filter_test)); + if (!ctx) { + flb_errno(); + return NULL; + } + + /* Service config */ + ctx->flb = flb_create(); + flb_service_set(ctx->flb, + "Flush", "0.200000000", + "Grace", "1", + NULL); + + /* Input */ + i_ffd = flb_input(ctx->flb, (char *) "lib", NULL); + TEST_CHECK(i_ffd >= 0); + flb_input_set(ctx->flb, i_ffd, "tag", "test", NULL); + ctx->i_ffd = i_ffd; + + /* Filter configuration */ + f_ffd = flb_filter(ctx->flb, (char *) "modify", NULL); + TEST_CHECK(f_ffd >= 0); + flb_filter_set(ctx->flb, f_ffd, "match", "*", NULL); + ctx->f_ffd = f_ffd; + + /* Output */ + o_ffd = flb_output(ctx->flb, (char *) "lib", (void *) data); + TEST_CHECK(o_ffd >= 0); + flb_output_set(ctx->flb, o_ffd, + "match", "test", + "format", "json", + NULL); + + return ctx; +} + +static void filter_test_destroy(struct filter_test *ctx) +{ + sleep(1); + flb_stop(ctx->flb); + flb_destroy(ctx->flb); + flb_free(ctx); +} + +/* Operation: SET / append new record */ +static void flb_test_op_set_append() +{ + int len; + int ret; + int bytes; + char *p; + struct flb_lib_out_cb cb_data; + struct filter_test *ctx; + + /* Create test context */ + ctx = filter_test_create((void *) &cb_data); + if (!ctx) { + exit(EXIT_FAILURE); + } + + /* Configure filter */ + ret = flb_filter_set(ctx->flb, ctx->f_ffd, + "set", "test_key test_value", + NULL); + TEST_CHECK(ret == 0); + + /* Prepare output callback with expected result */ + cb_data.cb = cb_check_result; + cb_data.data = "\"test_key\":\"test_value\""; + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + /* Ingest data samples */ + p = "[0, {\"k\":\"sample\"}]"; + len = strlen(p); + bytes = flb_lib_push(ctx->flb, ctx->i_ffd, p, len); + TEST_CHECK(bytes == len); + + filter_test_destroy(ctx); +} + +/* Operation: SET / replace value of existing key */ +static void flb_test_op_set_replace() +{ + int len; + int ret; + int bytes; + char *p; + struct flb_lib_out_cb cb_data; + struct filter_test *ctx; + + /* Create test context */ + ctx = filter_test_create((void *) &cb_data); + if (!ctx) { + exit(EXIT_FAILURE); + } + + /* Configure filter */ + ret = flb_filter_set(ctx->flb, ctx->f_ffd, + "set", "k test_value", + NULL); + TEST_CHECK(ret == 0); + + /* Prepare output callback with expected result */ + cb_data.cb = cb_check_result; + cb_data.data = "\"k\":\"test_value\""; + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + /* Ingest data samples */ + p = "[0, {\"k\":\"sample\"}]"; + len = strlen(p); + bytes = flb_lib_push(ctx->flb, ctx->i_ffd, p, len); + TEST_CHECK(bytes == len); + + filter_test_destroy(ctx); +} + +/* Operation: REMOVE */ +static void flb_test_op_remove() +{ + int len; + int ret; + int bytes; + char *p; + struct flb_lib_out_cb cb_data; + struct filter_test *ctx; + + /* Create test context */ + ctx = filter_test_create((void *) &cb_data); + if (!ctx) { + exit(EXIT_FAILURE); + } + + /* Configure filter */ + ret = flb_filter_set(ctx->flb, ctx->f_ffd, + "remove", "remove", + NULL); + TEST_CHECK(ret == 0); + + /* Prepare output callback with expected result */ + cb_data.cb = cb_check_result; + cb_data.data = "\"k\":\"sample\""; + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + /* Ingest data samples */ + p = "[0, {\"k\":\"sample\",\"remove\":\"sample to remove\"}]"; + len = strlen(p); + bytes = flb_lib_push(ctx->flb, ctx->i_ffd, p, len); + TEST_CHECK(bytes == len); + + filter_test_destroy(ctx); +} + +/* Operation: REMOVE_WILDCARD */ +static void flb_test_op_remove_wildcard() +{ + int len; + int ret; + int bytes; + char *p; + struct flb_lib_out_cb cb_data; + struct filter_test *ctx; + + /* Create test context */ + ctx = filter_test_create((void *) &cb_data); + if (!ctx) { + exit(EXIT_FAILURE); + } + + /* Configure filter */ + ret = flb_filter_set(ctx->flb, ctx->f_ffd, + "remove_wildcard", "k", + NULL); + TEST_CHECK(ret == 0); + + /* Prepare output callback with expected result */ + cb_data.cb = cb_check_result; + cb_data.data = "\"a3\":\"sample3\""; + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + /* Ingest data samples */ + p = "[0,{\"k1\":\"sample1\",\"k2\":\"sample2\",\"a3\":\"sample3\"}]"; + len = strlen(p); + bytes = flb_lib_push(ctx->flb, ctx->i_ffd, p, len); + TEST_CHECK(bytes == len); + + filter_test_destroy(ctx); +} + +/* Operation: REMOVE_REGEX */ +static void flb_test_op_remove_regex() +{ + int len; + int ret; + int bytes; + char *p; + struct flb_lib_out_cb cb_data; + struct filter_test *ctx; + + /* Create test context */ + ctx = filter_test_create((void *) &cb_data); + if (!ctx) { + exit(EXIT_FAILURE); + } + + /* Configure filter */ + ret = flb_filter_set(ctx->flb, ctx->f_ffd, + "remove_regex", "^[a-z][0-9]$", + NULL); + TEST_CHECK(ret == 0); + + /* Prepare output callback with expected result */ + cb_data.cb = cb_check_result; + cb_data.data = "\"A3\":\"sample3\""; + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + /* Ingest data samples */ + p = "[0,{\"k1\":\"sample1\",\"k2\":\"sample2\",\"A3\":\"sample3\"}]"; + len = strlen(p); + bytes = flb_lib_push(ctx->flb, ctx->i_ffd, p, len); + TEST_CHECK(bytes == len); + + filter_test_destroy(ctx); +} + +/* + * Operation: RENAME / Try to rename Key where 'renamed' key already + * exists: do nothing. + */ +static void flb_test_op_rename_exists() +{ + int len; + int ret; + int bytes; + char *p; + struct flb_lib_out_cb cb_data; + struct filter_test *ctx; + + /* Create test context */ + ctx = filter_test_create((void *) &cb_data); + if (!ctx) { + exit(EXIT_FAILURE); + } + + /* Configure filter */ + ret = flb_filter_set(ctx->flb, ctx->f_ffd, + "rename", "A3 k2", + NULL); + TEST_CHECK(ret == 0); + + /* Prepare output callback with expected result */ + cb_data.cb = cb_check_result; + cb_data.data = "\"k1\":\"sample1\",\"k2\":\"sample2\",\"A3\":\"sample3\""; + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + /* Ingest data samples */ + p = "[0,{\"k1\":\"sample1\",\"k2\":\"sample2\",\"A3\":\"sample3\"}]"; + len = strlen(p); + bytes = flb_lib_push(ctx->flb, ctx->i_ffd, p, len); + TEST_CHECK(bytes == len); + + filter_test_destroy(ctx); +} + +/* Operation: RENAME / Rename when key DON'T exists */ +static void flb_test_op_rename_no_exists() +{ + int len; + int ret; + int bytes; + char *p; + struct flb_lib_out_cb cb_data; + struct filter_test *ctx; + + /* Create test context */ + ctx = filter_test_create((void *) &cb_data); + if (!ctx) { + exit(EXIT_FAILURE); + } + + /* Configure filter */ + ret = flb_filter_set(ctx->flb, ctx->f_ffd, + "rename", "A3 a3", + NULL); + TEST_CHECK(ret == 0); + + /* Prepare output callback with expected result */ + cb_data.cb = cb_check_result; + cb_data.data = "\"k1\":\"sample1\",\"k2\":\"sample2\",\"a3\":\"sample3\""; + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + /* Ingest data samples */ + p = "[0,{\"k1\":\"sample1\",\"k2\":\"sample2\",\"A3\":\"sample3\"}]"; + len = strlen(p); + bytes = flb_lib_push(ctx->flb, ctx->i_ffd, p, len); + TEST_CHECK(bytes == len); + + filter_test_destroy(ctx); +} + +/* + * Operation: HARD_RENAME / Try to rename Key where 'renamed' key already + * exists: do nothing. + */ +static void flb_test_op_hard_rename_exists() +{ + int len; + int ret; + int bytes; + char *p; + struct flb_lib_out_cb cb_data; + struct filter_test *ctx; + + /* Create test context */ + ctx = filter_test_create((void *) &cb_data); + if (!ctx) { + exit(EXIT_FAILURE); + } + + /* Configure filter */ + ret = flb_filter_set(ctx->flb, ctx->f_ffd, + "hard_rename", "k2 k1", + NULL); + TEST_CHECK(ret == 0); + + /* Prepare output callback with expected result */ + cb_data.cb = cb_check_result; + cb_data.data = "\"k1\":\"sample2\""; + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + /* Ingest data samples */ + p = "[0,{\"k1\":\"sample1\",\"k2\":\"sample2\"}]"; + len = strlen(p); + bytes = flb_lib_push(ctx->flb, ctx->i_ffd, p, len); + TEST_CHECK(bytes == len); + + filter_test_destroy(ctx); +} + +/* Operation: HARD_RENAME / Rename when key DON'T exists */ +static void flb_test_op_hard_rename_no_exists() +{ + int len; + int ret; + int bytes; + char *p; + struct flb_lib_out_cb cb_data; + struct filter_test *ctx; + + /* Create test context */ + ctx = filter_test_create((void *) &cb_data); + if (!ctx) { + exit(EXIT_FAILURE); + } + + /* Configure filter */ + ret = flb_filter_set(ctx->flb, ctx->f_ffd, + "hard_rename", "k2 k3", + NULL); + TEST_CHECK(ret == 0); + + /* Prepare output callback with expected result */ + cb_data.cb = cb_check_result; + cb_data.data = "\"k3\":\"sample2\""; + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + /* Ingest data samples */ + p = "[0,{\"k1\":\"sample1\",\"k2\":\"sample2\"}]"; + len = strlen(p); + bytes = flb_lib_push(ctx->flb, ctx->i_ffd, p, len); + TEST_CHECK(bytes == len); + + filter_test_destroy(ctx); +} + +/* Operation: COPY / Target key already exists, do nothing */ +static void flb_test_op_copy_exists() +{ + int len; + int ret; + int bytes; + char *p; + struct flb_lib_out_cb cb_data; + struct filter_test *ctx; + + /* Create test context */ + ctx = filter_test_create((void *) &cb_data); + if (!ctx) { + exit(EXIT_FAILURE); + } + + /* Configure filter */ + ret = flb_filter_set(ctx->flb, ctx->f_ffd, + "copy", "k1 k2", + NULL); + TEST_CHECK(ret == 0); + + /* Prepare output callback with expected result */ + cb_data.cb = cb_check_result; + cb_data.data = "\"k1\":\"sample1\",\"k2\":\"sample2\""; + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + /* Ingest data samples */ + p = "[0,{\"k1\":\"sample1\",\"k2\":\"sample2\"}]"; + len = strlen(p); + bytes = flb_lib_push(ctx->flb, ctx->i_ffd, p, len); + TEST_CHECK(bytes == len); + + filter_test_destroy(ctx); +} + +/* Operation: COPY / Target key no exists, make a copy */ +static void flb_test_op_copy_no_exists() +{ + int len; + int ret; + int bytes; + char *p; + struct flb_lib_out_cb cb_data; + struct filter_test *ctx; + + /* Create test context */ + ctx = filter_test_create((void *) &cb_data); + if (!ctx) { + exit(EXIT_FAILURE); + } + + /* Configure filter */ + ret = flb_filter_set(ctx->flb, ctx->f_ffd, + "copy", "k1 k3", + NULL); + TEST_CHECK(ret == 0); + + /* Prepare output callback with expected result */ + cb_data.cb = cb_check_result; + cb_data.data = "\"k3\":\"sample1\""; + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + /* Ingest data samples */ + p = "[0,{\"k1\":\"sample1\",\"k2\":\"sample2\"}]"; + len = strlen(p); + bytes = flb_lib_push(ctx->flb, ctx->i_ffd, p, len); + TEST_CHECK(bytes == len); + + filter_test_destroy(ctx); +} + +/* Operation: HARD_COPY / if target key exists, replace value */ +static void flb_test_op_hard_copy_exists() +{ + int len; + int ret; + int bytes; + char *p; + struct flb_lib_out_cb cb_data; + struct filter_test *ctx; + + /* Create test context */ + ctx = filter_test_create((void *) &cb_data); + if (!ctx) { + exit(EXIT_FAILURE); + } + + /* Configure filter */ + ret = flb_filter_set(ctx->flb, ctx->f_ffd, + "hard_copy", "k1 k2", + NULL); + TEST_CHECK(ret == 0); + + /* Prepare output callback with expected result */ + cb_data.cb = cb_check_result; + cb_data.data = "\"k1\":\"sample1\",\"k2\":\"sample1\""; + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + /* Ingest data samples */ + p = "[0, {\"k1\":\"sample1\",\"k2\":\"sample2\"}]"; + len = strlen(p); + bytes = flb_lib_push(ctx->flb, ctx->i_ffd, p, len); + TEST_CHECK(bytes == len); + + filter_test_destroy(ctx); +} + +/* Operation: HARD_COPY / if key don't exists make a copy */ +static void flb_test_op_hard_copy_no_exists() +{ + int len; + int ret; + int bytes; + char *p; + struct flb_lib_out_cb cb_data; + struct filter_test *ctx; + + /* Create test context */ + ctx = filter_test_create((void *) &cb_data); + if (!ctx) { + exit(EXIT_FAILURE); + } + + /* Configure filter */ + ret = flb_filter_set(ctx->flb, ctx->f_ffd, + "hard_copy", "k1 k3", + NULL); + TEST_CHECK(ret == 0); + + /* Prepare output callback with expected result */ + cb_data.cb = cb_check_result; + cb_data.data = "\"k3\":\"sample1\""; + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + /* Ingest data samples */ + p = "[0,{\"k1\":\"sample1\",\"k2\":\"sample2\"}]"; + len = strlen(p); + bytes = flb_lib_push(ctx->flb, ctx->i_ffd, p, len); + TEST_CHECK(bytes == len); + + filter_test_destroy(ctx); +} + + +/* Condition: KEY_EXISTS / If key exists, make a copy */ +static void flb_test_cond_key_exists() +{ + int len; + int ret; + int bytes; + char *p; + struct flb_lib_out_cb cb_data; + struct filter_test *ctx; + + /* Create test context */ + ctx = filter_test_create((void *) &cb_data); + if (!ctx) { + exit(EXIT_FAILURE); + } + + /* Configure filter */ + ret = flb_filter_set(ctx->flb, ctx->f_ffd, + "condition", "key_exists k1", + "copy", "k1 k3_copy", + NULL); + TEST_CHECK(ret == 0); + + /* Prepare output callback with expected result */ + cb_data.cb = cb_check_result; + cb_data.data = "\"k3_copy\":\"sample1\""; + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + /* Ingest data samples */ + p = "[0,{\"k1\":\"sample1\",\"k2\":\"sample2\"}]"; + len = strlen(p); + bytes = flb_lib_push(ctx->flb, ctx->i_ffd, p, len); + TEST_CHECK(bytes == len); + + filter_test_destroy(ctx); +} + +/* Condition: KEY_EXISTS / If nested key exists, make a copy */ +static void flb_test_cond_key_exists_nest() +{ + int len; + int ret; + int bytes; + char *p; + struct flb_lib_out_cb cb_data; + struct filter_test *ctx; + + /* Create test context */ + ctx = filter_test_create((void *) &cb_data); + if (!ctx) { + exit(EXIT_FAILURE); + } + + /* Configure filter */ + ret = flb_filter_set(ctx->flb, ctx->f_ffd, + "condition", "key_exists $nest['k1']", + "add", "key found", + NULL); + TEST_CHECK(ret == 0); + + /* Prepare output callback with expected result */ + cb_data.cb = cb_check_result; + cb_data.data = "\"key\":\"found\""; + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + /* Ingest data samples */ + p = "[0,{\"k1\":\"sample1\",\"k2\":\"sample2\", \"nest\":{\"k1\":\"nest\"}}]"; + len = strlen(p); + bytes = flb_lib_push(ctx->flb, ctx->i_ffd, p, len); + TEST_CHECK(bytes == len); + + filter_test_destroy(ctx); +} + +/* Condition: KEY_DOES_NOT_EXISTS / If key does not exists, add a dummy key */ +static void flb_test_cond_key_does_not_exist() +{ + int len; + int ret; + int bytes; + char *p; + struct flb_lib_out_cb cb_data; + struct filter_test *ctx; + + /* Create test context */ + ctx = filter_test_create((void *) &cb_data); + if (!ctx) { + exit(EXIT_FAILURE); + } + + /* Configure filter */ + ret = flb_filter_set(ctx->flb, ctx->f_ffd, + "condition", "key_does_not_exist k3", + "copy", "k1 k3", + NULL); + TEST_CHECK(ret == 0); + + /* Prepare output callback with expected result */ + cb_data.cb = cb_check_result; + cb_data.data = "\"k3\":\"sample1\""; + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + /* Ingest data samples */ + p = "[0,{\"k1\":\"sample1\",\"k2\":\"sample2\"}]"; + len = strlen(p); + bytes = flb_lib_push(ctx->flb, ctx->i_ffd, p, len); + TEST_CHECK(bytes == len); + + filter_test_destroy(ctx); +} + +/* Condition: KEY_DOES_NOT_EXISTS / If key does not exists, add a dummy key */ +static void flb_test_cond_key_does_not_exist_nest() +{ + int len; + int ret; + int bytes; + char *p; + struct flb_lib_out_cb cb_data; + struct filter_test *ctx; + + /* Create test context */ + ctx = filter_test_create((void *) &cb_data); + if (!ctx) { + exit(EXIT_FAILURE); + } + + /* Configure filter */ + ret = flb_filter_set(ctx->flb, ctx->f_ffd, + "condition", "key_does_not_exist $nest['k1']", + "add", "key not_found", + NULL); + TEST_CHECK(ret == 0); + + /* Prepare output callback with expected result */ + cb_data.cb = cb_check_result; + cb_data.data = "\"key\":\"not_found\""; + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + /* Ingest data samples */ + p = "[0,{\"k1\":\"sample1\",\"k2\":\"sample2\", \"nest\":{\"k\":\"sample\"}}]"; + len = strlen(p); + bytes = flb_lib_push(ctx->flb, ctx->i_ffd, p, len); + TEST_CHECK(bytes == len); + + filter_test_destroy(ctx); +} + +/* Condition: A_KEY_MATCHES / If key matches, add a dummy key */ +static void flb_test_cond_a_key_matches() +{ + int len; + int ret; + int bytes; + char *p; + struct flb_lib_out_cb cb_data; + struct filter_test *ctx; + + /* Create test context */ + ctx = filter_test_create((void *) &cb_data); + if (!ctx) { + exit(EXIT_FAILURE); + } + + /* Configure filter */ + ret = flb_filter_set(ctx->flb, ctx->f_ffd, + "condition", "a_key_matches ^[a-z][0-9]$", + "copy", "c1 c1_matched", + NULL); + TEST_CHECK(ret == 0); + + /* Prepare output callback with expected result */ + cb_data.cb = cb_check_result; + cb_data.data = "\"c1_matched\":\"sample3\""; + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + /* Ingest data samples */ + p = "[0,{\"aa\":\"sample1\",\"bb\":\"sample2\",\"c1\":\"sample3\"}]"; + len = strlen(p); + bytes = flb_lib_push(ctx->flb, ctx->i_ffd, p, len); + TEST_CHECK(bytes == len); + + filter_test_destroy(ctx); +} + +/* Condition: NO_KEY_MATCHES / If no key matches, add a dummy key */ +static void flb_test_cond_no_key_matches() +{ + int len; + int ret; + int bytes; + char *p; + struct flb_lib_out_cb cb_data; + struct filter_test *ctx; + + /* Create test context */ + ctx = filter_test_create((void *) &cb_data); + if (!ctx) { + exit(EXIT_FAILURE); + } + + /* Configure filter */ + ret = flb_filter_set(ctx->flb, ctx->f_ffd, + "condition", "no_key_matches ^[0-9]$", + "copy", "c1 no_matches", + NULL); + TEST_CHECK(ret == 0); + + /* Prepare output callback with expected result */ + cb_data.cb = cb_check_result; + cb_data.data = "\"no_matches\":\"sample3\""; + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + /* Ingest data samples */ + p = "[0,{\"aa\":\"sample1\",\"bb\":\"sample2\",\"c1\":\"sample3\"}]"; + len = strlen(p); + bytes = flb_lib_push(ctx->flb, ctx->i_ffd, p, len); + TEST_CHECK(bytes == len); + + filter_test_destroy(ctx); +} + +/* Condition: KEY_VALUE_EQUALS / If key value matches, add a dummy key */ +static void flb_test_cond_key_value_equals() +{ + int len; + int ret; + int bytes; + char *p; + struct flb_lib_out_cb cb_data; + struct filter_test *ctx; + + /* Create test context */ + ctx = filter_test_create((void *) &cb_data); + if (!ctx) { + exit(EXIT_FAILURE); + } + + /* Configure filter */ + ret = flb_filter_set(ctx->flb, ctx->f_ffd, + "condition", "key_value_equals bb sample2", + "copy", "c1 no_matches", + NULL); + TEST_CHECK(ret == 0); + + /* Prepare output callback with expected result */ + cb_data.cb = cb_check_result; + cb_data.data = "\"no_matches\":\"sample3\""; + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + /* Ingest data samples */ + p = "[0,{\"aa\":\"sample1\",\"bb\":\"sample2\",\"c1\":\"sample3\"}]"; + len = strlen(p); + bytes = flb_lib_push(ctx->flb, ctx->i_ffd, p, len); + TEST_CHECK(bytes == len); + + filter_test_destroy(ctx); +} + +/* Condition: KEY_VALUE_EQUALS / If key value matches, add a dummy key */ +static void flb_test_cond_key_value_equals_nest() +{ + int len; + int ret; + int bytes; + char *p; + struct flb_lib_out_cb cb_data; + struct filter_test *ctx; + + /* Create test context */ + ctx = filter_test_create((void *) &cb_data); + if (!ctx) { + exit(EXIT_FAILURE); + } + + /* Configure filter */ + ret = flb_filter_set(ctx->flb, ctx->f_ffd, + "condition", "key_value_equals $nest['k1'] sample2", + "add", "key found", + NULL); + TEST_CHECK(ret == 0); + + /* Prepare output callback with expected result */ + cb_data.cb = cb_check_result; + cb_data.data = "\"key\":\"found\""; + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + /* Ingest data samples */ + p = "[0,{\"aa\":\"sample1\",\"bb\":\"sample2\",\"c1\":\"sample3\", \"nest\":{\"k1\":\"sample2\"}}]"; + len = strlen(p); + bytes = flb_lib_push(ctx->flb, ctx->i_ffd, p, len); + TEST_CHECK(bytes == len); + + filter_test_destroy(ctx); +} + +/* Condition: KEY_VALUE_DOES_NOT_EQUAL / If key value mismatch, add a key */ +static void flb_test_cond_key_value_does_not_equal() +{ + int len; + int ret; + int bytes; + char *p; + struct flb_lib_out_cb cb_data; + struct filter_test *ctx; + + /* Create test context */ + ctx = filter_test_create((void *) &cb_data); + if (!ctx) { + exit(EXIT_FAILURE); + } + + /* Configure filter */ + ret = flb_filter_set(ctx->flb, ctx->f_ffd, + "condition", "key_value_does_not_equal bb sample3", + "copy", "c1 no_matches", + NULL); + TEST_CHECK(ret == 0); + + /* Prepare output callback with expected result */ + cb_data.cb = cb_check_result; + cb_data.data = "\"no_matches\":\"sample3\""; + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + /* Ingest data samples */ + p = "[0,{\"aa\":\"sample1\",\"bb\":\"sample2\",\"c1\":\"sample3\"}]"; + len = strlen(p); + bytes = flb_lib_push(ctx->flb, ctx->i_ffd, p, len); + TEST_CHECK(bytes == len); + + filter_test_destroy(ctx); +} + +/* Condition: KEY_VALUE_DOES_NOT_EQUAL / If key value mismatch, add a key */ +static void flb_test_cond_key_value_does_not_equal_nest() +{ + int len; + int ret; + int bytes; + char *p; + struct flb_lib_out_cb cb_data; + struct filter_test *ctx; + + /* Create test context */ + ctx = filter_test_create((void *) &cb_data); + if (!ctx) { + exit(EXIT_FAILURE); + } + + /* Configure filter */ + ret = flb_filter_set(ctx->flb, ctx->f_ffd, + "condition", "key_value_does_not_equal $nest['k1'] sample2", + "add", "key not_found", + NULL); + TEST_CHECK(ret == 0); + + /* Prepare output callback with expected result */ + cb_data.cb = cb_check_result; + cb_data.data = "\"key\":\"not_found\""; + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + /* Ingest data samples */ + p = "[0,{\"aa\":\"sample1\",\"bb\":\"sample2\",\"c1\":\"sample3\", \"nest\":{\"k1\":\"sample3\"}}]"; + len = strlen(p); + bytes = flb_lib_push(ctx->flb, ctx->i_ffd, p, len); + TEST_CHECK(bytes == len); + + filter_test_destroy(ctx); +} + +/* Condition: KEY_VALUE_MATCHES / If key match, add a key */ +static void flb_test_cond_key_value_matches() +{ + int len; + int ret; + int bytes; + char *p; + struct flb_lib_out_cb cb_data; + struct filter_test *ctx; + + /* Create test context */ + ctx = filter_test_create((void *) &cb_data); + if (!ctx) { + exit(EXIT_FAILURE); + } + + /* Configure filter */ + ret = flb_filter_set(ctx->flb, ctx->f_ffd, + "condition", "key_value_matches k2 ^[a-z][0-9]$", + "copy", "k1 matches", + NULL); + TEST_CHECK(ret == 0); + + /* Prepare output callback with expected result */ + cb_data.cb = cb_check_result; + cb_data.data = "\"matches\":\"sample1\""; + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + /* Ingest data samples */ + p = "[0,{\"k1\":\"sample1\",\"k2\":\"z2\"}]"; + len = strlen(p); + bytes = flb_lib_push(ctx->flb, ctx->i_ffd, p, len); + TEST_CHECK(bytes == len); + + filter_test_destroy(ctx); +} + +/* Condition: KEY_VALUE_MATCHES / If key match, add a key */ +static void flb_test_cond_key_value_matches_nest() +{ + int len; + int ret; + int bytes; + char *p; + struct flb_lib_out_cb cb_data; + struct filter_test *ctx; + + /* Create test context */ + ctx = filter_test_create((void *) &cb_data); + if (!ctx) { + exit(EXIT_FAILURE); + } + + /* Configure filter */ + ret = flb_filter_set(ctx->flb, ctx->f_ffd, + "condition", "key_value_matches $nest['k2'] ^[a-z][0-9]$", + "add", "kv matches", + NULL); + TEST_CHECK(ret == 0); + + /* Prepare output callback with expected result */ + cb_data.cb = cb_check_result; + cb_data.data = "\"kv\":\"matches\""; + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + /* Ingest data samples */ + p = "[0,{\"k1\":\"sample1\",\"k2\":\"z2\", \"nest\":{\"k2\":\"z2\"}}]"; + len = strlen(p); + bytes = flb_lib_push(ctx->flb, ctx->i_ffd, p, len); + TEST_CHECK(bytes == len); + + filter_test_destroy(ctx); +} + +/* Condition: KEY_VALUE_DOES_NOT_MATCH / If key mismatch, add a key */ +static void flb_test_cond_key_value_does_not_match() +{ + int len; + int ret; + int bytes; + char *p; + struct flb_lib_out_cb cb_data; + struct filter_test *ctx; + + /* Create test context */ + ctx = filter_test_create((void *) &cb_data); + if (!ctx) { + exit(EXIT_FAILURE); + } + + /* Configure filter */ + ret = flb_filter_set(ctx->flb, ctx->f_ffd, + "condition", "key_value_does_not_match k2 ^[a-z][0-9]$", + "copy", "k1 no_matches", + NULL); + TEST_CHECK(ret == 0); + + /* Prepare output callback with expected result */ + cb_data.cb = cb_check_result; + cb_data.data = "\"no_matches\":\"sample1\""; + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + /* Ingest data samples */ + p = "[0,{\"k1\":\"sample1\",\"k2\":\"22\"}]"; + len = strlen(p); + bytes = flb_lib_push(ctx->flb, ctx->i_ffd, p, len); + TEST_CHECK(bytes == len); + + filter_test_destroy(ctx); +} + +/* Condition: KEY_VALUE_DOES_NOT_MATCH / If key mismatch, add a key */ +static void flb_test_cond_key_value_does_not_match_nest() +{ + int len; + int ret; + int bytes; + char *p; + struct flb_lib_out_cb cb_data; + struct filter_test *ctx; + + /* Create test context */ + ctx = filter_test_create((void *) &cb_data); + if (!ctx) { + exit(EXIT_FAILURE); + } + + /* Configure filter */ + ret = flb_filter_set(ctx->flb, ctx->f_ffd, + "condition", "key_value_does_not_match $nest['k2'] ^[a-z][0-9]$", + "add", "kv no_matches", + NULL); + TEST_CHECK(ret == 0); + + /* Prepare output callback with expected result */ + cb_data.cb = cb_check_result; + cb_data.data = "\"kv\":\"no_matches\""; + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + /* Ingest data samples */ + p = "[0,{\"k1\":\"sample1\",\"k2\":\"22\",\"nest\":{\"k2\":\"22\"}}]"; + len = strlen(p); + bytes = flb_lib_push(ctx->flb, ctx->i_ffd, p, len); + TEST_CHECK(bytes == len); + + filter_test_destroy(ctx); +} + +/* Condition: MATCHING_KEYS_HAVE_MATCHING_VALUES / If key match, add a key */ +static void flb_test_cond_matching_keys_have_matching_values() +{ + int len; + int ret; + int bytes; + char *p; + struct flb_lib_out_cb cb_data; + struct filter_test *ctx; + + /* Create test context */ + ctx = filter_test_create((void *) &cb_data); + if (!ctx) { + exit(EXIT_FAILURE); + } + + /* Configure filter */ + ret = flb_filter_set(ctx->flb, ctx->f_ffd, + "condition", + "matching_keys_have_matching_values "\ + "^[a-z][0-9]$ ^[a-z][0-9]$", + "copy", "k1 matches", + NULL); + TEST_CHECK(ret == 0); + + /* Prepare output callback with expected result */ + cb_data.cb = cb_check_result; + cb_data.data = "\"matches\":\"n1\""; + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + /* Ingest data samples */ + p = "[0,{\"k1\":\"n1\",\"k2\":\"n3\"}]"; + len = strlen(p); + bytes = flb_lib_push(ctx->flb, ctx->i_ffd, p, len); + TEST_CHECK(bytes == len); + + filter_test_destroy(ctx); +} + +/* Condition: MATCHING_KEYS_DOES_NOT_HAVE_MATCHING_VALUES */ +static void flb_test_cond_matching_keys_do_not_have_matching_values() +{ + int len; + int ret; + int bytes; + char *p; + struct flb_lib_out_cb cb_data; + struct filter_test *ctx; + + /* Create test context */ + ctx = filter_test_create((void *) &cb_data); + if (!ctx) { + exit(EXIT_FAILURE); + } + + /* Configure filter */ + ret = flb_filter_set(ctx->flb, ctx->f_ffd, + "condition", + "matching_keys_do_not_have_matching_values "\ + "^[a-z][0-9]$ ^[a-z][0-9]$", + "copy", "k1 no_matches", + NULL); + TEST_CHECK(ret == 0); + + /* Prepare output callback with expected result */ + cb_data.cb = cb_check_result; + cb_data.data = "\"no_matches\":\"aa\""; + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + /* Ingest data samples */ + p = "[0,{\"k1\":\"aa\",\"k2\":\"bb\"}]"; + len = strlen(p); + bytes = flb_lib_push(ctx->flb, ctx->i_ffd, p, len); + TEST_CHECK(bytes == len); + + filter_test_destroy(ctx); +} + +/* Test all operations */ +static void flb_test_cond_chain() +{ + int len; + int ret; + int bytes; + char *p; + struct flb_lib_out_cb cb_data; + struct filter_test *ctx; + + /* Create test context */ + ctx = filter_test_create((void *) &cb_data); + if (!ctx) { + exit(EXIT_FAILURE); + } + + /* Configure filter */ + ret = flb_filter_set(ctx->flb, ctx->f_ffd, + "condition", "key_exists k1", + "condition", "key_does_not_exist k2", + "add", "k2 sample_2", + "condition", "a_key_matches ^[a-z]1$", + "add", "k3 3", + "condition", "no_key_matches ^[0-9]$", + "condition", "key_value_equals k1 sample", + "add", "k4 4", + "condition", "key_value_does_not_equal k1 sampl", + "condition", "key_value_matches k1 ^[a-z]+$", + "condition", "key_value_does_not_match k1 aa", + + "condition", + "matching_keys_have_matching_values " \ + "^[a-z][0-9]$ ^[a-z]+$", + + "condition", + "matching_keys_do_not_have_matching_values " \ + "^[a-z][0-9]$ ^[a-z][0-9]$", + + "add", "k5 5", + NULL); + TEST_CHECK(ret == 0); + + /* Prepare output callback with expected result */ + cb_data.cb = cb_check_result; + cb_data.data = "\"k5\":\"5\""; + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + /* Ingest data samples */ + p = "[0,{\"k1\":\"sample\"}]"; + len = strlen(p); + bytes = flb_lib_push(ctx->flb, ctx->i_ffd, p, len); + TEST_CHECK(bytes == len); + + filter_test_destroy(ctx); +} + + +pthread_mutex_t result_mutex = PTHREAD_MUTEX_INITIALIZER; +int num_output = 0; + + +static void add_output_num() +{ + pthread_mutex_lock(&result_mutex); + num_output++; + pthread_mutex_unlock(&result_mutex); +} + +static void clear_output_num() +{ + pthread_mutex_lock(&result_mutex); + num_output = 0; + pthread_mutex_unlock(&result_mutex); +} + +static int get_output_num() +{ + int ret; + pthread_mutex_lock(&result_mutex); + ret = num_output; + pthread_mutex_unlock(&result_mutex); + + return ret; +} + +static int callback_count(void* data, size_t size, void* cb_data) +{ + if (size > 0) { + flb_debug("[test_filter_modify] received message: %s", (char*)data); + add_output_num(); /* success */ + flb_free(data); + } + return 0; +} + + +// to check issue https://github.com/fluent/fluent-bit/issues/1077 +static void flb_test_not_drop_multi_event() +{ + int count = 0; + int expected = 3; + + char *p; + int len; + int ret; + int bytes; + + struct filter_test *ctx; + struct flb_lib_out_cb cb_data; + + + clear_output_num(); + cb_data.cb = callback_count; + + /* Create test context */ + ctx = filter_test_create((void *) &cb_data); + if (!ctx) { + exit(EXIT_FAILURE); + } + + /* Configure filter */ + ret = flb_filter_set(ctx->flb, ctx->f_ffd, + "condition", "key_value_equals cond true", + "add", "data matched", + NULL); + TEST_CHECK(ret == 0); + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + /* Ingest multiple events */ + p = "[0, {\"cond\":\"false\", \"data\": \"something\"}]"; + len = strlen(p); + bytes = flb_lib_push(ctx->flb, ctx->i_ffd, p, len); + TEST_CHECK(bytes == len); + + p = "[0, {\"cond\":\"true\", \"data\": \"something\"}]"; + len = strlen(p); + bytes = flb_lib_push(ctx->flb, ctx->i_ffd, p, len); + TEST_CHECK(bytes == len); + + p = "[0, {\"data\": \"something\"}]"; + len = strlen(p); + bytes = flb_lib_push(ctx->flb, ctx->i_ffd, p, len); + TEST_CHECK(bytes == len); + + sleep(1); /* waiting flush */ + count = get_output_num(); + + TEST_CHECK_(count == expected, "Expected number of events %d, got %d", expected, count ); + + + filter_test_destroy(ctx); + +} + +/* to check issue https://github.com/fluent/fluent-bit/issues/4319 */ +static void flb_test_issue_4319() +{ + char *p; + int len; + int ret; + int bytes; + + struct filter_test *ctx; + struct flb_lib_out_cb cb_data; + + + clear_output_num(); + + /* Prepare output callback with expected result */ + cb_data.cb = cb_check_result; + cb_data.data = "\"ok\":\"sample\""; + + /* Create test context */ + ctx = filter_test_create((void *) &cb_data); + if (!ctx) { + exit(EXIT_FAILURE); + } + + /* Configure filter */ + ret = flb_filter_set(ctx->flb, ctx->f_ffd, + /* set key which doesn't exist */ + "condition", "key_value_does_not_equal aaa sample", + "rename", "ok error", + NULL); + TEST_CHECK(ret == 0); + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + /* Ingest event */ + p = "[0, {\"ok\":\"sample\"}]"; + len = strlen(p); + bytes = flb_lib_push(ctx->flb, ctx->i_ffd, p, len); + TEST_CHECK(bytes == len); + + sleep(1); /* waiting flush */ + + filter_test_destroy(ctx); +} + + +/* + * to check issue https://github.com/fluent/fluent-bit/issues/4319 + Key_value_does_not_match case +*/ +static void flb_test_issue_4319_2() +{ + char *p; + int len; + int ret; + int bytes; + + struct filter_test *ctx; + struct flb_lib_out_cb cb_data; + + + clear_output_num(); + + /* Prepare output callback with expected result */ + cb_data.cb = cb_check_result; + cb_data.data = "\"ok\":\"sample\""; + + /* Create test context */ + ctx = filter_test_create((void *) &cb_data); + if (!ctx) { + exit(EXIT_FAILURE); + } + + /* Configure filter */ + ret = flb_filter_set(ctx->flb, ctx->f_ffd, + /* set key which doesn't exist */ + "condition", "key_value_does_not_match aaa sample", + "rename", "ok error", + NULL); + TEST_CHECK(ret == 0); + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + /* Ingest event */ + p = "[0, {\"ok\":\"sample\"}]"; + len = strlen(p); + bytes = flb_lib_push(ctx->flb, ctx->i_ffd, p, len); + TEST_CHECK(bytes == len); + + sleep(1); /* waiting flush */ + + filter_test_destroy(ctx); +} + +/* https://github.com/fluent/fluent-bit/issues/1225 */ +static void flb_test_issue_1225() +{ + int len; + int ret; + int bytes; + char *p; + struct flb_lib_out_cb cb_data; + struct filter_test *ctx; + + /* Create test context */ + ctx = filter_test_create((void *) &cb_data); + if (!ctx) { + exit(EXIT_FAILURE); + } + + /* Configure filter */ + ret = flb_filter_set(ctx->flb, ctx->f_ffd, + "condition", "key_value_matches \"key 1\" \".*with spaces.*\"", + "add", "\"key 2\" \"second value with spaces\"", + NULL); + TEST_CHECK(ret == 0); + + /* Prepare output callback with expected result */ + cb_data.cb = cb_check_result; + cb_data.data = "\"key 1\":\"first value with spaces\","\ + "\"key 2\":\"second value with spaces\""; + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + /* Ingest data samples */ + p = "[0,{\"key 1\":\"first value with spaces\"}]"; + len = strlen(p); + bytes = flb_lib_push(ctx->flb, ctx->i_ffd, p, len); + TEST_CHECK(bytes == len); + + filter_test_destroy(ctx); +} + + +/* + * to check issue https://github.com/fluent/fluent-bit/issues/7075 +*/ +static void flb_test_issue_7075() +{ + char *p; + int len; + int ret; + int bytes; + + struct filter_test *ctx; + struct flb_lib_out_cb cb_data; + + clear_output_num(); + + /* Prepare output callback with expected result */ + cb_data.cb = cb_check_result; + cb_data.data = "\"matched\":true"; + + /* Create test context */ + ctx = filter_test_create((void *) &cb_data); + if (!ctx) { + exit(EXIT_FAILURE); + } + + /* Configure filter */ + ret = flb_filter_set(ctx->flb, ctx->f_ffd, + /* set key which doesn't exist */ + "condition", "key_value_matches ok true", + "rename", "ok matched", + NULL); + TEST_CHECK(ret == 0); + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + /* Ingest event */ + p = "[0, {\"ok\":true}]"; + len = strlen(p); + bytes = flb_lib_push(ctx->flb, ctx->i_ffd, p, len); + TEST_CHECK(bytes == len); + + sleep(1); /* waiting flush */ + + filter_test_destroy(ctx); +} + +static void flb_test_issue_7368() +{ + int ret; + struct flb_lib_out_cb cb_data; + struct filter_test *ctx; + + /* Create test context */ + ctx = filter_test_create((void *) &cb_data); + if (!ctx) { + exit(EXIT_FAILURE); + } + + /* Configure filter */ + ret = flb_filter_set(ctx->flb, ctx->f_ffd, + "remove_wildcard", "*s3", + NULL); + TEST_CHECK(ret == 0); + + /* Prepare output callback with expected result */ + cb_data.cb = cb_check_result; + cb_data.data = "\"r1\":\"someval\""; + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret != 0); + + filter_test_destroy(ctx); +} + +TEST_LIST = { + /* Operations / Commands */ + {"op_set_append" , flb_test_op_set_append }, + {"op_set_replace" , flb_test_op_set_replace }, + {"op_remove" , flb_test_op_remove }, + {"op_remove_wildcard" , flb_test_op_remove_wildcard }, + {"op_remove_regex" , flb_test_op_remove_regex }, + {"op_rename_exists" , flb_test_op_rename_exists }, + {"op_rename_no_exists" , flb_test_op_rename_no_exists }, + {"op_hard_rename_exists" , flb_test_op_hard_rename_exists }, + {"op_hard_rename_no_exists" , flb_test_op_hard_rename_no_exists }, + {"op_copy_exists" , flb_test_op_copy_exists }, + {"op_copy_no_exists" , flb_test_op_copy_no_exists }, + {"op_hard_copy_exists" , flb_test_op_hard_copy_exists }, + {"op_hard_copy_no_exists" , flb_test_op_hard_copy_no_exists }, + + /* Conditions (nested) */ + {"cond_key_value_matches_nest", flb_test_cond_key_value_matches_nest }, + {"cond_key_value_does_not_match_nest", flb_test_cond_key_value_does_not_match_nest }, + {"cond_key_exists_nest", flb_test_cond_key_exists_nest }, + {"cond_key_does_not_exist_nest", flb_test_cond_key_does_not_exist_nest }, + {"cond_key_value_equals_nest", flb_test_cond_key_value_equals_nest }, + {"cond_key_value_does_not_equal_nest", flb_test_cond_key_value_does_not_equal_nest }, + + /* Conditions */ + {"cond_key_exists", flb_test_cond_key_exists }, + {"cond_key_does_not_exist", flb_test_cond_key_does_not_exist }, + {"cond_a_key_matches", flb_test_cond_a_key_matches }, + {"cond_no_key_matches", flb_test_cond_no_key_matches }, + {"cond_key_value_equals", flb_test_cond_key_value_equals }, + {"cond_key_value_does_not_equal", flb_test_cond_key_value_does_not_equal }, + {"cond_key_value_matches", flb_test_cond_key_value_matches }, + {"cond_key_value_does_not_match", flb_test_cond_key_value_does_not_match }, + {"cond_matching_keys_have_matching_values", + flb_test_cond_matching_keys_have_matching_values }, + {"cond_matching_keys_do_not_have_matching_values", + flb_test_cond_matching_keys_do_not_have_matching_values }, + {"cond_chain", flb_test_cond_chain }, + + /* Bug fixes */ + {"multiple events are not dropped", flb_test_not_drop_multi_event }, + {"cond_key_value_does_not_equal and key does not exist", flb_test_issue_4319 }, + {"cond_key_value_does_not_matches and key does not exist", flb_test_issue_4319_2 }, + {"Key_value_matches and value is bool type", flb_test_issue_7075}, + {"operation_with_whitespace", flb_test_issue_1225 }, + {"invalid_wildcard", flb_test_issue_7368 }, + + {NULL, NULL} +}; diff --git a/src/fluent-bit/tests/runtime/filter_multiline.c b/src/fluent-bit/tests/runtime/filter_multiline.c new file mode 100644 index 000000000..18253a5b2 --- /dev/null +++ b/src/fluent-bit/tests/runtime/filter_multiline.c @@ -0,0 +1,699 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +#include <fluent-bit.h> +#include <fluent-bit/flb_sds.h> +#include "flb_tests_runtime.h" + +struct filter_test { + flb_ctx_t *flb; /* Fluent Bit library context */ + int i_ffd; /* Input fd */ + int f_ffd; /* Filter fd */ +}; + +struct filter_test_result { + char *expected_pattern; /* string that must occur in output */ + int expected_pattern_index; /* which record to check for the pattern */ + int expected_records; /* expected number of outputted records */ + int actual_records; /* actual number of outputted records */ +}; + +/* Callback to check expected results */ +static int cb_check_result(void *record, size_t size, void *data) +{ + char *p; + struct filter_test_result *expected; + char *result; + + expected = (struct filter_test_result *) data; + result = (char *) record; + + if (expected->expected_pattern_index == expected->actual_records) { + p = strstr(result, expected->expected_pattern); + TEST_CHECK(p != NULL); + + if (!p) { + flb_error("Expected to find: '%s' in result '%s'", + expected->expected_pattern, result); + } + /* + * If you want to debug your test + * + * printf("Expect: '%s' in result '%s'\n", expected->expected_pattern, result); + */ + } + + expected->actual_records++; + + flb_free(record); + return 0; +} + + +pthread_mutex_t result_mutex = PTHREAD_MUTEX_INITIALIZER; +int num_output = 0; +static int get_output_num() +{ + int ret; + pthread_mutex_lock(&result_mutex); + ret = num_output; + pthread_mutex_unlock(&result_mutex); + + return ret; +} + +static void set_output_num(int num) +{ + pthread_mutex_lock(&result_mutex); + num_output = num; + pthread_mutex_unlock(&result_mutex); +} + +static void clear_output_num() +{ + set_output_num(0); +} + +struct str_list { + size_t size; /* size of lists */ + int ignore_min_line_num; /* ignore line if the length is less than this value */ + char **lists; /* string lists */ +}; + +/* Callback to check expected results */ +static int cb_check_str_list(void *record, size_t size, void *data) +{ + char *p; + char *out_line = record; + int num = get_output_num(); + int count = 0; + size_t i; + struct str_list *l = (struct str_list *)data; + + if (!TEST_CHECK(out_line != NULL)) { + TEST_MSG("out_line is NULL"); + return -1; + } + + if (!TEST_CHECK(l != NULL)) { + TEST_MSG("l is NULL"); + flb_free(out_line); + return -1; + } + + if (strlen(out_line) < l->ignore_min_line_num) { + flb_free(out_line); + return 0; + } + + for (i=0; i<l->size; i++) { + p = strstr(out_line, l->lists[i]); + if (p != NULL) { + count++; + } + } + if(!TEST_CHECK(count != 0)) { + TEST_MSG("%s is not matched", out_line); + } + set_output_num(num+count); + + flb_free(out_line); + return 0; +} + + + +static struct filter_test *filter_test_create(struct flb_lib_out_cb *data) +{ + int i_ffd; + int f_ffd; + int o_ffd; + struct filter_test *ctx; + + ctx = flb_malloc(sizeof(struct filter_test)); + if (!ctx) { + flb_errno(); + return NULL; + } + + /* Service config */ + ctx->flb = flb_create(); + flb_service_set(ctx->flb, + "Flush", "0.200000000", + "Grace", "1", + NULL); + + /* Input */ + i_ffd = flb_input(ctx->flb, (char *) "lib", NULL); + TEST_CHECK(i_ffd >= 0); + flb_input_set(ctx->flb, i_ffd, "tag", "test", NULL); + ctx->i_ffd = i_ffd; + + /* Filter configuration */ + f_ffd = flb_filter(ctx->flb, (char *) "multiline", NULL); + TEST_CHECK(f_ffd >= 0); + flb_filter_set(ctx->flb, f_ffd, "match", "*", NULL); + ctx->f_ffd = f_ffd; + + /* Output */ + o_ffd = flb_output(ctx->flb, (char *) "lib", (void *) data); + TEST_CHECK(o_ffd >= 0); + flb_output_set(ctx->flb, o_ffd, + "match", "test*", + "format", "json", + NULL); + + return ctx; +} + +static void filter_test_destroy(struct filter_test *ctx) +{ + flb_stop(ctx->flb); + flb_destroy(ctx->flb); + flb_free(ctx); +} + +static void flb_test_multiline_buffered_two_output_record() +{ + int len; + int ret; + int bytes; + char *p; + struct flb_lib_out_cb cb_data; + struct filter_test *ctx; + struct filter_test_result expected = { 0 }; + + /* Create test context */ + ctx = filter_test_create((void *) &cb_data); + if (!ctx) { + exit(EXIT_FAILURE); + } + + /* Configure filter */ + ret = flb_filter_set(ctx->flb, ctx->f_ffd, + "multiline.key_content", "log", + "multiline.parser", "go", + "buffer", "on", + "flush_ms", "1500", + "debug_flush", "on", + NULL); + TEST_CHECK(ret == 0); + + /* Prepare output callback with expected result */ + expected.expected_records = 1; /* 1 record with all lines concatenated */ + expected.expected_pattern = "main.main.func1(0xc420024120)"; + expected.expected_pattern_index = 0; + cb_data.cb = cb_check_result; + cb_data.data = (void *) &expected; + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + /* Ingest data samples */ + p = "[0, {\"log\":\"panic: my panic\"}]"; + len = strlen(p); + bytes = flb_lib_push(ctx->flb, ctx->i_ffd, p, len); + TEST_CHECK(bytes == len); + p = "[0, {\"log\":\"\n\"}]"; + len = strlen(p); + bytes = flb_lib_push(ctx->flb, ctx->i_ffd, p, len); + TEST_CHECK(bytes == len); + p = "[0, {\"log\":\"goroutine 4 [running]:\"}]"; + len = strlen(p); + bytes = flb_lib_push(ctx->flb, ctx->i_ffd, p, len); + TEST_CHECK(bytes == len); + p = "[0, {\"log\":\"panic(0x45cb40, 0x47ad70)\"}]"; + len = strlen(p); + bytes = flb_lib_push(ctx->flb, ctx->i_ffd, p, len); + TEST_CHECK(bytes == len); + sleep(1); /* ensure records get sent separately */ + p = "[0, {\"log\":\" /usr/local/go/src/runtime/panic.go:542 +0x46c fp=0xc42003f7b8 sp=0xc42003f710 pc=0x422f7c\"}]"; + len = strlen(p); + bytes = flb_lib_push(ctx->flb, ctx->i_ffd, p, len); + TEST_CHECK(bytes == len); + p = "[0, {\"log\":\"main.main.func1(0xc420024120)\"}]"; + len = strlen(p); + bytes = flb_lib_push(ctx->flb, ctx->i_ffd, p, len); + TEST_CHECK(bytes == len); + + /* check number of outputted records */ + sleep(3); + TEST_CHECK(expected.actual_records == expected.expected_records); + filter_test_destroy(ctx); +} + +static void flb_test_multiline_buffered_one_output_record() +{ + int len; + int ret; + int bytes; + char *p; + struct flb_lib_out_cb cb_data; + struct filter_test *ctx; + struct filter_test_result expected = { 0 }; + + /* Create test context */ + ctx = filter_test_create((void *) &cb_data); + if (!ctx) { + exit(EXIT_FAILURE); + } + + /* Configure filter */ + ret = flb_filter_set(ctx->flb, ctx->f_ffd, + "multiline.key_content", "log", + "multiline.parser", "go", + "buffer", "on", + "debug_flush", "on", + NULL); + TEST_CHECK(ret == 0); + + /* Prepare output callback with expected result */ + expected.expected_records = 2; /* 1 record with all lines concatenated */ + expected.expected_pattern = "main.main.func1(0xc420024120)"; + cb_data.cb = cb_check_result; + cb_data.data = (void *) &expected; + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + /* Ingest data samples */ + p = "[0, {\"log\":\"panic: my panic\"}]"; + len = strlen(p); + bytes = flb_lib_push(ctx->flb, ctx->i_ffd, p, len); + TEST_CHECK(bytes == len); + p = "[0, {\"log\":\"\n\"}]"; + len = strlen(p); + bytes = flb_lib_push(ctx->flb, ctx->i_ffd, p, len); + TEST_CHECK(bytes == len); + p = "[0, {\"log\":\"goroutine 4 [running]:\"}]"; + len = strlen(p); + bytes = flb_lib_push(ctx->flb, ctx->i_ffd, p, len); + TEST_CHECK(bytes == len); + p = "[0, {\"log\":\"panic(0x45cb40, 0x47ad70)\"}]"; + len = strlen(p); + bytes = flb_lib_push(ctx->flb, ctx->i_ffd, p, len); + TEST_CHECK(bytes == len); + sleep(1); /* ensure records get sent separately */ + p = "[0, {\"log\":\" /usr/local/go/src/runtime/panic.go:542 +0x46c fp=0xc42003f7b8 sp=0xc42003f710 pc=0x422f7c\"}]"; + len = strlen(p); + bytes = flb_lib_push(ctx->flb, ctx->i_ffd, p, len); + TEST_CHECK(bytes == len); + p = "[0, {\"log\":\"main.main.func1(0xc420024120)\"}]"; + len = strlen(p); + bytes = flb_lib_push(ctx->flb, ctx->i_ffd, p, len); + TEST_CHECK(bytes == len); + p = "[0, {\"log\":\"one more line, no multiline\"}]"; + len = strlen(p); + bytes = flb_lib_push(ctx->flb, ctx->i_ffd, p, len); + TEST_CHECK(bytes == len); + + /* check number of outputted records */ + sleep(2); + TEST_CHECK(expected.actual_records == expected.expected_records); + filter_test_destroy(ctx); +} + +static void flb_test_multiline_unbuffered() +{ + int len; + int ret; + int bytes; + char *p; + struct flb_lib_out_cb cb_data; + struct filter_test *ctx; + struct filter_test_result expected = { 0 }; + + /* Create test context */ + ctx = filter_test_create((void *) &cb_data); + if (!ctx) { + exit(EXIT_FAILURE); + } + + /* Configure filter */ + ret = flb_filter_set(ctx->flb, ctx->f_ffd, + "multiline.key_content", "log", + "multiline.parser", "go", + "buffer", "off", + "debug_flush", "on", + NULL); + TEST_CHECK(ret == 0); + + /* Prepare output callback with expected result */ + expected.expected_records = 6; /* no concatenation */ + expected.expected_pattern = "panic"; + expected.expected_pattern_index = 0; + cb_data.cb = cb_check_result; + cb_data.data = (void *) &expected; + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + /* Ingest data samples */ + p = "[0, {\"log\":\"panic: my panic\"}]"; + len = strlen(p); + bytes = flb_lib_push(ctx->flb, ctx->i_ffd, p, len); + TEST_CHECK(bytes == len); + sleep(1); /* ensure records get sent one by one */ + p = "[0, {\"log\":\"\n\"}]"; + len = strlen(p); + bytes = flb_lib_push(ctx->flb, ctx->i_ffd, p, len); + TEST_CHECK(bytes == len); + sleep(1); /* ensure records get sent one by one */ + p = "[0, {\"log\":\"goroutine 4 [running]:\"}]"; + len = strlen(p); + bytes = flb_lib_push(ctx->flb, ctx->i_ffd, p, len); + TEST_CHECK(bytes == len); + sleep(1); /* ensure records get sent one by one */ + p = "[0, {\"log\":\"panic(0x45cb40, 0x47ad70)\"}]"; + len = strlen(p); + bytes = flb_lib_push(ctx->flb, ctx->i_ffd, p, len); + TEST_CHECK(bytes == len); + sleep(1); /* ensure records get sent one by one */ + p = "[0, {\"log\":\" /usr/local/go/src/runtime/panic.go:542 +0x46c fp=0xc42003f7b8 sp=0xc42003f710 pc=0x422f7c\"}]"; + len = strlen(p); + bytes = flb_lib_push(ctx->flb, ctx->i_ffd, p, len); + TEST_CHECK(bytes == len); + sleep(1); /* ensure records get sent one by one */ + p = "[0, {\"log\":\"main.main.func1(0xc420024120)\"}]"; + len = strlen(p); + bytes = flb_lib_push(ctx->flb, ctx->i_ffd, p, len); + TEST_CHECK(bytes == len); + + /* check number of outputted records */ + sleep(2); + TEST_CHECK(expected.actual_records == expected.expected_records); + filter_test_destroy(ctx); +} + +static void flb_test_multiline_partial_message_concat() +{ + int len; + int ret; + int bytes; + char *p; + struct flb_lib_out_cb cb_data; + struct filter_test *ctx; + struct filter_test_result expected = { 0 }; + + /* Create test context */ + ctx = filter_test_create((void *) &cb_data); + if (!ctx) { + exit(EXIT_FAILURE); + } + + /* Configure filter */ + ret = flb_filter_set(ctx->flb, ctx->f_ffd, + "multiline.key_content", "log", + "mode", "partial_message", + "buffer", "on", + "debug_flush", "on", + "flush_ms", "666", + NULL); + TEST_CHECK(ret == 0); + + /* Prepare output callback with expected result */ + expected.expected_records = 1; /* 1 record with all lines concatenated */ + expected.expected_pattern = "one..two"; + cb_data.cb = cb_check_result; + cb_data.data = (void *) &expected; + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + /* Ingest data samples */ + p = "[0, {\"log\":\"one..\", \"partial_message\":\"true\", \"partial_id\": \"1\", \"partial_ordinal\": \"1\", \"partial_last\": \"false\"}]"; + len = strlen(p); + bytes = flb_lib_push(ctx->flb, ctx->i_ffd, p, len); + TEST_CHECK(bytes == len); + p = "[0, {\"log\":\"two..\", \"partial_message\":\"true\", \"partial_id\": \"1\", \"partial_ordinal\": \"2\", \"partial_last\": \"false\"}]"; + len = strlen(p); + bytes = flb_lib_push(ctx->flb, ctx->i_ffd, p, len); + TEST_CHECK(bytes == len); + + /* check number of outputted records */ + sleep(2); + TEST_CHECK(expected.actual_records == expected.expected_records); + filter_test_destroy(ctx); +} + +static void flb_test_multiline_partial_message_concat_two_ids() +{ + int len; + int ret; + int bytes; + char *p; + struct flb_lib_out_cb cb_data; + struct filter_test *ctx; + struct filter_test_result expected = { 0 }; + + /* Create test context */ + ctx = filter_test_create((void *) &cb_data); + if (!ctx) { + exit(EXIT_FAILURE); + } + + /* Configure filter */ + ret = flb_filter_set(ctx->flb, ctx->f_ffd, + "multiline.key_content", "log", + "mode", "partial_message", + "buffer", "on", + "debug_flush", "on", + "flush_ms", "666", + NULL); + TEST_CHECK(ret == 0); + + /* Prepare output callback with expected result */ + expected.expected_records = 2; /* 2 records, one for each partial_id*/ + expected.expected_pattern = "one..two"; + cb_data.cb = cb_check_result; + cb_data.data = (void *) &expected; + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + /* two different partial IDs, interlaced */ + p = "[0, {\"log\":\"one..\", \"partial_message\":\"true\", \"partial_id\": \"1\", \"partial_ordinal\": \"1\", \"partial_last\": \"false\"}]"; + len = strlen(p); + bytes = flb_lib_push(ctx->flb, ctx->i_ffd, p, len); + TEST_CHECK(bytes == len); + p = "[0, {\"log\":\"three..\", \"partial_message\":\"true\", \"partial_id\": \"2\", \"partial_ordinal\": \"1\", \"partial_last\": \"false\"}]"; + len = strlen(p); + bytes = flb_lib_push(ctx->flb, ctx->i_ffd, p, len); + TEST_CHECK(bytes == len); + p = "[0, {\"log\":\"two..\", \"partial_message\":\"true\", \"partial_id\": \"1\", \"partial_ordinal\": \"2\", \"partial_last\": \"true\"}]"; + len = strlen(p); + bytes = flb_lib_push(ctx->flb, ctx->i_ffd, p, len); + TEST_CHECK(bytes == len); + p = "[0, {\"log\":\"two..\", \"partial_message\":\"true\", \"partial_id\": \"2\", \"partial_ordinal\": \"2\", \"partial_last\": \"true\"}]"; + len = strlen(p); + bytes = flb_lib_push(ctx->flb, ctx->i_ffd, p, len); + TEST_CHECK(bytes == len); + + /* check number of outputted records */ + sleep(2); + TEST_CHECK(expected.actual_records == expected.expected_records); + filter_test_destroy(ctx); +} + + +/* + * create 2 in_lib instances and pass multiline + * https://github.com/fluent/fluent-bit/issues/5524 +*/ +static void flb_test_ml_buffered_two_streams() +{ + struct flb_lib_out_cb cb_data; + struct filter_test *ctx; + int i_ffd_2; + int ret; + int i; + int bytes; + int len; + char line_buf[2048] = {0}; + int line_num; + int num; + + char *expected_strs[] = {"Exception in thread main java.lang.IllegalStateException: ..null property\\n at com.example.myproject.Author.getBookIds(xx.java:38)\\n at com.example.myproject.Bootstrap.main(Bootstrap.java:14)\\nCaused by: java.lang.NullPointerException\\n at com.example.myproject.Book.getId(Book.java:22)\\n at com.example.myproject.Author.getBookIds(Author.java:35)\\n ... 1 more", + "Dec 14 06:41:08 Exception in thread main java.lang.RuntimeException: Something has gone wrong, aborting!\\n at com.myproject.module.MyProject.badMethod(MyProject.java:22)\\n at com.myproject.module.MyProject.oneMoreMethod(MyProject.java:18)\\n at com.myproject.module.MyProject.anotherMethod(MyProject.java:14)\\n at com.myproject.module.MyProject.someMethod(MyProject.java:10)\\n at com.myproject.module.MyProject.main(MyProject.java:6)"}; + struct str_list expected = { + .size = sizeof(expected_strs)/sizeof(char*), + .lists = &expected_strs[0], + .ignore_min_line_num = 64, + }; + + char *ml_logs_1[] = {"Exception in thread main java.lang.IllegalStateException: ..null property", + " at com.example.myproject.Author.getBookIds(xx.java:38)", + " at com.example.myproject.Bootstrap.main(Bootstrap.java:14)", + "Caused by: java.lang.NullPointerException", + " at com.example.myproject.Book.getId(Book.java:22)", + " at com.example.myproject.Author.getBookIds(Author.java:35)", + " ... 1 more", + "single line"}; + char *ml_logs_2[] = { + "single line...", + "Dec 14 06:41:08 Exception in thread main java.lang.RuntimeException: Something has gone wrong, aborting!", + " at com.myproject.module.MyProject.badMethod(MyProject.java:22)", + " at com.myproject.module.MyProject.oneMoreMethod(MyProject.java:18)", + " at com.myproject.module.MyProject.anotherMethod(MyProject.java:14)", + " at com.myproject.module.MyProject.someMethod(MyProject.java:10)", + " at com.myproject.module.MyProject.main(MyProject.java:6)", + "another line..."}; + + cb_data.cb = cb_check_str_list; + cb_data.data = (void *)&expected; + + clear_output_num(); + + TEST_CHECK(sizeof(ml_logs_1)/sizeof(char*) == sizeof(ml_logs_2)/sizeof(char*)); + line_num = sizeof(ml_logs_1)/sizeof(char*); + + /* Create test context */ + ctx = filter_test_create((void *) &cb_data); + if (!ctx) { + exit(EXIT_FAILURE); + } + i_ffd_2 = flb_input(ctx->flb, (char *) "lib", NULL); + TEST_CHECK(i_ffd_2 >= 0); + flb_input_set(ctx->flb, i_ffd_2, "tag", "test2", NULL); + + /* Configure filter */ + ret = flb_filter_set(ctx->flb, ctx->f_ffd, + "multiline.key_content", "log", + "multiline.parser", "java", + "buffer", "on", + "debug_flush", "on", + NULL); + TEST_CHECK(ret == 0); + + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + for (i=0; i<line_num; i++) { + sprintf(&line_buf[0], "[%d, {\"log\":\"%s\"}]", i, ml_logs_1[i]); + len = strlen(line_buf); + bytes = flb_lib_push(ctx->flb, ctx->i_ffd, &line_buf[0], len); + TEST_CHECK(bytes == len); + + + sprintf(&line_buf[0], "[%d, {\"log\":\"%s\"}]", i, ml_logs_2[i]); + len = strlen(line_buf); + bytes = flb_lib_push(ctx->flb, i_ffd_2, &line_buf[0], len); + TEST_CHECK(bytes == len); + } + sleep(3); + + num = get_output_num(); + if (!TEST_CHECK(num == 2)) { + TEST_MSG("output error. got %d expect 2", num); + } + + filter_test_destroy(ctx); +} + +static void flb_test_ml_buffered_16_streams() +{ + struct flb_lib_out_cb cb_data; + struct filter_test *ctx; + int i_ffds[16] = {0}; + int ffd_num = sizeof(i_ffds)/sizeof(int); + int ret; + int i; + int j; + int bytes; + int len; + char line_buf[2048] = {0}; + char tag_buf[32] = {0}; + int line_num; + int num; + + char *expected_strs[] = {"Exception in thread main java.lang.IllegalStateException: ..null property\\n at com.example.myproject.Author.getBookIds(xx.java:38)\\n at com.example.myproject.Bootstrap.main(Bootstrap.java:14)\\nCaused by: java.lang.NullPointerException\\n at com.example.myproject.Book.getId(Book.java:22)\\n at com.example.myproject.Author.getBookIds(Author.java:35)\\n ... 1 more"}; + + struct str_list expected = { + .size = sizeof(expected_strs)/sizeof(char*), + .lists = &expected_strs[0], + .ignore_min_line_num = 64, + }; + + char *ml_logs[] = {"Exception in thread main java.lang.IllegalStateException: ..null property", + " at com.example.myproject.Author.getBookIds(xx.java:38)", + " at com.example.myproject.Bootstrap.main(Bootstrap.java:14)", + "Caused by: java.lang.NullPointerException", + " at com.example.myproject.Book.getId(Book.java:22)", + " at com.example.myproject.Author.getBookIds(Author.java:35)", + " ... 1 more", + "single line"}; + + cb_data.cb = cb_check_str_list; + cb_data.data = (void *)&expected; + + clear_output_num(); + + line_num = sizeof(ml_logs)/sizeof(char*); + + /* Create test context */ + ctx = filter_test_create((void *) &cb_data); + if (!ctx) { + exit(EXIT_FAILURE); + } + + i_ffds[0] = ctx->i_ffd; + for (i=1; i<ffd_num; i++) { + i_ffds[i] = flb_input(ctx->flb, (char *) "lib", NULL); + TEST_CHECK(i_ffds[i] >= 0); + sprintf(&tag_buf[0], "test%d", i); + flb_input_set(ctx->flb, i_ffds[i], "tag", tag_buf, NULL); + } + + /* Configure filter */ + ret = flb_filter_set(ctx->flb, ctx->f_ffd, + "multiline.key_content", "log", + "multiline.parser", "java", + "buffer", "on", + "debug_flush", "on", + NULL); + TEST_CHECK(ret == 0); + + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + for (i=0; i<line_num; i++) { + sprintf(&line_buf[0], "[%d, {\"log\":\"%s\"}]", i, ml_logs[i]); + len = strlen(line_buf); + for (j=0; j<ffd_num; j++) { + bytes = flb_lib_push(ctx->flb, i_ffds[j], &line_buf[0], len); + TEST_CHECK(bytes == len); + } + } + sleep(3); + + num = get_output_num(); + if (!TEST_CHECK(num == ffd_num)) { + TEST_MSG("output error. got %d expect %d", num, ffd_num); + } + + filter_test_destroy(ctx); +} + + + + +TEST_LIST = { + {"ml_buffered_two_streams" , flb_test_ml_buffered_two_streams}, + {"ml_buffered_16_streams" , flb_test_ml_buffered_16_streams}, + + {"multiline_buffered_one_record" , flb_test_multiline_buffered_one_output_record }, + {"multiline_buffered_two_record" , flb_test_multiline_buffered_two_output_record }, + {"flb_test_multiline_unbuffered" , flb_test_multiline_unbuffered }, + + {"flb_test_multiline_partial_message_concat" , flb_test_multiline_partial_message_concat }, + {"flb_test_multiline_partial_message_concat_two_ids" , flb_test_multiline_partial_message_concat_two_ids }, + {NULL, NULL} +}; diff --git a/src/fluent-bit/tests/runtime/filter_nest.c b/src/fluent-bit/tests/runtime/filter_nest.c new file mode 100644 index 000000000..36e6a3fe7 --- /dev/null +++ b/src/fluent-bit/tests/runtime/filter_nest.c @@ -0,0 +1,302 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +#include <fluent-bit.h> +#include <fluent-bit/flb_time.h> +#include "flb_tests_runtime.h" + +pthread_mutex_t result_mutex = PTHREAD_MUTEX_INITIALIZER; +char *output = NULL; +int num_output = 0; + +/* Test data */ + +/* Test functions */ +void flb_test_filter_nest_single(void); +void flb_test_filter_nest_multi_nest(void); +void flb_test_filter_nest_multi_lift(void); +/* Test list */ +TEST_LIST = { + {"single", flb_test_filter_nest_single }, + {"multiple events are not dropped(nest)", flb_test_filter_nest_multi_nest}, + {"multiple events are not dropped(lift)", flb_test_filter_nest_multi_lift}, + {NULL, NULL} +}; + + +void add_output_num() +{ + pthread_mutex_lock(&result_mutex); + num_output++; + pthread_mutex_unlock(&result_mutex); +} + +void clear_output_num() +{ + pthread_mutex_lock(&result_mutex); + num_output = 0; + pthread_mutex_unlock(&result_mutex); +} + +int get_output_num() +{ + int ret; + pthread_mutex_lock(&result_mutex); + ret = num_output; + pthread_mutex_unlock(&result_mutex); + + return ret; +} + +int callback_count(void* data, size_t size, void* cb_data) +{ + if (size > 0) { + flb_debug("[test_filter_nest] received message: %s", (char*)data); + add_output_num(); /* success */ + flb_free(data); + } + return 0; +} + +void set_output(char *val) +{ + pthread_mutex_lock(&result_mutex); + output = val; + pthread_mutex_unlock(&result_mutex); +} + +char *get_output(void) +{ + char *val; + + pthread_mutex_lock(&result_mutex); + val = output; + pthread_mutex_unlock(&result_mutex); + + return val; +} + +int callback_test(void* data, size_t size, void* cb_data) +{ + if (size > 0) { + flb_debug("[test_filter_nest] received message: %s", (char*)data); + set_output(data); /* success */ + } + return 0; +} + +void flb_test_filter_nest_multi_nest(void) +{ + int ret; + int bytes; + char *p; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + int filter_ffd; + struct flb_lib_out_cb cb_data; + + int count = 0; // should be number of events. + int expected = 2; + + clear_output_num(); + + ctx = flb_create(); + flb_service_set(ctx, + "Flush", "1", + "Grace", "1", + "Log_Level", "debug", + NULL); + + /* Input */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + /* Prepare output callback context*/ + cb_data.cb = callback_count; + cb_data.data = NULL; + + /* Filter */ + filter_ffd = flb_filter(ctx, (char *) "nest", NULL); + TEST_CHECK(filter_ffd >= 0); + + ret = flb_filter_set(ctx, filter_ffd, + "Match", "*", + "Operation", "nest", + "Wildcard", "to_nest", + "Nest_under", "nested_key", + NULL); + + TEST_CHECK(ret == 0); + + + /* Output */ + out_ffd = flb_output(ctx, (char *) "lib", (void *) &cb_data); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, + "match", "test", + "format", "json", + NULL); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + p = "[1448403340, {\"to_nest\":\"This is the data to nest\", \"extra\":\"Some more data\"}]"; + bytes = flb_lib_push(ctx, in_ffd, p, strlen(p)); + TEST_CHECK(bytes == strlen(p)); + + p = "[1448403341, {\"not_nest\":\"dummy data\", \"extra\":\"dummy more data\"}]"; + bytes = flb_lib_push(ctx, in_ffd, p, strlen(p)); + TEST_CHECK(bytes == strlen(p)); + + flb_time_msleep(1500); /* waiting flush */ + count = get_output_num(); + + TEST_CHECK_(count == expected, "Expected number of events %d, got %d", expected, count ); + + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_filter_nest_multi_lift(void) +{ + int ret; + int bytes; + char *p; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + int filter_ffd; + struct flb_lib_out_cb cb_data; + + int count = 0; // should be number of events. + int expected = 2; + + clear_output_num(); + + ctx = flb_create(); + flb_service_set(ctx, + "Flush", "1", + "Grace", "1", + "Log_Level", "debug", + NULL); + + /* Input */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + /* Prepare output callback context*/ + cb_data.cb = callback_count; + cb_data.data = NULL; + + /* Filter */ + filter_ffd = flb_filter(ctx, (char *) "nest", NULL); + TEST_CHECK(filter_ffd >= 0); + + ret = flb_filter_set(ctx, filter_ffd, + "Match", "*", + "Operation", "lift", + "Nested_under", "nested", + NULL); + + TEST_CHECK(ret == 0); + + + /* Output */ + out_ffd = flb_output(ctx, (char *) "lib", (void *) &cb_data); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, + "match", "test", + "format", "json", + NULL); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + p = "[1448403340, {\"nested\": {\"child\":\"nested data\"}, \"not_nestd\":\"not nested data\" }]"; + bytes = flb_lib_push(ctx, in_ffd, p, strlen(p)); + TEST_CHECK(bytes == strlen(p)); + + p = "[1448403341, {\"not_nest\":\"dummy data\", \"extra\":\"dummy more data\"}]"; + bytes = flb_lib_push(ctx, in_ffd, p, strlen(p)); + TEST_CHECK(bytes == strlen(p)); + + flb_time_msleep(1500); /* waiting flush */ + count = get_output_num(); + + TEST_CHECK_(count == expected, "Expected number of events %d, got %d", expected, count ); + + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_filter_nest_single(void) +{ + int ret; + int bytes; + char *p, *output, *expected; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + int filter_ffd; + struct flb_lib_out_cb cb_data; + + ctx = flb_create(); + flb_service_set(ctx, + "Flush", "1", + "Grace", "1", + "Log_Level", "debug", + NULL); + + /* Input */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + /* Prepare output callback context*/ + cb_data.cb = callback_test; + cb_data.data = NULL; + + /* Filter */ + filter_ffd = flb_filter(ctx, (char *) "nest", NULL); + TEST_CHECK(filter_ffd >= 0); + + ret = flb_filter_set(ctx, filter_ffd, + "Match", "*", + "Operation", "nest", + "Wildcard", "to_nest", + "Nest_under", "nested_key", + NULL); + + TEST_CHECK(ret == 0); + + + /* Output */ + out_ffd = flb_output(ctx, (char *) "lib", (void *) &cb_data); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, + "match", "test", + "format", "json", + NULL); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + p = "[1448403340, {\"to_nest\":\"This is the data to nest\", \"extra\":\"Some more data\"}]"; + bytes = flb_lib_push(ctx, in_ffd, p, strlen(p)); + TEST_CHECK(bytes == strlen(p)); + + flb_time_msleep(1500); /* waiting flush */ + output = get_output(); + + TEST_CHECK_(output != NULL, "Expected output to not be NULL"); + + if (output != NULL) { + expected = "\"nested_key\":{\"to_nest\":\"This is the data to nest\"}}]"; + TEST_CHECK_(strstr(output, expected) != NULL, "Expected output to contain '%s', got '%s'", expected, output); + free(output); + } + flb_stop(ctx); + flb_destroy(ctx); +} diff --git a/src/fluent-bit/tests/runtime/filter_parser.c b/src/fluent-bit/tests/runtime/filter_parser.c new file mode 100644 index 000000000..c31fbcf84 --- /dev/null +++ b/src/fluent-bit/tests/runtime/filter_parser.c @@ -0,0 +1,833 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +#include <fluent-bit.h> +#include <fluent-bit/flb_time.h> +#include <fluent-bit/flb_parser.h> +#include "flb_tests_runtime.h" + +#define FLUSH_INTERVAL "1.0" + +pthread_mutex_t result_mutex = PTHREAD_MUTEX_INITIALIZER; +char *output = NULL; + +void set_output(char *val) +{ + pthread_mutex_lock(&result_mutex); + output = val; + pthread_mutex_unlock(&result_mutex); +} + +char *get_output(void) +{ + char *val; + + pthread_mutex_lock(&result_mutex); + val = output; + pthread_mutex_unlock(&result_mutex); + + return val; +} + +void clear_output() +{ + pthread_mutex_lock(&result_mutex); + output = NULL; + pthread_mutex_unlock(&result_mutex); +} + +int callback_test(void* data, size_t size, void* cb_data) +{ + if (size > 0) { + flb_debug("[test_filter_parser] received message: %s", (char*)data); + set_output(data); /* success */ + } + return 0; +} + +void wait_with_timeout(uint32_t timeout_ms, char **out_result) +{ + struct flb_time start_time; + struct flb_time end_time; + struct flb_time diff_time; + uint64_t elapsed_time_flb = 0; + char *output = NULL; + + flb_time_get(&start_time); + + while (true) { + output = get_output(); + + if (output != NULL) { + *out_result = output; + break; + } + + flb_time_msleep(100); + flb_time_get(&end_time); + flb_time_diff(&end_time, &start_time, &diff_time); + elapsed_time_flb = flb_time_to_nanosec(&diff_time) / 1000000; + + if (elapsed_time_flb > timeout_ms) { + flb_warn("[timeout] elapsed_time: %ld", elapsed_time_flb); + // Reached timeout. + break; + } + } +} + + +void flb_test_filter_parser_extract_fields() +{ + int ret; + int bytes; + char *p, *output, *expected; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + int filter_ffd; + struct flb_parser *parser; + + struct flb_lib_out_cb cb; + cb.cb = callback_test; + cb.data = NULL; + + clear_output(); + + ctx = flb_create(); + + /* Configure service */ + flb_service_set(ctx, "Flush", FLUSH_INTERVAL, "Grace" "1", "Log_Level", "debug", NULL); + + /* Input */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx, in_ffd, + "Tag", "test", + NULL); + + /* Parser */ + parser = flb_parser_create("dummy_test", "regex", "^(?<INT>[^ ]+) (?<FLOAT>[^ ]+) (?<BOOL>[^ ]+) (?<STRING>.+)$", + FLB_TRUE, + NULL, NULL, NULL, MK_FALSE, MK_TRUE, FLB_FALSE, NULL, 0, + NULL, ctx->config); + TEST_CHECK(parser != NULL); + + /* Filter */ + filter_ffd = flb_filter(ctx, (char *) "parser", NULL); + TEST_CHECK(filter_ffd >= 0); + ret = flb_filter_set(ctx, filter_ffd, + "Match", "test", + "Key_Name", "data", + "Parser", "dummy_test", + "Reserve_Data", "On", + "Preserve_Key", "Off", + NULL); + TEST_CHECK(ret == 0); + + /* Output */ + out_ffd = flb_output(ctx, (char *) "lib", &cb); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, + "Match", "*", + "format", "json", + NULL); + + /* Start the engine */ + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Ingest data */ + p = "[1448403340, {\"data\":\"100 0.5 true This is an example\", \"extra\":\"Some more data\"}]"; + bytes = flb_lib_push(ctx, in_ffd, p, strlen(p)); + TEST_CHECK(bytes == strlen(p)); + + wait_with_timeout(2000, &output); /* waiting flush and ensuring data flush */ + TEST_CHECK_(output != NULL, "Expected output to not be NULL"); + if (output != NULL) { + /* check timestamp */ + expected = "[1448403340.000000,{"; + TEST_CHECK_(strstr(output, expected) != NULL, "Expected output to contain '%s', got '%s'", expected, output); + /* check fields were extracted */ + expected = "\"INT\":\"100\",\"FLOAT\":\"0.5\",\"BOOL\":\"true\",\"STRING\":\"This is an example\""; + TEST_CHECK_(strstr(output, expected) != NULL, "Expected output to contain '%s', got '%s'", expected, output); + /* check original field was not preserved */ + expected = "\"data\":\"100 0.5 true This is an example\""; + TEST_CHECK_(strstr(output, expected) == NULL, "Expected output to not contain '%s', got '%s'", expected, output); + /* check extra data was preserved */ + expected = "\"extra\":\"Some more data\""; + TEST_CHECK_(strstr(output, expected) != NULL, "Expected output to preserve extra field, got '%s'", output); + free(output); + } + + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_filter_parser_reserve_data_off() +{ + int ret; + int bytes; + char *p, *output, *expected; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + int filter_ffd; + struct flb_parser *parser; + + struct flb_lib_out_cb cb; + cb.cb = callback_test; + cb.data = NULL; + + clear_output(); + + ctx = flb_create(); + + /* Configure service */ + flb_service_set(ctx, "Flush", FLUSH_INTERVAL, "Grace", "1", "Log_Level", "debug", NULL); + + /* Input */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx, in_ffd, + "Tag", "test", + NULL); + + /* Parser */ + parser = flb_parser_create("dummy_test", "regex", "^(?<INT>[^ ]+) (?<FLOAT>[^ ]+) (?<BOOL>[^ ]+) (?<STRING>.+)$", + FLB_TRUE, + NULL, NULL, NULL, MK_FALSE, MK_TRUE, FLB_FALSE, NULL, 0, + NULL, ctx->config); + TEST_CHECK(parser != NULL); + + /* Filter */ + filter_ffd = flb_filter(ctx, (char *) "parser", NULL); + TEST_CHECK(filter_ffd >= 0); + ret = flb_filter_set(ctx, filter_ffd, + "Match", "test", + "Key_Name", "data", + "Parser", "dummy_test", + "Reserve_Data", "Off", + NULL); + TEST_CHECK(ret == 0); + + /* Output */ + out_ffd = flb_output(ctx, (char *) "lib", &cb); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, + "Match", "*", + "format", "json", + NULL); + + /* Start the engine */ + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Ingest data */ + p = "[1448403340,{\"data\":\"100 0.5 true This is an example\",\"extra\":\"Some more data\"}]"; + bytes = flb_lib_push(ctx, in_ffd, p, strlen(p)); + TEST_CHECK(bytes == strlen(p)); + + wait_with_timeout(2000, &output); /* waiting flush and ensuring data flush */ + TEST_CHECK_(output != NULL, "Expected output to not be NULL"); + if (output != NULL) { + /* check extra data was not preserved */ + expected = "\"extra\":\"Some more data\""; + TEST_CHECK_(strstr(output, expected) == NULL, "Expected output to not preserve extra field, got '%s'", output); + free(output); + } + + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_filter_parser_handle_time_key() +{ + int ret; + int bytes; + char *p, *output, *expected; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + int filter_ffd; + struct flb_parser *parser; + + struct flb_lib_out_cb cb; + cb.cb = callback_test; + cb.data = NULL; + + clear_output(); + + ctx = flb_create(); + + /* Configure service */ + flb_service_set(ctx, "Flush", FLUSH_INTERVAL, "Grace", "1", "Log_Level", "debug", NULL); + + /* Input */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx, in_ffd, + "Tag", "test", + NULL); + + /* Parser */ + parser = flb_parser_create("timestamp", "regex", "^(?<time>.*)$", FLB_TRUE, + "%Y-%m-%dT%H:%M:%S.%L", + "time", + NULL, MK_FALSE, MK_TRUE, FLB_FALSE, + NULL, 0, NULL, ctx->config); + TEST_CHECK(parser != NULL); + + /* Filter */ + filter_ffd = flb_filter(ctx, (char *) "parser", NULL); + TEST_CHECK(filter_ffd >= 0); + ret = flb_filter_set(ctx, filter_ffd, + "Match", "test", + "Key_Name", "@timestamp", + "Parser", "timestamp", + "Reserve_Data", "On", + NULL); + TEST_CHECK(ret == 0); + + /* Output */ + out_ffd = flb_output(ctx, (char *) "lib", &cb); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, + "Match", "*", + "format", "json", + NULL); + + /* Start the engine */ + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Ingest data */ + p = "[1448403340, {\"@timestamp\":\"2017-11-01T22:25:21.648+00:00\", \"message\":\"This is an example\"}]"; + bytes = flb_lib_push(ctx, in_ffd, p, strlen(p)); + TEST_CHECK(bytes == strlen(p)); + + wait_with_timeout(2000, &output); /* waiting flush and ensuring data flush */ + TEST_CHECK_(output != NULL, "Expected output to not be NULL"); + if (output != NULL) { + /* check the timestamp field was updated correctly */ + /* this is in fluent-bits extended timestamp format */ + expected = "[1509575121.648000,{"; + TEST_CHECK_(strstr(output, expected) != NULL, "Expected output to contain '%s', got '%s'", expected, output); + /* check additional field is preserved */ + expected = "\"message\":\"This is an example\""; + TEST_CHECK_(strstr(output, expected) != NULL, "Expected output to contain '%s', got '%s'", expected, output); + free(output); + } + + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_filter_parser_handle_time_key_with_fractional_timestamp() +{ + int ret; + int bytes; + char *p, *output, *expected; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + int filter_ffd; + struct flb_parser *parser; + + clear_output(); + + ctx = flb_create(); + + /* Configure service */ + flb_service_set(ctx, "Flush", FLUSH_INTERVAL, "Grace", "1", "Log_Level", "debug", NULL); + + /* Input */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx, in_ffd, + "Tag", "test", + NULL); + + /* Parser */ + parser = flb_parser_create("timestamp", "regex", "^(?<time>.*)$", FLB_TRUE, + "%s.%L", + "time", + NULL, MK_FALSE, MK_TRUE, FLB_FALSE, + NULL, 0, NULL, ctx->config); + TEST_CHECK(parser != NULL); + + /* Filter */ + filter_ffd = flb_filter(ctx, (char *) "parser", NULL); + TEST_CHECK(filter_ffd >= 0); + ret = flb_filter_set(ctx, filter_ffd, + "Match", "test", + "Key_Name", "@timestamp", + "Parser", "timestamp", + "Reserve_Data", "On", + NULL); + TEST_CHECK(ret == 0); + + /* Output */ + out_ffd = flb_output(ctx, (char *) "lib", (void*)callback_test); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, + "Match", "*", + "format", "json", + NULL); + + /* Start the engine */ + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Ingest data */ + p = "[1448403340, {\"@timestamp\":\"1509575121.648\", \"message\":\"This is an example\"}]"; + bytes = flb_lib_push(ctx, in_ffd, p, strlen(p)); + TEST_CHECK(bytes == strlen(p)); + + wait_with_timeout(2000, &output); /* waiting flush and ensuring data flush */ + TEST_CHECK_(output != NULL, "Expected output to not be NULL"); + if (output != NULL) { + /* check the timestamp field was updated correctly */ + /* this is in fluent-bits extended timestamp format */ + expected = "[\"\\x59\\xfffffffa\\x49\\xffffffd1\\x26\\xffffff9f\\xffffffb2\\x00\", {"; + TEST_CHECK_(strstr(output, expected) != NULL, "Expected output to contain '%s', got '%s'", expected, output); + /* check additional field is preserved */ + expected = "\"message\":\"This is an example\""; + TEST_CHECK_(strstr(output, expected) != NULL, "Expected output to contain '%s', got '%s'", expected, output); + free(output); + } + + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_filter_parser_handle_time_key_with_time_zone() +{ + int ret; + int bytes; + char *p, *output, *expected; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + int filter_ffd; + struct flb_parser *parser; + + struct flb_lib_out_cb cb; + cb.cb = callback_test; + cb.data = NULL; + + clear_output(); + + ctx = flb_create(); + + /* Configure service */ + flb_service_set(ctx, "Flush", FLUSH_INTERVAL, "Grace", "1", "Log_Level", "debug", NULL); + + /* Input */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx, in_ffd, + "Tag", "test", + NULL); + + /* Parser */ + parser = flb_parser_create("timestamp", // name + "regex", // format + "^(?<time>.*)$", // regex + FLB_TRUE, // skip_empty + "%Y-%m-%dT%H:%M:%S.%L %z", // time_fmt + "time", // time_key + NULL, // time_offset + MK_FALSE, // time_keep + MK_TRUE, // time_strict + MK_FALSE, // logfmt_no_bare_keys + NULL, // types + 0, // types_len + NULL, // decoders + ctx->config); // config + TEST_CHECK(parser != NULL); + + /* Filter */ + filter_ffd = flb_filter(ctx, (char *) "parser", NULL); + TEST_CHECK(filter_ffd >= 0); + ret = flb_filter_set(ctx, filter_ffd, + "Match", "test", + "Key_Name", "@timestamp", + "Parser", "timestamp", + "Reserve_Data", "On", + NULL); + TEST_CHECK(ret == 0); + + /* Output */ + out_ffd = flb_output(ctx, (char *) "lib", &cb); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, + "Match", "*", + "format", "json", + NULL); + + /* Start the engine */ + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Ingest data */ + p = "[1448403340, {\"@timestamp\":\"2017-11-01T22:25:21.648-04:00\", \"message\":\"This is an example\"}]"; + bytes = flb_lib_push(ctx, in_ffd, p, strlen(p)); + TEST_CHECK(bytes == strlen(p)); + + wait_with_timeout(2000, &output); /* waiting flush and ensuring data flush */ + TEST_CHECK_(output != NULL, "Expected output to not be NULL"); + if (output != NULL) { + /* check the timestamp field was updated correctly */ + /* this is in fluent-bits extended timestamp format */ + expected = "[1509589521.648000,{"; + TEST_CHECK_(strstr(output, expected) != NULL, "Expected output to contain '%s', got '%s'", expected, output); + /* check additional field is preserved */ + expected = "\"message\":\"This is an example\""; + TEST_CHECK_(strstr(output, expected) != NULL, "Expected output to contain '%s', got '%s'", expected, output); + free(output); + } + + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_filter_parser_ignore_malformed_time() +{ + int ret; + int bytes; + char *p, *output, *expected; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + int filter_ffd; + struct flb_parser *parser; + + struct flb_lib_out_cb cb; + cb.cb = callback_test; + cb.data = NULL; + + clear_output(); + + ctx = flb_create(); + + /* Configure service */ + flb_service_set(ctx, "Flush", FLUSH_INTERVAL, "Grace", "1", "Log_Level", "debug", NULL); + + /* Input */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx, in_ffd, + "Tag", "test", + NULL); + + /* Parser */ + parser = flb_parser_create("timestamp", "regex", + "^(?<time>.*)$", FLB_TRUE, + "%Y-%m-%dT%H:%M:%S.%L", "time", + NULL, FLB_FALSE, MK_TRUE, FLB_FALSE, + NULL, 0, NULL, ctx->config); + TEST_CHECK(parser != NULL); + + /* Filter */ + filter_ffd = flb_filter(ctx, (char *) "parser", NULL); + TEST_CHECK(filter_ffd >= 0); + ret = flb_filter_set(ctx, filter_ffd, + "Match", "test", + "Key_Name", "@timestamp", + "Parser", "timestamp", + "Reserve_Data", "On", + "Preserve_Key", "On", + NULL); + TEST_CHECK(ret == 0); + + /* Output */ + out_ffd = flb_output(ctx, (char *) "lib", &cb); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, + "Match", "*", + "format", "json", + NULL); + + /* Start the engine */ + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Ingest data */ + p = "[1448403340, {\"@timestamp\":\"2017_$!^-11-01T22:25:21.648\", \"log\":\"An example\"}]"; + bytes = flb_lib_push(ctx, in_ffd, p, strlen(p)); + TEST_CHECK(bytes == strlen(p)); + + wait_with_timeout(2000, &output); /* waiting flush and ensuring data flush */ + TEST_CHECK_(output != NULL, "Expected output to not be NULL"); + if (output != NULL) { + /* check the timestamp field was ignored and we received everything else */ + expected = "[1448403340.000000,{\"@timestamp\":\"2017_$!^-11-01T22:25:21.648\",\"log\":\"An example\"}]"; + TEST_CHECK_(strcmp(output, expected) == 0, "Expected output to be '%s', got '%s'", expected, output); + free(output); + } + + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_filter_parser_preserve_original_field() +{ + int ret; + int bytes; + char *p, *output, *expected; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + int filter_ffd; + struct flb_parser *parser; + + struct flb_lib_out_cb cb; + cb.cb = callback_test; + cb.data = NULL; + + clear_output(); + + ctx = flb_create(); + + /* Configure service */ + flb_service_set(ctx, "Flush", FLUSH_INTERVAL, "Grace", "1", "Log_Level", "debug", NULL); + + /* Input */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx, in_ffd, + "Tag", "test", + NULL); + + /* Parser */ + parser = flb_parser_create("dummy_test", "regex", "^(?<INT>[^ ]+) (?<FLOAT>[^ ]+) (?<BOOL>[^ ]+) (?<STRING>.+)$", + FLB_TRUE, + NULL, NULL, NULL, MK_FALSE, MK_TRUE, FLB_FALSE, NULL, 0, + NULL, ctx->config); + TEST_CHECK(parser != NULL); + + /* Filter */ + filter_ffd = flb_filter(ctx, (char *) "parser", NULL); + TEST_CHECK(filter_ffd >= 0); + ret = flb_filter_set(ctx, filter_ffd, + "Match", "test", + "Key_Name", "data", + "Parser", "dummy_test", + "Reserve_Data", "On", + "Preserve_Key", "On", + NULL); + TEST_CHECK(ret == 0); + + /* Output */ + out_ffd = flb_output(ctx, (char *) "lib", &cb); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, + "Match", "*", + "format", "json", + NULL); + + /* Start the engine */ + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Ingest data */ + p = "[1448403340,{\"data\":\"100 0.5 true This is an example\",\"log\":\"An example\"}]"; + bytes = flb_lib_push(ctx, in_ffd, p, strlen(p)); + TEST_CHECK(bytes == strlen(p)); + + wait_with_timeout(2000, &output); /* waiting flush and ensuring data flush */ + TEST_CHECK_(output != NULL, "Expected output to not be NULL"); + if (output != NULL) { + /* check original field is preserved */ + expected = "\"data\":\"100 0.5 true This is an example\""; + TEST_CHECK_(strstr(output, expected) != NULL, "Expected output to contain '%s', got '%s'", expected, output); + /* check fields were extracted */ + expected = "\"INT\":\"100\",\"FLOAT\":\"0.5\",\"BOOL\":\"true\",\"STRING\":\"This is an example\""; + TEST_CHECK_(strstr(output, expected) != NULL, "Expected output to contain '%s', got '%s'", expected, output); + /* check other fields are preserved */ + expected = "\"log\":\"An example\""; + TEST_CHECK_(strstr(output, expected) != NULL, "Expected output to contain '%s', got '%s'", expected, output); + free(output); + } + + flb_stop(ctx); + flb_destroy(ctx); +} + +// https://github.com/fluent/fluent-bit/issues/2250 +void flb_test_filter_parser_first_matched_when_mutilple_parser() +{ + int ret; + int bytes; + char *p, *output, *expected; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + int filter_ffd; + struct flb_parser *parser; + + struct flb_lib_out_cb cb; + cb.cb = callback_test; + cb.data = NULL; + + clear_output(); + + ctx = flb_create(); + + /* Configure service */ + flb_service_set(ctx, "Flush", FLUSH_INTERVAL, "Grace" "1", "Log_Level", "debug", NULL); + + /* Input */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx, in_ffd, + "Tag", "test", + NULL); + + /* Parser */ + parser = flb_parser_create("one", "regex", "^(?<one>.+?)$", + FLB_TRUE, + NULL, NULL, NULL, MK_FALSE, MK_TRUE, FLB_FALSE, + NULL, 0, NULL, ctx->config); + TEST_CHECK(parser != NULL); + + parser = flb_parser_create("two", "regex", "^(?<two>.+?)$", + FLB_TRUE, + NULL, NULL, NULL, MK_FALSE, MK_TRUE, FLB_FALSE, + NULL, 0, NULL, ctx->config); + TEST_CHECK(parser != NULL); + + /* Filter */ + filter_ffd = flb_filter(ctx, (char *) "parser", NULL); + TEST_CHECK(filter_ffd >= 0); + ret = flb_filter_set(ctx, filter_ffd, + "Match", "test", + "Key_Name", "data", + "Parser", "one", + "Parser", "two", + "Reserve_Data", "On", + "Preserve_Key", "On", + NULL); + TEST_CHECK(ret == 0); + + /* Output */ + out_ffd = flb_output(ctx, (char *) "lib", &cb); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, + "Match", "*", + "format", "json", + NULL); + + /* Start the engine */ + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Ingest data */ + p = "[1,{\"data\":\"hoge\"}]"; + bytes = flb_lib_push(ctx, in_ffd, p, strlen(p)); + TEST_CHECK(bytes == strlen(p)); + + wait_with_timeout(2000, &output); /* waiting flush and ensuring data flush */ + TEST_CHECK_(output != NULL, "Expected output to not be NULL"); + if (output != NULL) { + /* check extra data was not preserved */ + expected = "\"one\":\"hoge\",\"data\":\"hoge\""; + TEST_CHECK_(strstr(output, expected) != NULL, "Expected output to contain key one , got '%s'", output); + free(output); + } + + flb_stop(ctx); + flb_destroy(ctx); +} + +// https://github.com/fluent/fluent-bit/issues/1486 +// https://github.com/fluent/fluent-bit/issues/2939 +void flb_test_filter_parser_skip_empty_values_false() +{ + int ret; + int bytes; + char *p, *output, *expected; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + int filter_ffd; + struct flb_parser *parser; + + struct flb_lib_out_cb cb; + cb.cb = callback_test; + cb.data = NULL; + + clear_output(); + + ctx = flb_create(); + + /* Configure service */ + flb_service_set(ctx, "Flush", FLUSH_INTERVAL, "Grace" "1", "Log_Level", "debug", NULL); + + /* Input */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx, in_ffd, + "Tag", "test", + NULL); + + /* Parser */ + parser = flb_parser_create("one", "regex", "^(?<one>.+?)$", + FLB_FALSE, + NULL, NULL, NULL, MK_FALSE, MK_TRUE, FLB_FALSE, + NULL, 0, NULL, ctx->config); + TEST_CHECK(parser != NULL); + + /* Filter */ + filter_ffd = flb_filter(ctx, (char *) "parser", NULL); + TEST_CHECK(filter_ffd >= 0); + ret = flb_filter_set(ctx, filter_ffd, + "Match", "test", + "Key_Name", "data", + "Parser", "one", + NULL); + TEST_CHECK(ret == 0); + + /* Output */ + out_ffd = flb_output(ctx, (char *) "lib", &cb); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, + "Match", "*", + "format", "json", + NULL); + + /* Start the engine */ + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Ingest data */ + p = "[1,{\"data\":\"\"}]"; + bytes = flb_lib_push(ctx, in_ffd, p, strlen(p)); + TEST_CHECK(bytes == strlen(p)); + + wait_with_timeout(1500, &output); /* waiting flush and ensuring data flush */ + TEST_CHECK_(output != NULL, "Expected output to not be NULL"); + if (output != NULL) { + /* check extra data was not preserved */ + expected = "\"data\":\"\""; + TEST_CHECK_(strstr(output, expected) != NULL, "Expected output to contain key one , got '%s'", output); + free(output); + } + + flb_stop(ctx); + flb_destroy(ctx); +} + + +TEST_LIST = { + {"filter_parser_extract_fields", flb_test_filter_parser_extract_fields }, + {"filter_parser_reserve_data_off", flb_test_filter_parser_reserve_data_off }, + {"filter_parser_handle_time_key", flb_test_filter_parser_handle_time_key }, + {"filter_parser_handle_time_key_with_time_zone", flb_test_filter_parser_handle_time_key_with_time_zone }, + {"filter_parser_ignore_malformed_time", flb_test_filter_parser_ignore_malformed_time }, + {"filter_parser_preserve_original_field", flb_test_filter_parser_preserve_original_field }, + {"filter_parser_first_matched_when_multiple_parser", flb_test_filter_parser_first_matched_when_mutilple_parser }, + {"filter_parser_skip_empty_values_false", flb_test_filter_parser_skip_empty_values_false}, + {NULL, NULL} +}; + diff --git a/src/fluent-bit/tests/runtime/filter_record_modifier.c b/src/fluent-bit/tests/runtime/filter_record_modifier.c new file mode 100644 index 000000000..2dbc9322c --- /dev/null +++ b/src/fluent-bit/tests/runtime/filter_record_modifier.c @@ -0,0 +1,595 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2019-2021 The Fluent Bit Authors + * Copyright (C) 2015-2018 Treasure Data Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <fluent-bit.h> +#include <fluent-bit/flb_time.h> +#include <msgpack.h> +#include "flb_tests_runtime.h" +#include "data/common/json_long.h" /* JSON_LONG */ + +struct filter_test { + flb_ctx_t *flb; /* Fluent Bit library context */ + int i_ffd; /* Input fd */ + int f_ffd; /* Filter fd */ + int o_ffd; /* Output fd */ +}; + +struct expect_str { + char *str; + int found; +}; + + +/* Callback to check expected results */ +static int cb_check_result(void *record, size_t size, void *data) +{ + char *p; + char *result; + struct expect_str *expected; + + expected = (struct expect_str*)data; + result = (char *) record; + + if (!TEST_CHECK(expected != NULL)) { + flb_error("expected is NULL"); + } + if (!TEST_CHECK(result != NULL)) { + flb_error("result is NULL"); + } + + while(expected != NULL && expected->str != NULL) { + if (expected->found == FLB_TRUE) { + p = strstr(result, expected->str); + if(!TEST_CHECK(p != NULL)) { + flb_error("Expected to find: '%s' in result '%s'", + expected->str, result); + } + } + else { + p = strstr(result, expected->str); + if(!TEST_CHECK(p == NULL)) { + flb_error("'%s' should be removed in result '%s'", + expected->str, result); + } + } + + /* + * If you want to debug your test + * + * printf("Expect: '%s' in result '%s'", expected, result); + */ + + expected++; + } + + flb_free(record); + return 0; +} + +static struct filter_test *filter_test_create(struct flb_lib_out_cb *data) +{ + int i_ffd; + int f_ffd; + int o_ffd; + struct filter_test *ctx; + + ctx = flb_malloc(sizeof(struct filter_test)); + if (!ctx) { + flb_errno(); + return NULL; + } + + /* Service config */ + ctx->flb = flb_create(); + flb_service_set(ctx->flb, + "Flush", "0.200000000", + "Grace", "1", + "Log_Level", "error", + NULL); + + /* Input */ + i_ffd = flb_input(ctx->flb, (char *) "lib", NULL); + TEST_CHECK(i_ffd >= 0); + flb_input_set(ctx->flb, i_ffd, "tag", "test", NULL); + ctx->i_ffd = i_ffd; + + /* Filter configuration */ + f_ffd = flb_filter(ctx->flb, (char *) "record_modifier", NULL); + TEST_CHECK(f_ffd >= 0); + flb_filter_set(ctx->flb, f_ffd, "match", "*", NULL); + ctx->f_ffd = f_ffd; + + /* Output */ + o_ffd = flb_output(ctx->flb, (char *) "lib", (void *) data); + TEST_CHECK(o_ffd >= 0); + flb_output_set(ctx->flb, o_ffd, + "match", "test", + NULL); + ctx->o_ffd = o_ffd; + + return ctx; +} + +static void filter_test_destroy(struct filter_test *ctx) +{ + sleep(1); + flb_stop(ctx->flb); + flb_destroy(ctx->flb); + flb_free(ctx); +} + +static void flb_records() +{ + int len; + int ret; + int bytes; + char *p; + struct flb_lib_out_cb cb_data; + struct filter_test *ctx; + struct expect_str expect[] = { + {"\"new_key\":\"new_val\"", FLB_TRUE}, + {"\"add_key\":\"add_val\"", FLB_TRUE}, + {"\"k\":\"sample\"", FLB_TRUE}, + {NULL, FLB_TRUE} + }; + + /* Create test context */ + ctx = filter_test_create((void *) &cb_data); + if (!ctx) { + exit(EXIT_FAILURE); + } + + /* Configure filter */ + ret = flb_filter_set(ctx->flb, ctx->f_ffd, + "record", "new_key new_val", + "record", "add_key add_val", + NULL); + TEST_CHECK(ret == 0); + + ret = flb_output_set(ctx->flb, ctx->o_ffd, + "format", "json", + NULL); + TEST_CHECK(ret == 0); + + /* Prepare output callback with expected result */ + cb_data.cb = cb_check_result; + cb_data.data = &expect; + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + /* Ingest data samples */ + p = "[0, {\"k\":\"sample\"}]"; + len = strlen(p); + bytes = flb_lib_push(ctx->flb, ctx->i_ffd, p, len); + TEST_CHECK(bytes == len); + + filter_test_destroy(ctx); +} + +static void flb_allowlist_keys() +{ + int len; + int ret; + int bytes; + char *p; + struct flb_lib_out_cb cb_data; + struct filter_test *ctx; + struct expect_str expect[] = { + {"\"aaa\":\"ok\"", FLB_TRUE}, + {"\"bbb\":\"ok\"", FLB_TRUE}, + {"\"ccc\":\"removed\"", FLB_FALSE}, + {NULL, FLB_TRUE} + }; + + /* Create test context */ + ctx = filter_test_create((void *) &cb_data); + if (!ctx) { + exit(EXIT_FAILURE); + } + + /* Configure filter */ + ret = flb_filter_set(ctx->flb, ctx->f_ffd, + "allowlist_key", "aaa", + "allowlist_key", "bbb", + NULL); + TEST_CHECK(ret == 0); + + ret = flb_output_set(ctx->flb, ctx->o_ffd, + "format", "json", + NULL); + TEST_CHECK(ret == 0); + + /* Prepare output callback with expected result */ + cb_data.cb = cb_check_result; + cb_data.data = &expect; + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + /* Ingest data samples */ + /* The pair "ccc":"removed" should be removed */ + p = "[0, {\"aaa\":\"ok\",\"ccc\":\"removed\",\"bbb\":\"ok\"}]"; + len = strlen(p); + bytes = flb_lib_push(ctx->flb, ctx->i_ffd, p, len); + TEST_CHECK(bytes == len); + + filter_test_destroy(ctx); +} + +static void flb_whitelist_keys() +{ + int len; + int ret; + int bytes; + char *p; + struct flb_lib_out_cb cb_data; + struct filter_test *ctx; + struct expect_str expect[] = { + {"\"aaa\":\"ok\"", FLB_TRUE}, + {"\"bbb\":\"ok\"", FLB_TRUE}, + {"\"ccc\":\"removed\"", FLB_FALSE}, + {NULL, FLB_TRUE} + }; + + /* Create test context */ + ctx = filter_test_create((void *) &cb_data); + if (!ctx) { + exit(EXIT_FAILURE); + } + + /* Configure filter */ + ret = flb_filter_set(ctx->flb, ctx->f_ffd, + "whitelist_key", "aaa", + "whitelist_key", "bbb", + NULL); + TEST_CHECK(ret == 0); + + ret = flb_output_set(ctx->flb, ctx->o_ffd, + "format", "json", + NULL); + TEST_CHECK(ret == 0); + + /* Prepare output callback with expected result */ + cb_data.cb = cb_check_result; + cb_data.data = &expect; + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + /* Ingest data samples */ + /* The pair "ccc":"removed" should be removed */ + p = "[0, {\"aaa\":\"ok\",\"ccc\":\"removed\",\"bbb\":\"ok\"}]"; + len = strlen(p); + bytes = flb_lib_push(ctx->flb, ctx->i_ffd, p, len); + TEST_CHECK(bytes == len); + + filter_test_destroy(ctx); +} + +static void flb_remove_keys() +{ + int len; + int ret; + int bytes; + char *p; + struct flb_lib_out_cb cb_data; + struct filter_test *ctx; + struct expect_str expect[] = { + {"\"aaa\":\"ok\"", FLB_TRUE}, + {"\"bbb\":\"ok\"", FLB_TRUE}, + {"\"ccc\":\"removed\"", FLB_FALSE}, + {"\"ddd\":\"removed\"", FLB_FALSE}, + {NULL, FLB_TRUE} + }; + + /* Create test context */ + ctx = filter_test_create((void *) &cb_data); + if (!ctx) { + exit(EXIT_FAILURE); + } + + /* Configure filter */ + ret = flb_filter_set(ctx->flb, ctx->f_ffd, + "remove_key", "ccc", + "remove_key", "ddd", + NULL); + TEST_CHECK(ret == 0); + + ret = flb_output_set(ctx->flb, ctx->o_ffd, + "format", "json", + NULL); + TEST_CHECK(ret == 0); + + /* Prepare output callback with expected result */ + cb_data.cb = cb_check_result; + cb_data.data = &expect; + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + /* Ingest data samples */ + /* The pairs "ccc" and "ddd" should be removed */ + p = "[0, {\"aaa\":\"ok\",\"ccc\":\"removed\",\"ddd\":\"removed\",\"bbb\":\"ok\"}]"; + len = strlen(p); + bytes = flb_lib_push(ctx->flb, ctx->i_ffd, p, len); + TEST_CHECK(bytes == len); + + filter_test_destroy(ctx); +} + +static void flb_multiple() +{ + int len; + int ret; + int bytes; + char *p; + struct flb_lib_out_cb cb_data; + struct filter_test *ctx; + struct expect_str expect[] = { + {"\"aaa\":\"ok\"", FLB_TRUE}, + {"\"new_key\":\"new_val\"", FLB_TRUE}, + {"\"ddd\":\"removed\"", FLB_FALSE}, + {NULL, FLB_TRUE} + }; + + /* Create test context */ + ctx = filter_test_create((void *) &cb_data); + if (!ctx) { + exit(EXIT_FAILURE); + } + + /* Configure filter */ + ret = flb_filter_set(ctx->flb, ctx->f_ffd, + "record", "new_key new_val", + "allowlist_key", "new_key", + "allowlist_key", "aaa", + NULL); + TEST_CHECK(ret == 0); + + ret = flb_output_set(ctx->flb, ctx->o_ffd, + "format", "json", + NULL); + TEST_CHECK(ret == 0); + + /* Prepare output callback with expected result */ + cb_data.cb = cb_check_result; + cb_data.data = &expect; + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + /* Ingest data samples */ + p = "[0, {\"aaa\":\"ok\",\"ddd\":\"removed\",\"bbb\":\"ok\"}]"; + len = strlen(p); + bytes = flb_lib_push(ctx->flb, ctx->i_ffd, p, len); + TEST_CHECK(bytes == len); + + filter_test_destroy(ctx); +} + + +/* https://github.com/fluent/fluent-bit/issues/3968 */ +void flb_test_json_long() +{ + int ret; + int size = sizeof(JSON_LONG) - 1; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + int filter_ffd; + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", "1", "grace", "1", + "log_level", "error", + NULL); + + /* Lib input mode */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + TEST_CHECK(in_ffd >= 0); + + out_ffd = flb_output(ctx, (char *) "null", NULL); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, "match", "test", NULL); + + filter_ffd = flb_filter(ctx, (char *) "record_modifier", NULL); + TEST_CHECK(filter_ffd >= 0); + flb_filter_set(ctx, filter_ffd, "match", "test", NULL); + + /* Start */ + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + flb_lib_push(ctx, in_ffd, (char *) JSON_LONG, size); + + flb_time_msleep(1500); /* waiting flush */ + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_exclusive_setting() +{ + int ret; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + int filter_ffd; + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", "1", "grace", "1", + "log_level", "error", + NULL); + + /* Lib input mode */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + TEST_CHECK(in_ffd >= 0); + + out_ffd = flb_output(ctx, (char *) "null", NULL); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, "match", "test", NULL); + + filter_ffd = flb_filter(ctx, (char *) "record_modifier", NULL); + TEST_CHECK(filter_ffd >= 0); + flb_filter_set(ctx, filter_ffd, "match", "test", NULL); + + ret = flb_filter_set(ctx, filter_ffd, + "allowlist_key", "aaa", + "remove_key", "bbb", + NULL); + + /* Start */ + ret = flb_start(ctx); + + /* It should be error since "allowlist_key" and "remove_key" are exclusive */ + TEST_CHECK(ret != 0); + + flb_stop(ctx); + flb_destroy(ctx); +} + +#define UUID_KEY_NAME "uuid" +static int cb_check_uuid(void *record, size_t size, void *data) +{ + char uuid[256] = {0}; + msgpack_unpacked result; + msgpack_object obj; + size_t off = 0; + int ret; + int i_map; + int map_size; + int uuid_found = FLB_FALSE; + char uuid_part[5][16]; + +#ifndef FLB_HAVE_OPENSSL + if (!TEST_CHECK(1 == 0)) { + /* flb_utils_uuid_v4_gen function needs openssl */ + TEST_MSG("uuid_key needs OpenSSL"); + return -1; + } +#endif + + if (!TEST_CHECK(record != NULL)) { + TEST_MSG("data is null"); + return -1; + } + + msgpack_unpacked_init(&result); + while (msgpack_unpack_next(&result, record, size, &off) == MSGPACK_UNPACK_SUCCESS) { + obj = result.data; + /* + msgpack_object_print(stdout, obj); + */ + if (!TEST_CHECK(obj.type == MSGPACK_OBJECT_ARRAY && obj.via.array.size == 2)) { + TEST_MSG("array error. type = %d", obj.type); + continue; + } + obj = obj.via.array.ptr[1]; + if (!TEST_CHECK(obj.type == MSGPACK_OBJECT_MAP)) { + TEST_MSG("map error. type = %d", obj.type); + continue; + } + map_size = obj.via.map.size; + for (i_map=0; i_map<map_size; i_map++) { + if (obj.via.map.ptr[i_map].key.type != MSGPACK_OBJECT_STR) { + continue; + } + if (strncmp(UUID_KEY_NAME, obj.via.map.ptr[i_map].key.via.str.ptr, strlen(UUID_KEY_NAME)) == 0) { + memcpy(&uuid[0], obj.via.map.ptr[i_map].val.via.str.ptr, obj.via.map.ptr[i_map].val.via.str.size); + ret = sscanf(&uuid[0], "%8s-%4s-%4s-%4s-%12s", &uuid_part[0][0], &uuid_part[1][0], &uuid_part[2][0], &uuid_part[3][0], &uuid_part[4][0]); + if (!TEST_CHECK(ret == 5)) { + TEST_MSG("ret should be 5. ret=%d", ret); + } + uuid_found = FLB_TRUE; + } + } + } + + if (!TEST_CHECK(uuid_found == FLB_TRUE)) { + TEST_MSG("uuid not found"); + } + + msgpack_unpacked_destroy(&result); + flb_free(record); + + return 0; +} + +static void flb_uuid_key() +{ + int len; + int ret; + int bytes; + int not_used; + char *p; + struct flb_lib_out_cb cb_data; + struct filter_test *ctx; + + /* Prepare output callback with expected result */ + cb_data.cb = cb_check_uuid; + cb_data.data = ¬_used; + + /* Create test context */ + ctx = filter_test_create((void *) &cb_data); + if (!ctx) { + exit(EXIT_FAILURE); + } + + /* Configure filter */ + ret = flb_filter_set(ctx->flb, ctx->f_ffd, + "uuid_key", UUID_KEY_NAME, + NULL); + TEST_CHECK(ret == 0); + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + /* Ingest data samples */ + p = "[0, {\"key_name\":\"sample\"}]"; + len = strlen(p); + bytes = flb_lib_push(ctx->flb, ctx->i_ffd, p, len); + TEST_CHECK(bytes == len); + + filter_test_destroy(ctx); +} + + +/* test list */ +TEST_LIST = { + {"json_long" , flb_test_json_long }, + {"remove_keys" , flb_remove_keys}, + {"records" , flb_records}, + {"allowlist_keys" , flb_allowlist_keys}, + {"whitelist_keys" , flb_whitelist_keys}, + {"multiple" , flb_multiple}, + {"exclusive_setting" , flb_exclusive_setting}, + {"uuid_key" , flb_uuid_key}, + {NULL, NULL} +}; diff --git a/src/fluent-bit/tests/runtime/filter_rewrite_tag.c b/src/fluent-bit/tests/runtime/filter_rewrite_tag.c new file mode 100644 index 000000000..08cf37511 --- /dev/null +++ b/src/fluent-bit/tests/runtime/filter_rewrite_tag.c @@ -0,0 +1,564 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2019-2022 The Fluent Bit Authors + * Copyright (C) 2015-2018 Treasure Data Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <fluent-bit.h> +#include <fluent-bit/flb_time.h> +#include <msgpack.h> +#include "flb_tests_runtime.h" + +struct filter_test { + flb_ctx_t *flb; /* Fluent Bit library context */ + int i_ffd; /* Input fd */ + int f_ffd; /* Filter fd */ + int o_ffd; /* Output fd */ +}; + +struct expect_str { + char *str; + int found; +}; + +pthread_mutex_t result_mutex = PTHREAD_MUTEX_INITIALIZER; +int num_output = 0; + +static int cb_count_msgpack(void *record, size_t size, void *data) +{ + msgpack_unpacked result; + size_t off = 0; + + if (!TEST_CHECK(data != NULL)) { + flb_error("data is NULL"); + } + + /* Iterate each item array and apply rules */ + msgpack_unpacked_init(&result); + while (msgpack_unpack_next(&result, record, size, &off) == MSGPACK_UNPACK_SUCCESS) { + pthread_mutex_lock(&result_mutex); + num_output++; + pthread_mutex_unlock(&result_mutex); + } + msgpack_unpacked_destroy(&result); + + flb_free(record); + return 0; +} + +static void clear_output_num() +{ + pthread_mutex_lock(&result_mutex); + num_output = 0; + pthread_mutex_unlock(&result_mutex); +} + +static int get_output_num() +{ + int ret; + pthread_mutex_lock(&result_mutex); + ret = num_output; + pthread_mutex_unlock(&result_mutex); + + return ret; +} + +static struct filter_test *filter_test_create(struct flb_lib_out_cb *data) +{ + int i_ffd; + int f_ffd; + int o_ffd; + struct filter_test *ctx; + + ctx = flb_malloc(sizeof(struct filter_test)); + if (!ctx) { + flb_errno(); + return NULL; + } + + /* Service config */ + ctx->flb = flb_create(); + flb_service_set(ctx->flb, + "Flush", "0.200000000", + "Grace", "1", + "Log_Level", "error", + NULL); + + /* Input */ + i_ffd = flb_input(ctx->flb, (char *) "lib", NULL); + TEST_CHECK(i_ffd >= 0); + flb_input_set(ctx->flb, i_ffd, "tag", "rewrite", NULL); + ctx->i_ffd = i_ffd; + + /* Filter configuration */ + f_ffd = flb_filter(ctx->flb, (char *) "rewrite_tag", NULL); + TEST_CHECK(f_ffd >= 0); + flb_filter_set(ctx->flb, f_ffd, "match", "rewrite", NULL); + ctx->f_ffd = f_ffd; + + /* Output */ + o_ffd = flb_output(ctx->flb, (char *) "lib", (void *) data); + ctx->o_ffd = o_ffd; + + return ctx; +} + +static void filter_test_destroy(struct filter_test *ctx) +{ + sleep(1); + flb_stop(ctx->flb); + flb_destroy(ctx->flb); + flb_free(ctx); +} + + +/* + * Original tag: rewrite + * Rewritten tag: updated + */ +static void flb_test_matched() +{ + struct flb_lib_out_cb cb_data; + struct filter_test *ctx; + int ret; + int not_used = 0; + int bytes; + int got; + char *p = "[0, {\"key\":\"rewrite\"}]"; + + /* Prepare output callback with expected result */ + cb_data.cb = cb_count_msgpack; + cb_data.data = ¬_used; + + /* Create test context */ + ctx = filter_test_create((void *) &cb_data); + if (!ctx) { + exit(EXIT_FAILURE); + } + clear_output_num(); + /* Configure filter */ + ret = flb_filter_set(ctx->flb, ctx->f_ffd, + "Rule", "$key ^(rewrite)$ updated false", + NULL); + TEST_CHECK(ret == 0); + + /* Configure output */ + ret = flb_output_set(ctx->flb, ctx->o_ffd, + "Match", "updated", + NULL); + TEST_CHECK(ret == 0); + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + /* ingest record */ + bytes = flb_lib_push(ctx->flb, ctx->i_ffd, p, strlen(p)); + TEST_CHECK(bytes == strlen(p)); + + flb_time_msleep(1500); /* waiting flush */ + got = get_output_num(); + + if (!TEST_CHECK(got != 0)) { + TEST_MSG("expect: %d got: %d", 1, got); + } + + filter_test_destroy(ctx); +} + +/* + * Original tag: rewrite + * Rewritten tag: updated + */ +static void flb_test_not_matched() +{ + struct flb_lib_out_cb cb_data; + struct filter_test *ctx; + int ret; + int not_used = 0; + int bytes; + int got; + char *p = "[0, {\"key\":\"not_match\"}]"; + + /* Prepare output callback with expected result */ + cb_data.cb = cb_count_msgpack; + cb_data.data = ¬_used; + + /* Create test context */ + ctx = filter_test_create((void *) &cb_data); + if (!ctx) { + exit(EXIT_FAILURE); + } + clear_output_num(); + /* Configure filter */ + ret = flb_filter_set(ctx->flb, ctx->f_ffd, + "Rule", "$key ^(rewrite)$ updated false", + NULL); + TEST_CHECK(ret == 0); + + /* Configure output */ + ret = flb_output_set(ctx->flb, ctx->o_ffd, + "Match", "rewrite", + NULL); + TEST_CHECK(ret == 0); + + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + /* ingest record */ + bytes = flb_lib_push(ctx->flb, ctx->i_ffd, p, strlen(p)); + TEST_CHECK(bytes == strlen(p)); + + flb_time_msleep(1500); /* waiting flush */ + got = get_output_num(); + + if (!TEST_CHECK(got != 0)) { + TEST_MSG("expect: %d got: %d", 1, got); + } + + filter_test_destroy(ctx); +} + +/* + * Original tag: rewrite + * Rewritten tag: updated + */ +static void flb_test_keep_true() +{ + struct flb_lib_out_cb cb_data; + struct filter_test *ctx; + int ret; + int not_used = 0; + int bytes; + int got; + char *p = "[0, {\"key\":\"rewrite\"}]"; + + /* Prepare output callback with expected result */ + cb_data.cb = cb_count_msgpack; + cb_data.data = ¬_used; + + /* Create test context */ + ctx = filter_test_create((void *) &cb_data); + if (!ctx) { + exit(EXIT_FAILURE); + } + clear_output_num(); + /* Configure filter */ + ret = flb_filter_set(ctx->flb, ctx->f_ffd, + "Rule", "$key ^(rewrite)$ updated true", + NULL); + TEST_CHECK(ret == 0); + + /* Configure output to count up all record */ + ret = flb_output_set(ctx->flb, ctx->o_ffd, "Match", "*", NULL); + TEST_CHECK(ret == 0); + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + /* ingest record */ + bytes = flb_lib_push(ctx->flb, ctx->i_ffd, p, strlen(p)); + TEST_CHECK(bytes == strlen(p)); + + flb_time_msleep(1500); /* waiting flush */ + got = get_output_num(); + + /* original record(keep) + rewritten record */ + if (!TEST_CHECK(got == 2)) { + TEST_MSG("expect: %d got: %d", 2, got); + } + + filter_test_destroy(ctx); +} + +/* https://github.com/fluent/fluent-bit/issues/4049 + * Emitter should pause if tons of input come. + */ +static void flb_test_heavy_input_pause_emitter() +{ + struct flb_lib_out_cb cb_data; + struct filter_test *ctx; + int ret; + int not_used = 0; + int bytes; + int heavy_loop = 100000; + int got; + char p[256]; + int i; + + /* Prepare output callback with expected result */ + cb_data.cb = cb_count_msgpack; + cb_data.data = ¬_used; + + /* Create test context */ + ctx = filter_test_create((void *) &cb_data); + if (!ctx) { + exit(EXIT_FAILURE); + } + clear_output_num(); + /* Configure filter */ + ret = flb_filter_set(ctx->flb, ctx->f_ffd, + "Rule", "$key ^(rewrite)$ updated false", + "Emitter_Mem_Buf_Limit", "1kb", + NULL); + TEST_CHECK(ret == 0); + + /* Configure output */ + ret = flb_output_set(ctx->flb, ctx->o_ffd, + "Match", "updated", + NULL); + TEST_CHECK(ret == 0); + + /* Suppress emitter log. error registering chunk with tag: updated */ + ret = flb_service_set(ctx->flb, "Log_Level", "Off", NULL); + TEST_CHECK(ret == 0); + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + for (i = 0; i < heavy_loop; i++) { + memset(p, '\0', sizeof(p)); + snprintf(p, sizeof(p), "[%d, {\"val\": \"%d\",\"key\": \"rewrite\"}]", i, i); + bytes = flb_lib_push(ctx->flb, ctx->i_ffd, p, strlen(p)); + TEST_CHECK(bytes == strlen(p)); + } + + flb_time_msleep(1500); /* waiting flush */ + got = get_output_num(); + + if (!TEST_CHECK(got != 0)) { + TEST_MSG("callback is not invoked"); + } + + /* Input should be paused since Mem_Buf_Limit is small size. + * So got is less than heavy_loop. + */ + if(!TEST_CHECK(heavy_loop > got)) { + TEST_MSG("expect: %d got: %d", heavy_loop, got); + } + + filter_test_destroy(ctx); +} + +static void flb_test_issue_4793() +{ + struct flb_lib_out_cb cb_data; + struct filter_test *ctx; + int ret; + int not_used = 0; + int loop_max = 4; + int bytes; + int got; + char p[256]; + int i; + + /* Prepare output callback with expected result */ + cb_data.cb = cb_count_msgpack; + cb_data.data = ¬_used; + + /* Create test context */ + ctx = filter_test_create((void *) &cb_data); + if (!ctx) { + exit(EXIT_FAILURE); + } + clear_output_num(); + /* Configure filter */ + ret = flb_filter_set(ctx->flb, ctx->f_ffd, + "Rule", "$destination ^(server)$ updated false", + NULL); + TEST_CHECK(ret == 0); + + /* Configure output */ + ret = flb_output_set(ctx->flb, ctx->o_ffd, "Match", "*", NULL); + TEST_CHECK(ret == 0); + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + + /* emit (loop_max * 2) records */ + for (i = 0; i < loop_max; i++) { + /* "destination": "server" */ + memset(p, '\0', sizeof(p)); + snprintf(p, sizeof(p), "[%d, {\"val\": \"%d\",\"destination\": \"server\"}]", i, i); + bytes = flb_lib_push(ctx->flb, ctx->i_ffd, p, strlen(p)); + TEST_CHECK(bytes == strlen(p)); + + /* "destination": "other" */ + memset(p, '\0', sizeof(p)); + snprintf(p, sizeof(p), "[%d, {\"val\": \"%d\",\"destination\": \"other\"}]", i+1, i+1); + bytes = flb_lib_push(ctx->flb, ctx->i_ffd, p, strlen(p)); + TEST_CHECK(bytes == strlen(p)); + } + + flb_time_msleep(1500); /* waiting flush */ + got = get_output_num(); + + if (!TEST_CHECK(got != 0)) { + TEST_MSG("callback is not invoked"); + } + + if(!TEST_CHECK(2*loop_max == got)) { + TEST_MSG("expect: %d got: %d", 2 * loop_max, got); + } + + filter_test_destroy(ctx); +} + +static void flb_test_issue_4518() +{ + struct flb_lib_out_cb cb_data; + struct filter_test *ctx; + int ret; + int not_used = 0; + int loop_max = 2; + int bytes; + int got; + char p[256]; + int i; + int f_ffd; + + /* Prepare output callback with expected result */ + cb_data.cb = cb_count_msgpack; + cb_data.data = ¬_used; + + /* Create test context */ + ctx = filter_test_create((void *) &cb_data); + if (!ctx) { + exit(EXIT_FAILURE); + } + clear_output_num(); + + /* Configure output */ + ret = flb_output_set(ctx->flb, ctx->o_ffd, + "Match", "*", + NULL); + + /* create 2nd filter */ + f_ffd = flb_filter(ctx->flb, (char *) "rewrite_tag", NULL); + TEST_CHECK(f_ffd >= 0); + flb_filter_set(ctx->flb, f_ffd, "match", "rewrite", NULL); + /* Configure filter */ + ret = flb_filter_set(ctx->flb, f_ffd, + "Rule", "$test3 ^(true)$ updated true", + NULL); + TEST_CHECK(ret == 0); + + /* Configure 1st filter */ + ret = flb_filter_set(ctx->flb, ctx->f_ffd, + "Rule", "$test2 ^(true)$ updated true", + NULL); + TEST_CHECK(ret == 0); + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + for (i = 0; i < loop_max; i++) { + memset(p, '\0', sizeof(p)); + /* 1st filter duplicates below record. */ + snprintf(p, sizeof(p), "[%d, {\"msg\":\"DEBUG\", \"val\": \"%d\",\"test2\": \"true\"}]", i, i); + bytes = flb_lib_push(ctx->flb, ctx->i_ffd, p, strlen(p)); + TEST_CHECK(bytes == strlen(p)); + + /* 2nd filter duplicates below record. */ + memset(p, '\0', sizeof(p)); + snprintf(p, sizeof(p), "[%d, {\"msg\":\"ERROR\", \"val\": \"%d\",\"test3\": \"true\"}]", i, i); + bytes = flb_lib_push(ctx->flb, ctx->i_ffd, p, strlen(p)); + TEST_CHECK(bytes == strlen(p)); + } + + flb_time_msleep(1500); /* waiting flush */ + got = get_output_num(); + + if (!TEST_CHECK(got != 0)) { + TEST_MSG("callback is not invoked"); + } + + /* Output should be 4 * loop_max. + 1st filter appends 1 record and 2nd filter also appends 1 record. + Original 2 records + 1 record(1st filter) + 1 record(2nd filter) = 4 records. + */ + if(!TEST_CHECK(4*loop_max == got)) { + TEST_MSG("expect: %d got: %d", 4 * loop_max, got); + } + + filter_test_destroy(ctx); +} + +/* $TAG as a key of rule causes SIGSEGV */ +static void flb_test_issue_5846() +{ + struct flb_lib_out_cb cb_data; + struct filter_test *ctx; + int ret; + int not_used = 0; + int bytes; + char *p = "[0, {\"key\":\"rewrite\"}]"; + + /* Prepare output callback with expected result */ + cb_data.cb = cb_count_msgpack; + cb_data.data = ¬_used; + + /* Create test context */ + ctx = filter_test_create((void *) &cb_data); + if (!ctx) { + exit(EXIT_FAILURE); + } + clear_output_num(); + /* Configure filter */ + ret = flb_filter_set(ctx->flb, ctx->f_ffd, + "Rule", "$TAG ^(rewrite)$ updated false", + NULL); + TEST_CHECK(ret == 0); + + /* Configure output */ + ret = flb_output_set(ctx->flb, ctx->o_ffd, + "Match", "updated", + NULL); + TEST_CHECK(ret == 0); + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + /* ingest record */ + bytes = flb_lib_push(ctx->flb, ctx->i_ffd, p, strlen(p)); + TEST_CHECK(bytes == strlen(p)); + + flb_time_msleep(1500); /* waiting flush */ + + /* It is OK, if there is no SIGSEGV. */ + + filter_test_destroy(ctx); +} + +TEST_LIST = { + {"matched", flb_test_matched}, + {"not_matched", flb_test_not_matched}, + {"keep_true", flb_test_keep_true}, + {"heavy_input_pause_emitter", flb_test_heavy_input_pause_emitter}, + {"issue_4518", flb_test_issue_4518}, + {"issue_4793", flb_test_issue_4793}, + {"sigsegv_issue_5846", flb_test_issue_5846}, + {NULL, NULL} +}; diff --git a/src/fluent-bit/tests/runtime/filter_stdout.c b/src/fluent-bit/tests/runtime/filter_stdout.c new file mode 100644 index 000000000..835ad8eda --- /dev/null +++ b/src/fluent-bit/tests/runtime/filter_stdout.c @@ -0,0 +1,84 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +#include <fluent-bit.h> +#include "flb_tests_runtime.h" + +/* Test data */ + +/* Test functions */ +void flb_test_filter_stdout_json_multiple(void); +void flb_test_filter_stdout_case_insensitive(void); + +/* Test list */ +TEST_LIST = { + {"json_multiple", flb_test_filter_stdout_json_multiple }, + {"case_insensitive_name", flb_test_filter_stdout_case_insensitive}, + {NULL, NULL} +}; + +/* + * This test case is to check if fluent-bit allows case-insensitive plugin name. + * This test is not unique to filter_stdout, but we test here :) , + */ + +void flb_test_filter_stdout_case_insensitive(void) +{ + int filter_ffd; + char filter_name[] = "stDoUt"; + flb_ctx_t *ctx; + + ctx = flb_create(); + + filter_ffd = flb_filter(ctx, (char *) filter_name, NULL); + if(!TEST_CHECK(filter_ffd >= 0)) { + TEST_MSG("%s should be valid\n", filter_name); + } + + /* Initialize thread local storage (FLB_TLS) properly when without calling flb_start(). + * Then, FLB_TLS_GET working on macOS. + * In general, macOS requests surely initialization for pthread stuffs. + */ + flb_init_env(); + flb_destroy(ctx); +} + +void flb_test_filter_stdout_json_multiple(void) +{ + int i; + int ret; + int bytes; + char p[100]; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + int filter_ffd; + + ctx = flb_create(); + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + out_ffd = flb_output(ctx, (char *) "null", NULL); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, "match", "test", NULL); + + filter_ffd = flb_filter(ctx, (char *) "stdout", NULL); + TEST_CHECK(filter_ffd >= 0); + flb_filter_set(ctx, filter_ffd, "match", "test", NULL); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + for (i = 0; i < 256; i++) { + memset(p, '\0', sizeof(p)); + snprintf(p, sizeof(p), "[%d, {\"val\": %d,\"END_KEY\": \"JSON_END\"}]", i, (i * i)); + bytes = flb_lib_push(ctx, in_ffd, p, strlen(p)); + TEST_CHECK(bytes == strlen(p)); + } + + sleep(1); /* waiting flush */ + + flb_stop(ctx); + flb_destroy(ctx); +} diff --git a/src/fluent-bit/tests/runtime/filter_throttle.c b/src/fluent-bit/tests/runtime/filter_throttle.c new file mode 100644 index 000000000..7654e6fe8 --- /dev/null +++ b/src/fluent-bit/tests/runtime/filter_throttle.c @@ -0,0 +1,105 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +#include <fluent-bit.h> +#include "flb_tests_runtime.h" + +/* Test data */ + +/* Utility functions */ +pthread_mutex_t result_mutex = PTHREAD_MUTEX_INITIALIZER; + +/* Test functions */ +void flb_test_filter_throttle(void); +void flb_test_filter_window_0(void); + +/* Test list */ +TEST_LIST = { + {"throttle", flb_test_filter_throttle }, + {"window_0", flb_test_filter_window_0 }, + {NULL, NULL} +}; + + +void flb_test_filter_throttle(void) +{ + int i; + int ret; + int bytes; + char p[100]; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + int filter_ffd; + + ctx = flb_create(); + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + out_ffd = flb_output(ctx, (char *) "stdout", NULL); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, "match", "test", NULL); + + filter_ffd = flb_filter(ctx, (char *) "throttle", NULL); + TEST_CHECK(filter_ffd >= 0); + ret = flb_filter_set(ctx, filter_ffd, "match", "*", NULL); + TEST_CHECK(ret == 0); + ret = flb_filter_set(ctx, filter_ffd, "rate", "9", NULL); + TEST_CHECK(ret == 0); + ret = flb_filter_set(ctx, filter_ffd, "window", "3", NULL); + TEST_CHECK(ret == 0); + ret = flb_filter_set(ctx, filter_ffd, "interval", "3s", NULL); + TEST_CHECK(ret == 0); + ret = flb_filter_set(ctx, filter_ffd, "print_status", "true", NULL); + TEST_CHECK(ret == 0); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Send log messages all should go through */ + for (i = 0; i < 256; i++) { + memset(p, '\0', sizeof(p)); + snprintf(p, sizeof(p), "[%d, {\"val\": \"%d\",\"END_KEY\": \"JSON_END\"}]", i, i); + bytes = flb_lib_push(ctx, in_ffd, p, strlen(p)); + TEST_CHECK(bytes == strlen(p)); + } + + sleep(1); /* waiting flush */ + + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_filter_window_0(void) +{ + int ret; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + int filter_ffd; + + ctx = flb_create(); + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + out_ffd = flb_output(ctx, (char *) "stdout", NULL); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, "match", "test", NULL); + + filter_ffd = flb_filter(ctx, (char *) "throttle", NULL); + TEST_CHECK(filter_ffd >= 0); + ret = flb_filter_set(ctx, filter_ffd, "match", "*", + "window", "0", + NULL); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + sleep(1); /* waiting flush */ + + flb_stop(ctx); + flb_destroy(ctx); +} diff --git a/src/fluent-bit/tests/runtime/filter_throttle_size.c b/src/fluent-bit/tests/runtime/filter_throttle_size.c new file mode 100644 index 000000000..b3f7c3c8c --- /dev/null +++ b/src/fluent-bit/tests/runtime/filter_throttle_size.c @@ -0,0 +1,608 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +#include <fluent-bit.h> +#include "flb_tests_runtime.h" +#include <stdio.h> + +#define FLUSH_DURATION 1 +#define WAIT_FOR_FLUSH sleep(FLUSH_DURATION); + +/* Test data */ +static const char *simple_log = + "[1519233660.000000, {\"log\":\"%s\", \"stream\":\"%s\", \"time\":\"2018-02-21T17:26:53.360920913Z\", \"value\":\"%d\"}]"; +static const char *simple_log_with_missing_log_key = + "[1519233660.000000, {\"msg\":\"%s\", \"stream\":\"%s\", \"time\":\"2018-02-21T17:26:53.360920913Z\", \"value\":\"%d\"}]"; +static const char *simple_log_with_missing_stream_key = + "[1519233660.000000, {\"log\":\"%s\", \"source\":\"%s\", \"time\":\"2018-02-21T17:26:53.360920913Z\", \"value\":\"%d\"}]"; +static const char *nested_log = + "[1519234013.360921, {\"logwrapper\":{\"log\":\"%s\", \"extra\":\"field\"}, \"stream\":\"stdout\", \"time\":\"2018-02-21T17:26:53.360920913Z\", \"kubernetes\":{\"pod_name\":\"%s\", \"namespace_name\":\"default\", \"pod_id\":\"64f7da23-172c-11e8-bfad-080027749cbc\", \"labels\":{\"run\":\"apache-logs\"}, \"host\":\"minikube\", \"container_name\":\"apache-logs\", \"docker_id\":\"ac6095b6c715d823d732dcc9067f75b1299de5cc69a012b08d616a6058bdc0ad\"}, \"va\":\"%d\"}]"; +static const char *swap_log_field = + "[1519234013.360921, {\"log\":{\"logwrapper\":\"%s\", \"extra\":\"field\"}, \"stream\":\"stdout\", \"time\":\"2018-02-21T17:26:53.360920913Z\", \"kubernetes\":{\"pod_name\":\"%s\", \"namespace_name\":\"default\", \"pod_id\":\"64f7da23-172c-11e8-bfad-080027749cbc\", \"labels\":{\"run\":\"apache-logs\"}, \"host\":\"minikube\", \"container_name\":\"apache-logs\", \"docker_id\":\"ac6095b6c715d823d732dcc9067f75b1299de5cc69a012b08d616a6058bdc0ad\"}, \"va\":\"%d\"}]"; +static const char *swap_name_field = + "[1519234013.360921, {\"logwrapper\":{\"log\":\"%s\", \"extra\":\"field\"}, \"stream\":\"stdout\", \"time\":\"2018-02-21T17:26:53.360920913Z\", \"pod_name\":{\"kubernetes\":\"%s\", \"namespace_name\":\"default\", \"pod_id\":\"64f7da23-172c-11e8-bfad-080027749cbc\", \"labels\":{\"run\":\"apache-logs\"}, \"host\":\"minikube\", \"container_name\":\"apache-logs\", \"docker_id\":\"ac6095b6c715d823d732dcc9067f75b1299de5cc69a012b08d616a6058bdc0ad\"}, \"va\":\"%d\"}]"; +static const char *nested_log_with_missing_log_field = + "[1519234013.360921, {\"logwrapper\":{\"msg\":\"%s\", \"extra\":\"field\"}, \"stream\":\"stdout\", \"time\":\"2018-02-21T17:26:53.360920913Z\", \"kubernetes\":{\"pod_name\":\"%s\", \"namespace_name\":\"default\", \"pod_id\":\"64f7da23-172c-11e8-bfad-080027749cbc\", \"labels\":{\"run\":\"apache-logs\"}, \"host\":\"minikube\", \"container_name\":\"apache-logs\", \"docker_id\":\"ac6095b6c715d823d732dcc9067f75b1299de5cc69a012b08d616a6058bdc0ad\"}, \"va\":\"%d\"}]"; +static const char *nested_log_with_missing_name_field = + "[1519234013.360921, {\"logwrapper\":{\"log\":\"%s\", \"extra\":\"field\"}, \"stream\":\"stdout\", \"time\":\"2018-02-21T17:26:53.360920913Z\", \"kubernetes\":{\"pod_alias\":\"%s\", \"namespace_name\":\"default\", \"pod_id\":\"64f7da23-172c-11e8-bfad-080027749cbc\", \"labels\":{\"run\":\"apache-logs\"}, \"host\":\"minikube\", \"container_name\":\"apache-logs\", \"docker_id\":\"ac6095b6c715d823d732dcc9067f75b1299de5cc69a012b08d616a6058bdc0ad\"}, \"va\":\"%d\"}]"; +static const char *_32_bytes_msg = "This meeesage is 32 symbols long"; +static const char *_11_bytes_msg = "I will pass"; +static const char *_6_bytes_msg = "I pass"; +static const char *_180_bytes_msg = + "This message is 180 bytes long, so it will be used where we are sure that this message will pass every time or not at all. So used it carefully and at the proper place. Understand?"; +static const char *stdout_str = "stdout"; +static const char *stderr_str = "stderr"; +static const char *apiserver = "kube-apiserver"; +static const char *alertmanager = "alertmanager"; + +/* Utility functions */ +char *push_data_to_engine_and_take_output(flb_ctx_t * ctx, int in_ffd, + char *message); +void check_if_message_pass_through_engine(flb_ctx_t * ctx, int in_ffd, + char *message); +void check_if_message_doesnt_pass_through_engine(flb_ctx_t * ctx, int in_ffd, + char *message); +pthread_mutex_t result_mutex = PTHREAD_MUTEX_INITIALIZER; +char *output = NULL; + + + +/* Test functions */ + +void flb_test_simple_log(void); +void test_nestest_name_fields(void); +void test_default_name_field(void); +void test_default_log_field(void); + + +/* Test list */ +TEST_LIST = { + { + "throttle_size", flb_test_simple_log}, { + "throttle_size2", test_nestest_name_fields}, { + "throttle_size3", test_default_name_field}, { + "throttle_size4", test_default_log_field}, { + NULL, NULL} +}; + +void set_output(char *val) +{ + pthread_mutex_lock(&result_mutex); + output = val; + pthread_mutex_unlock(&result_mutex); +} + +char *get_output(void) +{ + char *val; + + pthread_mutex_lock(&result_mutex); + val = output; + output = NULL; + pthread_mutex_unlock(&result_mutex); + + return val; +} + +int callback_test(void *data, size_t size, void *cb_data) +{ + if (size > 0) { + flb_debug("[test_filter_throttle_size] received message: %s", data); + set_output(data); /* success */ + } + return 0; +} + +void flb_test_simple_log(void) +{ + int i; + int ret; + char p[1000]; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + int filter_ffd; + + + struct flb_lib_out_cb cb; + cb.cb = callback_test; + cb.data = NULL; + + ctx = flb_create(); + + /* Configure service */ + flb_service_set(ctx, "Flush", "1", "Grace" "1", "Log_Level", "debug", + NULL); + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + /* Output */ + out_ffd = flb_output(ctx, (char *) "lib", &cb); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, "Match", "*", "format", "json", NULL); + + filter_ffd = flb_filter(ctx, (char *) "throttle_size", NULL); + TEST_CHECK(filter_ffd >= 0); + ret = flb_filter_set(ctx, filter_ffd, "match", "*", NULL); + TEST_CHECK(ret == 0); + ret = flb_filter_set(ctx, filter_ffd, "rate", "10", NULL); + TEST_CHECK(ret == 0); + ret = flb_filter_set(ctx, filter_ffd, "window", "30", NULL); + TEST_CHECK(ret == 0); + ret = flb_filter_set(ctx, filter_ffd, "interval", "3s", NULL); + TEST_CHECK(ret == 0); + ret = flb_filter_set(ctx, filter_ffd, "print_status", "true", NULL); + TEST_CHECK(ret == 0); + ret = flb_filter_set(ctx, filter_ffd, "log_field", "log", NULL); + TEST_CHECK(ret == 0); + ret = flb_filter_set(ctx, filter_ffd, "name_field", "stream", NULL); + TEST_CHECK(ret == 0); + ret = + flb_filter_set(ctx, filter_ffd, "window_time_duration", "10s", NULL); + TEST_CHECK(ret == 0); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Verify that the size throttle plugin differentiates logs by a non nested name_field. + We put 9 logs 32 bytes long which is 288 bytes of total or rate of 9.6. + If all logs passed this means that the the plugin sees them as two seperates tipes or + does now work at all.Or each logs is seen as different group of logs. */ + for (i = 0; i < 9; i++) { + /*Make message with sream: stdout */ + memset(p, '\0', sizeof(p)); + snprintf(p, sizeof(p), simple_log, _32_bytes_msg, stdout_str, i); + check_if_message_pass_through_engine(ctx, in_ffd, p); + + /*Make message with sream: stderr */ + memset(p, '\0', sizeof(p)); + snprintf(p, sizeof(p), simple_log, _32_bytes_msg, stderr_str, i); + check_if_message_pass_through_engine(ctx, in_ffd, p); + } + + /* Verify that the plugin cut logs wen rate exceeds 10. + By add next message which is 32 bytes log the total must become 320 which + makes the rate 10.66. If the messege is dropped this means that the plugin + works properly. */ + memset(p, '\0', sizeof(p)); + snprintf(p, sizeof(p), simple_log, _32_bytes_msg, stdout_str, 9); + check_if_message_doesnt_pass_through_engine(ctx, in_ffd, p); + + + memset(p, '\0', sizeof(p)); + snprintf(p, sizeof(p), simple_log, _32_bytes_msg, stderr_str, 9); + check_if_message_doesnt_pass_through_engine(ctx, in_ffd, p); + + /*Now we will pass two messages with 11 bytes of lenght an they will make the + rate 9.96 which is less than 10 and they must pass */ + memset(p, '\0', sizeof(p)); + snprintf(p, sizeof(p), simple_log, _11_bytes_msg, stdout_str, 10); + check_if_message_pass_through_engine(ctx, in_ffd, p); + + memset(p, '\0', sizeof(p)); + snprintf(p, sizeof(p), simple_log, _11_bytes_msg, stderr_str, 10); + check_if_message_pass_through_engine(ctx, in_ffd, p); + + /*check that if log field is missing then the messages will pass throughout the engine */ + for (i = 0; i < 2; i++) { + /*Make message with sream: stdout */ + memset(p, '\0', sizeof(p)); + snprintf(p, sizeof(p), simple_log_with_missing_log_key, + _180_bytes_msg, stdout_str, i + 11); + check_if_message_pass_through_engine(ctx, in_ffd, p); + + /*Make message with sream: stderr */ + memset(p, '\0', sizeof(p)); + snprintf(p, sizeof(p), simple_log_with_missing_log_key, + _180_bytes_msg, stderr_str, i + 11); + check_if_message_pass_through_engine(ctx, in_ffd, p); + } + + /*check that if stream field is missing then the messages will pass throughout the engine */ + for (i = 0; i < 2; i++) { + /*Make message with sream: stdout */ + memset(p, '\0', sizeof(p)); + snprintf(p, sizeof(p), simple_log_with_missing_stream_key, + _180_bytes_msg, stdout_str, i + 13); + check_if_message_pass_through_engine(ctx, in_ffd, p); + + /*Make message with sream: stderr */ + memset(p, '\0', sizeof(p)); + snprintf(p, sizeof(p), simple_log_with_missing_stream_key, + _180_bytes_msg, stderr_str, i + 13); + check_if_message_pass_through_engine(ctx, in_ffd, p); + } + + flb_stop(ctx); + flb_destroy(ctx); +} + +void test_nestest_name_fields(void) +{ + int i; + int ret; + char p[1000]; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + int filter_ffd; + + + struct flb_lib_out_cb cb; + cb.cb = callback_test; + cb.data = NULL; + + ctx = flb_create(); + + /* Configure service */ + flb_service_set(ctx, "Flush", "1", "Grace" "1", "Log_Level", "debug", + NULL); + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + /* Output */ + out_ffd = flb_output(ctx, (char *) "lib", &cb); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, "Match", "*", "format", "json", NULL); + + filter_ffd = flb_filter(ctx, (char *) "throttle_size", NULL); + TEST_CHECK(filter_ffd >= 0); + ret = flb_filter_set(ctx, filter_ffd, "match", "*", NULL); + TEST_CHECK(ret == 0); + ret = flb_filter_set(ctx, filter_ffd, "rate", "10", NULL); + TEST_CHECK(ret == 0); + ret = flb_filter_set(ctx, filter_ffd, "window", "30", NULL); + TEST_CHECK(ret == 0); + ret = flb_filter_set(ctx, filter_ffd, "interval", "3s", NULL); + TEST_CHECK(ret == 0); + ret = flb_filter_set(ctx, filter_ffd, "print_status", "false", NULL); + TEST_CHECK(ret == 0); + ret = + flb_filter_set(ctx, filter_ffd, "log_field", "logwrapper|log", NULL); + TEST_CHECK(ret == 0); + ret = + flb_filter_set(ctx, filter_ffd, "name_field", "kubernetes|pod_name", + NULL); + TEST_CHECK(ret == 0); + ret = + flb_filter_set(ctx, filter_ffd, "window_time_duration", "10s", NULL); + TEST_CHECK(ret == 0); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Verify that the size throttle plugin differentiates logs by a nested name_field and nested log_field. + We put 9 logs 32 bytes long which is 288 bytes of total or rate of 9.6. + If all logs passed this means that the the plugin sees them as two seperates tipes or + does now work at all.Or each logs is seen as different group of logs. */ + for (i = 0; i < 9; i++) { + /*Make message with sream: stdout */ + memset(p, '\0', sizeof(p)); + snprintf(p, sizeof(p), nested_log, _32_bytes_msg, apiserver, i); + check_if_message_pass_through_engine(ctx, in_ffd, p); + + /*Make message with kubernetes.pod_name: alermanager */ + memset(p, '\0', sizeof(p)); + snprintf(p, sizeof(p), nested_log, _32_bytes_msg, alertmanager, i); + check_if_message_pass_through_engine(ctx, in_ffd, p); + } + + /* Verify that the plugin cut logs when rate exceeds 10. + By add next message which is 32 bytes log the total must become 320 which + makes the rate 10.66. If the messege is dropped this means that the plugin + works properly. */ + memset(p, '\0', sizeof(p)); + snprintf(p, sizeof(p), nested_log, _32_bytes_msg, apiserver, 9); + check_if_message_doesnt_pass_through_engine(ctx, in_ffd, p); + + + memset(p, '\0', sizeof(p)); + snprintf(p, sizeof(p), nested_log, _32_bytes_msg, alertmanager, 9); + check_if_message_doesnt_pass_through_engine(ctx, in_ffd, p); + + /*Now we will pass two messages with 11 bytes of lenght an they will make the + rate 9.96 which is less than 10 and they must pass */ + memset(p, '\0', sizeof(p)); + snprintf(p, sizeof(p), nested_log, _11_bytes_msg, apiserver, 10); + check_if_message_pass_through_engine(ctx, in_ffd, p); + + memset(p, '\0', sizeof(p)); + snprintf(p, sizeof(p), nested_log, _11_bytes_msg, alertmanager, 10); + check_if_message_pass_through_engine(ctx, in_ffd, p); + + /*check that if log field is missing then the messages will pass throughout the engine */ + for (i = 0; i < 2; i++) { + /*Make message with kubernetes.pod_name: kube-apiserver */ + memset(p, '\0', sizeof(p)); + snprintf(p, sizeof(p), nested_log_with_missing_log_field, + _180_bytes_msg, apiserver, i + 11); + check_if_message_pass_through_engine(ctx, in_ffd, p); + + /*Make message with kubernetes.pod_name: alertmanager */ + memset(p, '\0', sizeof(p)); + snprintf(p, sizeof(p), nested_log_with_missing_log_field, + _180_bytes_msg, alertmanager, i + 11); + check_if_message_pass_through_engine(ctx, in_ffd, p); + } + + /*check that if pod_name field is missing then the messages will pass throughout the engine */ + for (i = 0; i < 2; i++) { + /*Make message with kubernetes.pod_name: kube-apiserver */ + memset(p, '\0', sizeof(p)); + snprintf(p, sizeof(p), nested_log_with_missing_name_field, + _180_bytes_msg, apiserver, i + 13); + check_if_message_pass_through_engine(ctx, in_ffd, p); + + /*Make message with kubernetes.pod_name: alermanager */ + memset(p, '\0', sizeof(p)); + snprintf(p, sizeof(p), nested_log_with_missing_name_field, + _180_bytes_msg, alertmanager, i + 13); + check_if_message_pass_through_engine(ctx, in_ffd, p); + } + + /*check that if pod_name is not in the right order then the messages will pass throughout the engine */ + for (i = 0; i < 2; i++) { + /*Make message with kubernetes.pod_name: kube-apiserver */ + memset(p, '\0', sizeof(p)); + snprintf(p, sizeof(p), swap_name_field, _180_bytes_msg, apiserver, + i + 15); + check_if_message_pass_through_engine(ctx, in_ffd, p); + + /*Make message with kubernets.pod_name: alertmanager */ + memset(p, '\0', sizeof(p)); + snprintf(p, sizeof(p), swap_name_field, _180_bytes_msg, alertmanager, + i + 15); + check_if_message_pass_through_engine(ctx, in_ffd, p); + } + + /*check that if log field is wrong order then the messages will pass throughout the engine */ + for (i = 0; i < 2; i++) { + /*Make message with kubernetes.pod_name: kube-apiserver */ + memset(p, '\0', sizeof(p)); + snprintf(p, sizeof(p), swap_log_field, _180_bytes_msg, apiserver, + i + 17); + check_if_message_pass_through_engine(ctx, in_ffd, p); + + /*Make message with kubernetes: alertmanager */ + memset(p, '\0', sizeof(p)); + snprintf(p, sizeof(p), swap_log_field, _180_bytes_msg, alertmanager, + i + 17); + check_if_message_pass_through_engine(ctx, in_ffd, p); + } + + flb_stop(ctx); + flb_destroy(ctx); +} + +void test_default_name_field(void) +{ + int i; + int ret; + char p[200]; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + int filter_ffd; + + struct flb_lib_out_cb cb; + cb.cb = callback_test; + cb.data = NULL; + + ctx = flb_create(); + + /* Configure service */ + flb_service_set(ctx, "Flush", "1", "Grace" "1", "Log_Level", "debug", + NULL); + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + /* Output */ + out_ffd = flb_output(ctx, (char *) "lib", &cb); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, "Match", "*", "format", "json", NULL); + + filter_ffd = flb_filter(ctx, (char *) "throttle_size", NULL); + TEST_CHECK(filter_ffd >= 0); + ret = flb_filter_set(ctx, filter_ffd, "match", "*", NULL); + TEST_CHECK(ret == 0); + ret = flb_filter_set(ctx, filter_ffd, "rate", "10", NULL); + TEST_CHECK(ret == 0); + ret = flb_filter_set(ctx, filter_ffd, "window", "30", NULL); + TEST_CHECK(ret == 0); + ret = flb_filter_set(ctx, filter_ffd, "interval", "3s", NULL); + TEST_CHECK(ret == 0); + ret = flb_filter_set(ctx, filter_ffd, "print_status", "false", NULL); + TEST_CHECK(ret == 0); + ret = flb_filter_set(ctx, filter_ffd, "log_field", "log", NULL); + TEST_CHECK(ret == 0); + ret = + flb_filter_set(ctx, filter_ffd, "window_time_duration", "10s", NULL); + TEST_CHECK(ret == 0); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Verify that the size throttle plugin do not differentiates logs by name_field. + We put 8 logs 32 bytes long which is 256 bytes of total or rate of 8.53. + If all logs passed this means that the the plugin sees them as one or + does now work at all.Or each logs is seen as different group of logs. */ + for (i = 0; i < 4; i++) { + /*Make message with sream: stdout */ + memset(p, '\0', sizeof(p)); + snprintf(p, sizeof(p), simple_log, _32_bytes_msg, stdout_str, i); + check_if_message_pass_through_engine(ctx, in_ffd, p); + + /*Make message with sream: stderr */ + memset(p, '\0', sizeof(p)); + snprintf(p, sizeof(p), simple_log, _32_bytes_msg, stderr_str, i); + check_if_message_pass_through_engine(ctx, in_ffd, p); + } + + /*Add one exra message with lenght of 32 bytes to make the total 288 or rate of 9.6 */ + memset(p, '\0', sizeof(p)); + snprintf(p, sizeof(p), simple_log, _32_bytes_msg, stdout_str, 9); + check_if_message_pass_through_engine(ctx, in_ffd, p); + + /* Verify that the plugin cut logs when rate exceeds 10. + By add next message which is 32 bytes log the total must become 320 which + makes the rate 10.66. If the messege is dropped this means that the plugin + works properly. */ + memset(p, '\0', sizeof(p)); + snprintf(p, sizeof(p), simple_log, _32_bytes_msg, stdout_str, 9); + check_if_message_doesnt_pass_through_engine(ctx, in_ffd, p); + /* Verify that the plugin cut logs when rate exceeds 10. + By add next message which is 32 bytes log the total must become 320 which + makes the rate 10.66. If the messege is dropped this means that the plugin + works properly. */ + memset(p, '\0', sizeof(p)); + snprintf(p, sizeof(p), simple_log, _32_bytes_msg, stderr_str, 9); + check_if_message_doesnt_pass_through_engine(ctx, in_ffd, p); + + + /*Now we will pass two messages with 6 bytes of lenght an they will make the + rate 10 whch is the limit and the message must pass. */ + memset(p, '\0', sizeof(p)); + snprintf(p, sizeof(p), simple_log, _6_bytes_msg, stdout_str, 10); + check_if_message_pass_through_engine(ctx, in_ffd, p); + + memset(p, '\0', sizeof(p)); + snprintf(p, sizeof(p), simple_log, _6_bytes_msg, stderr_str, 10); + check_if_message_pass_through_engine(ctx, in_ffd, p); + + + flb_stop(ctx); + flb_destroy(ctx); +} + + +void test_default_log_field(void) +{ + int i; + int ret; + char p[1000]; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + int filter_ffd; + + + struct flb_lib_out_cb cb; + cb.cb = callback_test; + cb.data = NULL; + + ctx = flb_create(); + + /* Configure service */ + flb_service_set(ctx, "Flush", "1", "Grace" "1", "Log_Level", "debug", + NULL); + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + /* Output */ + out_ffd = flb_output(ctx, (char *) "lib", &cb); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, "Match", "*", "format", "json", NULL); + + filter_ffd = flb_filter(ctx, (char *) "throttle_size", NULL); + TEST_CHECK(filter_ffd >= 0); + ret = flb_filter_set(ctx, filter_ffd, "match", "*", NULL); + TEST_CHECK(ret == 0); + ret = flb_filter_set(ctx, filter_ffd, "rate", "43", NULL); + TEST_CHECK(ret == 0); + ret = flb_filter_set(ctx, filter_ffd, "window", "30", NULL); + TEST_CHECK(ret == 0); + ret = flb_filter_set(ctx, filter_ffd, "interval", "3s", NULL); + TEST_CHECK(ret == 0); + ret = + flb_filter_set(ctx, filter_ffd, "name_field", "kubernetes|pod_name", + NULL); + TEST_CHECK(ret == 0); + ret = flb_filter_set(ctx, filter_ffd, "print_status", "false", NULL); + TEST_CHECK(ret == 0); + ret = + flb_filter_set(ctx, filter_ffd, "window_time_duration", "10s", NULL); + TEST_CHECK(ret == 0); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Verify that fluent-bit take in account all of the message payload when log_field is missing. + We shall put two messages with differen kubernetes.podname which will pass. + One message is about 463 bytes long and two makes the rate about 30.87 */ + for (i = 0; i < 2; i++) { + /*Make message with kubernetes.pod_name:kube-apiserver */ + memset(p, '\0', sizeof(p)); + snprintf(p, sizeof(p), nested_log, _180_bytes_msg, apiserver, 1); + check_if_message_pass_through_engine(ctx, in_ffd, p); + + /*Make message with kubernetes.pod_name:alermanager */ + memset(p, '\0', sizeof(p)); + snprintf(p, sizeof(p), nested_log, _180_bytes_msg, alertmanager, 1); + check_if_message_pass_through_engine(ctx, in_ffd, p); + } + + /* Verify that the plugin cut logs when rate exceeds 37. + We shall add again two messages with size 463 and they will + fail passing. */ + memset(p, '\0', sizeof(p)); + snprintf(p, sizeof(p), nested_log, _180_bytes_msg, apiserver, 1); + check_if_message_doesnt_pass_through_engine(ctx, in_ffd, p); + + /*Make message with kubernetes.pod_name:alermanager */ + memset(p, '\0', sizeof(p)); + snprintf(p, sizeof(p), nested_log, _180_bytes_msg, alertmanager, 1); + check_if_message_doesnt_pass_through_engine(ctx, in_ffd, p); + + /*The next two must pass */ + /*Make message with kubernetes.pod_name:kube-apiserver */ + memset(p, '\0', sizeof(p)); + snprintf(p, sizeof(p), nested_log, _32_bytes_msg, apiserver, 1); + check_if_message_pass_through_engine(ctx, in_ffd, p); + + /*Make message with kubernetes.pod_name:alermanager */ + memset(p, '\0', sizeof(p)); + snprintf(p, sizeof(p), nested_log, _32_bytes_msg, alertmanager, 1); + check_if_message_pass_through_engine(ctx, in_ffd, p); + + + flb_stop(ctx); + flb_destroy(ctx); +} + +char *push_data_to_engine_and_take_output(flb_ctx_t * ctx, int in_ffd, + char *message) +{ + char *result = NULL; + int bytes; + /*Push the message into the engine */ + bytes = flb_lib_push(ctx, in_ffd, (void *) message, strlen(message)); + WAIT_FOR_FLUSH /*wait the output data to be flushed */ + result = get_output(); /*get the output message */ + TEST_CHECK(bytes == strlen(message)); /*Chech if all of the message was proceesed */ + return result; +} + +void check_if_message_pass_through_engine(flb_ctx_t * ctx, int in_ffd, + char *message) +{ + char *result; + result = push_data_to_engine_and_take_output(ctx, in_ffd, message); + /*Check that the message go throught engine without modification */ + TEST_CHECK(strncmp(result, message, strlen(result)) == 0); +} + +void check_if_message_doesnt_pass_through_engine(flb_ctx_t * ctx, int in_ffd, + char *message) +{ + char *result; + result = push_data_to_engine_and_take_output(ctx, in_ffd, message); + /*Check that the message didn't throught engine */ + TEST_CHECK(result == NULL); +} diff --git a/src/fluent-bit/tests/runtime/filter_type_converter.c b/src/fluent-bit/tests/runtime/filter_type_converter.c new file mode 100644 index 000000000..da2556e7c --- /dev/null +++ b/src/fluent-bit/tests/runtime/filter_type_converter.c @@ -0,0 +1,389 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2019-2021 The Fluent Bit Authors + * Copyright (C) 2015-2018 Treasure Data Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <fluent-bit.h> +#include <fluent-bit/flb_time.h> +#include "flb_tests_runtime.h" + +#define JSON_TEST_DATA "[12345678, {\"numstr\":\"123.456\", \"int\":123, \"float\":123.456, \"hexstr\":\"0xff\"}]" +#define JSON_NEST_DATA "[12345678, {\"nest\":{\"numstr\":\"123.456\", \"float\":123.456}}]" + +struct filter_test { + flb_ctx_t *flb; /* Fluent Bit library context */ + int i_ffd; /* Input fd */ + int f_ffd; /* Filter fd */ +}; + +/* Callback to check expected results */ +static int cb_check_result(void *record, size_t size, void *data) +{ + char *p; + char *expected; + char *result; + + expected = (char *) data; + result = (char *) record; + + p = strstr(result, expected); + TEST_CHECK(p != NULL); + + if (p==NULL) { + flb_error("Expected to find: '%s' in result '%s'", + expected, result); + } + /* + * If you want to debug your test + * + * printf("Expect: '%s' in result '%s'", expected, result); + */ + flb_free(record); + return 0; +} + +static struct filter_test *filter_test_create(struct flb_lib_out_cb *data) +{ + int i_ffd; + int f_ffd; + int o_ffd; + struct filter_test *ctx; + + ctx = flb_malloc(sizeof(struct filter_test)); + if (!ctx) { + flb_errno(); + return NULL; + } + + /* Service config */ + ctx->flb = flb_create(); + flb_service_set(ctx->flb, + "Flush", "0.200000000", + "Grace", "1", + "Log_Level", "error", + NULL); + + /* Input */ + i_ffd = flb_input(ctx->flb, (char *) "lib", NULL); + TEST_CHECK(i_ffd >= 0); + flb_input_set(ctx->flb, i_ffd, "tag", "test", NULL); + ctx->i_ffd = i_ffd; + + /* Filter configuration */ + f_ffd = flb_filter(ctx->flb, (char *) "type_converter", NULL); + TEST_CHECK(f_ffd >= 0); + flb_filter_set(ctx->flb, f_ffd, "match", "*", NULL); + ctx->f_ffd = f_ffd; + + /* Output */ + o_ffd = flb_output(ctx->flb, (char *) "lib", (void *) data); + TEST_CHECK(o_ffd >= 0); + flb_output_set(ctx->flb, o_ffd, + "match", "test", + "format", "json", + NULL); + + return ctx; +} + +static void filter_test_destroy(struct filter_test *ctx) +{ + sleep(1); + flb_stop(ctx->flb); + flb_destroy(ctx->flb); + flb_free(ctx); +} + +void flb_test_str_to_int() +{ + struct flb_lib_out_cb cb_data; + struct filter_test *ctx; + int len; + int ret; + int bytes; + char *p = JSON_TEST_DATA; + + /* Prepare output callback with expected result */ + cb_data.cb = cb_check_result; + cb_data.data = "\"new\":123"; + + /* Create test context */ + ctx = filter_test_create((void *) &cb_data); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("failed to create context"); + exit(EXIT_FAILURE); + } + + /* Configure filter */ + ret = flb_filter_set(ctx->flb, ctx->f_ffd, + "str_key", "numstr new int", + NULL); + TEST_CHECK(ret == 0); + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + /* Ingest data samples */ + len = strlen(p); + bytes = flb_lib_push(ctx->flb, ctx->i_ffd, p, len); + TEST_CHECK(bytes == len); + + filter_test_destroy(ctx); +} + +void flb_test_str_to_hex() +{ + struct flb_lib_out_cb cb_data; + struct filter_test *ctx; + int len; + int ret; + int bytes; + char *p = JSON_TEST_DATA; + + /* Prepare output callback with expected result */ + cb_data.cb = cb_check_result; + cb_data.data = "\"new\":255"; + + /* Create test context */ + ctx = filter_test_create((void *) &cb_data); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("failed to create context"); + exit(EXIT_FAILURE); + } + + /* Configure filter */ + ret = flb_filter_set(ctx->flb, ctx->f_ffd, + "str_key", "hexstr new hex", + NULL); + TEST_CHECK(ret == 0); + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + /* Ingest data samples */ + len = strlen(p); + bytes = flb_lib_push(ctx->flb, ctx->i_ffd, p, len); + TEST_CHECK(bytes == len); + + filter_test_destroy(ctx); +} + +void flb_test_str_to_float() +{ + struct flb_lib_out_cb cb_data; + struct filter_test *ctx; + int len; + int ret; + int bytes; + char *p = JSON_TEST_DATA; + + /* Prepare output callback with expected result */ + cb_data.cb = cb_check_result; + cb_data.data = "\"new\":123.456"; + + /* Create test context */ + ctx = filter_test_create((void *) &cb_data); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("failed to create context"); + exit(EXIT_FAILURE); + } + + /* Configure filter */ + ret = flb_filter_set(ctx->flb, ctx->f_ffd, + "str_key", "numstr new float", + NULL); + TEST_CHECK(ret == 0); + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + /* Ingest data samples */ + len = strlen(p); + bytes = flb_lib_push(ctx->flb, ctx->i_ffd, p, len); + TEST_CHECK(bytes == len); + + filter_test_destroy(ctx); +} + +void flb_test_int_to_str() +{ + struct flb_lib_out_cb cb_data; + struct filter_test *ctx; + int len; + int ret; + int bytes; + char *p = JSON_TEST_DATA; + + /* Prepare output callback with expected result */ + cb_data.cb = cb_check_result; + cb_data.data = "\"new\":\"123\""; + + /* Create test context */ + ctx = filter_test_create((void *) &cb_data); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("failed to create context"); + exit(EXIT_FAILURE); + } + + /* Configure filter */ + ret = flb_filter_set(ctx->flb, ctx->f_ffd, + "int_key", "int new str", + NULL); + TEST_CHECK(ret == 0); + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + /* Ingest data samples */ + len = strlen(p); + bytes = flb_lib_push(ctx->flb, ctx->i_ffd, p, len); + TEST_CHECK(bytes == len); + + filter_test_destroy(ctx); +} + +void flb_test_int_to_float() +{ + struct flb_lib_out_cb cb_data; + struct filter_test *ctx; + int len; + int ret; + int bytes; + char *p = JSON_TEST_DATA; + + /* Prepare output callback with expected result */ + cb_data.cb = cb_check_result; + cb_data.data = "\"new\":123."; + + /* Create test context */ + ctx = filter_test_create((void *) &cb_data); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("failed to create context"); + exit(EXIT_FAILURE); + } + + /* Configure filter */ + ret = flb_filter_set(ctx->flb, ctx->f_ffd, + "int_key", "int new float", + NULL); + TEST_CHECK(ret == 0); + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + /* Ingest data samples */ + len = strlen(p); + bytes = flb_lib_push(ctx->flb, ctx->i_ffd, p, len); + TEST_CHECK(bytes == len); + + filter_test_destroy(ctx); +} + +void flb_test_str_to_int_and_int_to_str() +{ + struct flb_lib_out_cb cb_data; + struct filter_test *ctx; + int len; + int ret; + int bytes; + char *p = JSON_TEST_DATA; + + /* Prepare output callback with expected result */ + cb_data.cb = cb_check_result; + cb_data.data = "\"new1\":123,\"new2\":\"123\""; + + /* Create test context */ + ctx = filter_test_create((void *) &cb_data); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("failed to create context"); + exit(EXIT_FAILURE); + } + + /* Configure filter */ + ret = flb_filter_set(ctx->flb, ctx->f_ffd, + "str_key", "numstr new1 int", + "int_key", "int new2 str", + NULL); + TEST_CHECK(ret == 0); + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + /* Ingest data samples */ + len = strlen(p); + bytes = flb_lib_push(ctx->flb, ctx->i_ffd, p, len); + TEST_CHECK(bytes == len); + + filter_test_destroy(ctx); +} + +void flb_test_nest_key() +{ + struct flb_lib_out_cb cb_data; + struct filter_test *ctx; + int len; + int ret; + int bytes; + char *p = JSON_NEST_DATA; + + /* Prepare output callback with expected result */ + cb_data.cb = cb_check_result; + cb_data.data = "\"new\":123"; + + /* Create test context */ + ctx = filter_test_create((void *) &cb_data); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("failed to create context"); + exit(EXIT_FAILURE); + } + + /* Configure filter */ + ret = flb_filter_set(ctx->flb, ctx->f_ffd, + "str_key", "$nest['numstr'] new int", + NULL); + TEST_CHECK(ret == 0); + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + /* Ingest data samples */ + len = strlen(p); + bytes = flb_lib_push(ctx->flb, ctx->i_ffd, p, len); + TEST_CHECK(bytes == len); + + filter_test_destroy(ctx); +} + +/* Test list */ +TEST_LIST = { + {"str_to_int" , flb_test_str_to_int }, + {"str_to_float" , flb_test_str_to_float }, + {"str_to_hex" , flb_test_str_to_hex }, + {"int_to_str" , flb_test_int_to_str }, + {"int_to_float" , flb_test_int_to_float }, + {"str<->int" , flb_test_str_to_int_and_int_to_str }, + {"nest_key" , flb_test_nest_key}, + {NULL, NULL} +}; diff --git a/src/fluent-bit/tests/runtime/filter_wasm.c b/src/fluent-bit/tests/runtime/filter_wasm.c new file mode 100644 index 000000000..4cc1afcb8 --- /dev/null +++ b/src/fluent-bit/tests/runtime/filter_wasm.c @@ -0,0 +1,468 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <fluent-bit.h> +#include <fluent-bit/flb_time.h> +#include <sys/types.h> +#include <sys/stat.h> +#include <fcntl.h> +#include "flb_tests_runtime.h" + +#define DPATH_WASM FLB_TESTS_DATA_PATH "/data/wasm" +#define FLUSH_INTERVAL "1.0" +#ifdef _WIN32 + #define TIME_EPSILON_MS 30 +#else + #define TIME_EPSILON_MS 10 +#endif + +pthread_mutex_t result_mutex = PTHREAD_MUTEX_INITIALIZER; +char *output = NULL; +int num_output = 0; + +void set_output(char *val) +{ + pthread_mutex_lock(&result_mutex); + output = val; + pthread_mutex_unlock(&result_mutex); +} + +char *get_output(void) +{ + char *val; + + pthread_mutex_lock(&result_mutex); + val = output; + pthread_mutex_unlock(&result_mutex); + + return val; +} + +static void clear_output() +{ + pthread_mutex_lock(&result_mutex); + output = NULL; + pthread_mutex_unlock(&result_mutex); +} + +static void clear_output_num() +{ + pthread_mutex_lock(&result_mutex); + num_output = 0; + pthread_mutex_unlock(&result_mutex); +} + +static int get_output_num() +{ + int ret; + pthread_mutex_lock(&result_mutex); + ret = num_output; + pthread_mutex_unlock(&result_mutex); + + return ret; +} + +static int cb_count_msgpack_events(void *record, size_t size, void *data) +{ + msgpack_unpacked result; + size_t off = 0; + + if (!TEST_CHECK(data != NULL)) { + flb_error("data is NULL"); + } + + /* Iterate each item array and apply rules */ + msgpack_unpacked_init(&result); + while (msgpack_unpack_next(&result, record, size, &off) == MSGPACK_UNPACK_SUCCESS) { + pthread_mutex_lock(&result_mutex); + num_output++; + pthread_mutex_unlock(&result_mutex); + } + msgpack_unpacked_destroy(&result); + + flb_free(record); + return 0; +} + +int callback_test(void* data, size_t size, void* cb_data) +{ + if (size > 0) { + flb_debug("[test_filter_wasm] received message: %s", (char*)data); + set_output(data); /* success */ + } + return 0; +} + +void wait_with_timeout(uint32_t timeout_ms, char **out_result) +{ + struct flb_time start_time; + struct flb_time end_time; + struct flb_time diff_time; + uint64_t elapsed_time_flb = 0; + char *output = NULL; + + flb_time_get(&start_time); + + while (true) { + output = get_output(); + + if (output != NULL) { + *out_result = output; + break; + } + + flb_time_msleep(100); + flb_time_get(&end_time); + flb_time_diff(&end_time, &start_time, &diff_time); + elapsed_time_flb = flb_time_to_nanosec(&diff_time) / 1000000; + + if (elapsed_time_flb > timeout_ms - TIME_EPSILON_MS) { + flb_warn("[timeout] elapsed_time: %ld", elapsed_time_flb); + // Reached timeout. + break; + } + } +} + + +void flb_test_append_tag(void) +{ + int ret; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + int filter_ffd; + char *output = NULL; + char *input = "[0, {\"key\":\"val\"}]"; + char *result; + struct flb_lib_out_cb cb_data; + + /* clear previous output */ + clear_output(); + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", FLUSH_INTERVAL, "grace", "1", NULL); + + /* Prepare output callback context*/ + cb_data.cb = callback_test; + cb_data.data = NULL; + + /* Filter */ + filter_ffd = flb_filter(ctx, (char *) "wasm", NULL); + TEST_CHECK(filter_ffd >= 0); + ret = flb_filter_set(ctx, filter_ffd, + "Match", "*", + "wasm_path", DPATH_WASM "/append_tag.wasm", + "function_name", "filter_append_tag", + NULL); + + /* Input */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + flb_input_set(ctx, in_ffd, "tag", "test.wasm", NULL); + TEST_CHECK(in_ffd >= 0); + + /* Lib output */ + out_ffd = flb_output(ctx, (char *) "lib", (void *)&cb_data); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, + "match", "test.wasm", + "format", "json", + NULL); + + ret = flb_start(ctx); + TEST_CHECK(ret==0); + + flb_lib_push(ctx, in_ffd, input, strlen(input)); + wait_with_timeout(2000, &output); + result = strstr(output, "\"tag\":\"test.wasm\""); + TEST_CHECK(result != NULL); + + /* clean up */ + flb_lib_free(output); + + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_helloworld(void) +{ + int ret; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + int filter_ffd; + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", FLUSH_INTERVAL, "grace", "1", NULL); + + /* Filter */ + filter_ffd = flb_filter(ctx, (char *) "wasm", NULL); + TEST_CHECK(filter_ffd >= 0); + ret = flb_filter_set(ctx, filter_ffd, + "Match", "*", + "wasm_path", DPATH_WASM "/say_hello.wasm", + "function_name", "filter_say_hello", + NULL); + /* Input */ + in_ffd = flb_input(ctx, (char *) "dummy", NULL); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + TEST_CHECK(in_ffd >= 0); + + /* Lib output */ + out_ffd = flb_output(ctx, (char *) "stdout", NULL); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, + "match", "test", + NULL); + + ret = flb_start(ctx); + TEST_CHECK(ret==0); + + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_numerics_records(void) +{ + int ret; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + int filter_ffd; + char *output = NULL; + char *input = "[0, {\"key\":\"val\"}]"; + char *result; + struct flb_lib_out_cb cb_data; + + /* clear previous output */ + clear_output(); + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", FLUSH_INTERVAL, "grace", "1", NULL); + + /* Prepare output callback context*/ + cb_data.cb = callback_test; + cb_data.data = NULL; + + /* Filter */ + filter_ffd = flb_filter(ctx, (char *) "wasm", NULL); + TEST_CHECK(filter_ffd >= 0); + ret = flb_filter_set(ctx, filter_ffd, + "Match", "*", + "wasm_path", DPATH_WASM "/numeric_records.wasm", + "function_name", "filter_numeric_records", + NULL); + + /* Input */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + flb_input_set(ctx, in_ffd, "tag", "test.wasm", NULL); + TEST_CHECK(in_ffd >= 0); + + /* Lib output */ + out_ffd = flb_output(ctx, (char *) "lib", (void *)&cb_data); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, + "match", "test.wasm", + "format", "json", + NULL); + + ret = flb_start(ctx); + TEST_CHECK(ret==0); + + flb_lib_push(ctx, in_ffd, input, strlen(input)); + wait_with_timeout(2000, &output); + + /* check if float (for int keys)*/ + result = strstr(output, "\"wasm_int1\":10."); + if (!TEST_CHECK(result == NULL)) { + TEST_MSG("output:%s\n", output); + } + result = strstr(output, "\"wasm_int2\":100."); + if (!TEST_CHECK(result == NULL)) { + TEST_MSG("output:%s\n", output); + } + + /* check if float (for float keys)*/ + result = strstr(output, "\"wasm_float1\":10.5"); + if (!TEST_CHECK(result != NULL)) { + TEST_MSG("output:%s\n", output); + } + result = strstr(output, "\"wasm_float2\":100.5"); + if (!TEST_CHECK(result != NULL)) { + TEST_MSG("output:%s\n", output); + } + + /* check if float (for exp style float key)*/ + result = strstr(output, "\"wasm_exp_float\":0.00354"); + if (!TEST_CHECK(result != NULL)) { + TEST_MSG("output:%s\n", output); + } + + /* check if float (for truncated float key)*/ + result = strstr(output, "\"wasm_truncate_float\":120"); + if (!TEST_CHECK(result != NULL)) { + TEST_MSG("output:%s\n", output); + } + result = strstr(output, "\"wasm_truncate_float\":120."); + if (!TEST_CHECK(result == NULL)) { + TEST_MSG("output:%s\n", output); + } + + /* clean up */ + flb_lib_free(output); + + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_array_contains_null(void) +{ + int ret; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + int filter_ffd; + char *output = NULL; + char *input = "[0, {\"hello\": [1, null, \"world\"]}]"; + char *result; + struct flb_lib_out_cb cb_data; + + /* clear previous output */ + clear_output(); + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", FLUSH_INTERVAL, "grace", "1", NULL); + + /* Prepare output callback context*/ + cb_data.cb = callback_test; + cb_data.data = NULL; + + /* Filter */ + filter_ffd = flb_filter(ctx, (char *) "wasm", NULL); + TEST_CHECK(filter_ffd >= 0); + ret = flb_filter_set(ctx, filter_ffd, + "Match", "*", + "wasm_path", DPATH_WASM "/modify_record.wasm", + "function_name", "filter_modify_record", + NULL); + + /* Input */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + flb_input_set(ctx, in_ffd, "tag", "test.wasm", NULL); + TEST_CHECK(in_ffd >= 0); + + /* Lib output */ + out_ffd = flb_output(ctx, (char *) "lib", (void *)&cb_data); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, + "match", "test.wasm", + "format", "json", + NULL); + + ret = flb_start(ctx); + TEST_CHECK(ret==0); + + flb_lib_push(ctx, in_ffd, input, strlen(input)); + wait_with_timeout(2000, &output); + + result = strstr(output, "[1,null,\"world\"]"); + if(!TEST_CHECK(result != NULL)) { + TEST_MSG("output:%s\n", output); + } + result = strstr(output, "\"modify\":\"yes\""); + if(!TEST_CHECK(result != NULL)) { + TEST_MSG("output:%s\n", output); + } + + /* clean up */ + flb_lib_free(output); + + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_drop_all_records(void) +{ + int ret; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + int filter_ffd; + char *output = NULL; + char *input = "[0, {\"key\":\"val\"}]"; + struct flb_lib_out_cb cb_data; + + clear_output_num(); + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", FLUSH_INTERVAL, "grace", "1", NULL); + + /* Prepare output callback context*/ + cb_data.cb = cb_count_msgpack_events; + cb_data.data = NULL; + + /* Filter */ + filter_ffd = flb_filter(ctx, (char *) "wasm", NULL); + TEST_CHECK(filter_ffd >= 0); + ret = flb_filter_set(ctx, filter_ffd, + "Match", "*", + "wasm_path", DPATH_WASM "/drop_record.wasm", + "function_name", "filter_drop_record", + NULL); + /* Input */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + TEST_CHECK(in_ffd >= 0); + + /* Lib output */ + out_ffd = flb_output(ctx, (char *) "lib", (void *)&cb_data); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, + "match", "test", + NULL); + + ret = flb_start(ctx); + TEST_CHECK(ret==0); + + flb_lib_push(ctx, in_ffd, input, strlen(input)); + wait_with_timeout(2000, &output); /* waiting flush */ + + ret = get_output_num(); + if (!TEST_CHECK(ret == 0)) { + TEST_MSG("error. got %d expect 0", ret); + } + + flb_stop(ctx); + flb_destroy(ctx); +} + + +TEST_LIST = { + {"hello_world", flb_test_helloworld}, + {"append_tag", flb_test_append_tag}, + {"numeric_records", flb_test_numerics_records}, + {"array_contains_null", flb_test_array_contains_null}, + {"drop_all_records", flb_test_drop_all_records}, + {NULL, NULL} +}; diff --git a/src/fluent-bit/tests/runtime/flb_tests_runtime.h.in b/src/fluent-bit/tests/runtime/flb_tests_runtime.h.in new file mode 100644 index 000000000..085fe6f1e --- /dev/null +++ b/src/fluent-bit/tests/runtime/flb_tests_runtime.h.in @@ -0,0 +1,52 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2019-2020 The Fluent Bit Authors + * Copyright (C) 2015-2018 Treasure Data Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_TESTS_RUNTIME_H +#define FLB_TESTS_RUNTIME_H + +#include <sys/stat.h> + +#include "../lib/acutest/acutest.h" +#define FLB_TESTS_DATA_PATH "@FLB_TESTS_DATA_PATH@" + +static inline int wait_for_file(char *path, + size_t minimum_size, + int time_limit) +{ + int elapsed_time; + struct stat file_info; + int result; + + for (elapsed_time = 0 ; elapsed_time < time_limit ; elapsed_time++) { + result = stat(path, &file_info); + + if (result == 0) { + if (file_info.st_size >= minimum_size) { + break; + } + } + + sleep(1); + } + + return result; +} + +#endif diff --git a/src/fluent-bit/tests/runtime/gen_data.py b/src/fluent-bit/tests/runtime/gen_data.py new file mode 100755 index 000000000..dd1453090 --- /dev/null +++ b/src/fluent-bit/tests/runtime/gen_data.py @@ -0,0 +1,63 @@ +#!/usr/bin/python + +# This script generate samples at data/ directory + +def write_header(handle, name): + handle.write("#define %s\t\"[\"\t\t\\\n" % name) + handle.write("\t\"1448403340,\"\t\t\t\\\n") + handle.write("\t\"{\"\t\t\t\t\\\n") + +def write_footer(handle): + handle.write("\t\"\\\"END_KEY\\\": \\\"JSON_END\\\"\"\t\t\\\n") + handle.write("\t\"}]\"\n") + handle.write("\n") + +def write_entry(handle, key, string, num_bool, eof=False): + if string: + handle.write(("\t\"\\\"%s\\\": \\\"%s\\\"" % (key, string))) + else: + handle.write(("\t\"\\\"%s\\\": %s" % (key, num_bool))) + + handle.write(",\"\t\t\\\n") + +# Invalid JSON +f = open("data/common/json_invalid.h", 'w') +write_header(f, "JSON_INVALID") +f.write("\t\"{{{{{{{{\"") +write_footer(f) +f.close() + +# A small JSON +f = open("data/common/json_small.h", 'w') +write_header(f, "JSON_SMALL") +for i in range(0, 250): + write_entry(f, "key_%i" % i, None, "false", True) +write_footer(f) +f.close() + +# Long JSON +f = open("data/common/json_long.h", 'w') +write_header(f, "JSON_LONG") +for i in range(0, 1000): + write_entry(f, "key_%i" % i, "val_%i" % i, None) +write_footer(f) +f.close() + +# Long JSON for TD +f = open("data/td/json_td.h", 'w') +write_header(f, "JSON_TD") +for i in range(0, 500): + write_entry(f, "key_%i" % i, "val_%i" % i, None) +write_footer(f) +f.close() + +# JSON for ES +f = open("data/es/json_es.h", 'w') +write_header(f, "JSON_ES") +write_entry(f, "key_0", None, "false") +write_entry(f, "key_1", None, "true") +write_entry(f, "key_2", "some string", None) +write_entry(f, "key_3", None, 0.12345678) +write_entry(f, "key_4", None, 5000) +write_footer(f) +f.close() diff --git a/src/fluent-bit/tests/runtime/http_callbacks.c b/src/fluent-bit/tests/runtime/http_callbacks.c new file mode 100644 index 000000000..76e1c5f3e --- /dev/null +++ b/src/fluent-bit/tests/runtime/http_callbacks.c @@ -0,0 +1,58 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +#include <fluent-bit.h> +#include <fluent-bit/flb_http_client.h> + +#include "flb_tests_runtime.h" + +static void debug_cb_request_headers(char *name, void *p1, void *p2) +{ + struct flb_http_client *c = p1; + + fprintf(stderr, "[http] request headers\n%s", c->header_buf); +} + +void flb_test_http_callbacks() +{ + int ret; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + ctx = flb_create(); + TEST_CHECK(ctx != NULL); + + flb_service_set(ctx, + "Flush", "1", + "Grace", "5", + "Daemon", "false", + NULL); + + in_ffd = flb_input(ctx, (char *) "dummy", NULL); + flb_input_set(ctx, in_ffd, "samples", "1", NULL); + + out_ffd = flb_output(ctx, (char *) "http", NULL); + flb_output_set(ctx, out_ffd, + "match", "*", + "host", "google.com", + "port", "80", + "uri" , "/", + NULL); + + flb_output_set_callback(ctx, out_ffd, "_debug.http.request_headers", + debug_cb_request_headers); + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + sleep(5); + + flb_stop(ctx); + flb_destroy(ctx); + +} + +/* Test list */ +TEST_LIST = { + {"http_callbacks", flb_test_http_callbacks }, + {NULL, NULL} +}; diff --git a/src/fluent-bit/tests/runtime/in_cpu.c b/src/fluent-bit/tests/runtime/in_cpu.c new file mode 120000 index 000000000..b194a8539 --- /dev/null +++ b/src/fluent-bit/tests/runtime/in_cpu.c @@ -0,0 +1 @@ +in_simple_systems.c
\ No newline at end of file diff --git a/src/fluent-bit/tests/runtime/in_disk.c b/src/fluent-bit/tests/runtime/in_disk.c new file mode 120000 index 000000000..b194a8539 --- /dev/null +++ b/src/fluent-bit/tests/runtime/in_disk.c @@ -0,0 +1 @@ +in_simple_systems.c
\ No newline at end of file diff --git a/src/fluent-bit/tests/runtime/in_dummy.c b/src/fluent-bit/tests/runtime/in_dummy.c new file mode 120000 index 000000000..b194a8539 --- /dev/null +++ b/src/fluent-bit/tests/runtime/in_dummy.c @@ -0,0 +1 @@ +in_simple_systems.c
\ No newline at end of file diff --git a/src/fluent-bit/tests/runtime/in_elasticsearch.c b/src/fluent-bit/tests/runtime/in_elasticsearch.c new file mode 100644 index 000000000..b373d2859 --- /dev/null +++ b/src/fluent-bit/tests/runtime/in_elasticsearch.c @@ -0,0 +1,899 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2019-2023 The Fluent Bit Authors + * Copyright (C) 2015-2018 Treasure Data Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <stdlib.h> +#include <fluent-bit.h> +#include <fluent-bit/flb_compat.h> +#include <fluent-bit/flb_time.h> +#include <fluent-bit/flb_pack.h> +#include <fluent-bit/flb_http_client.h> +#include <fluent-bit/flb_gzip.h> +#include <monkey/mk_core.h> +#include "flb_tests_runtime.h" + +/* Test data */ +#include "data/in_elasticsearch/json_bulk.h" /* NDBULK_JSON */ + +#define NDJSON_CONTENT_TYPE "application/x-ndjson" + +struct in_elasticsearch_client_ctx { + struct flb_upstream *u; + struct flb_connection *u_conn; + struct flb_config *config; + struct mk_event_loop *evl; +}; + +struct test_ctx { + flb_ctx_t *flb; /* Fluent Bit library context */ + int i_ffd; /* Input fd */ + int f_ffd; /* Filter fd (unused) */ + int o_ffd; /* Output fd */ + struct in_elasticsearch_client_ctx *httpc; +}; + + +pthread_mutex_t result_mutex = PTHREAD_MUTEX_INITIALIZER; +int num_output = 0; +static int get_output_num() +{ + int ret; + pthread_mutex_lock(&result_mutex); + ret = num_output; + pthread_mutex_unlock(&result_mutex); + + return ret; +} + +static void set_output_num(int num) +{ + pthread_mutex_lock(&result_mutex); + num_output = num; + pthread_mutex_unlock(&result_mutex); +} + +static void clear_output_num() +{ + set_output_num(0); +} + +/* Callback to check expected results */ +static int cb_check_result_json(void *record, size_t size, void *data) +{ + char *p; + char *expected; + char *result; + int num = get_output_num(); + + set_output_num(num+1); + + expected = (char *) data; + result = (char *) record; + + p = strstr(result, expected); + TEST_CHECK(p != NULL); + + if (p==NULL) { + flb_error("Expected to find: '%s' in result '%s'", + expected, result); + } + /* + * If you want to debug your test + * + * printf("Expect: '%s' in result '%s'", expected, result); + */ + flb_free(record); + return 0; +} + +struct in_elasticsearch_client_ctx* in_elasticsearch_client_ctx_create(int port) +{ + struct in_elasticsearch_client_ctx *ret_ctx = NULL; + struct mk_event_loop *evl = NULL; + + ret_ctx = flb_calloc(1, sizeof(struct in_elasticsearch_client_ctx)); + if (!TEST_CHECK(ret_ctx != NULL)) { + flb_errno(); + TEST_MSG("flb_calloc(in_elasticsearch_client_ctx) failed"); + return NULL; + } + + evl = mk_event_loop_create(16); + if (!TEST_CHECK(evl != NULL)) { + TEST_MSG("mk_event_loop failed"); + flb_free(ret_ctx); + return NULL; + } + ret_ctx->evl = evl; + flb_engine_evl_init(); + flb_engine_evl_set(evl); + + ret_ctx->config = flb_config_init(); + if(!TEST_CHECK(ret_ctx->config != NULL)) { + TEST_MSG("flb_config_init failed"); + mk_event_loop_destroy(evl); + flb_free(ret_ctx); + return NULL; + } + + ret_ctx->u = flb_upstream_create(ret_ctx->config, "127.0.0.1", port, 0, NULL); + if (!TEST_CHECK(ret_ctx->u != NULL)) { + TEST_MSG("flb_upstream_create failed"); + flb_config_exit(ret_ctx->config); + mk_event_loop_destroy(evl); + flb_free(ret_ctx); + return NULL; + } + + ret_ctx->u_conn = flb_upstream_conn_get(ret_ctx->u); + TEST_CHECK(ret_ctx->u_conn != NULL); + + ret_ctx->u_conn->upstream = ret_ctx->u; + + return ret_ctx; +} + +static struct test_ctx *test_ctx_create(struct flb_lib_out_cb *data) +{ + int i_ffd; + int o_ffd; + struct test_ctx *ctx = NULL; + + ctx = flb_calloc(1, sizeof(struct test_ctx)); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("flb_calloc failed"); + flb_errno(); + return NULL; + } + + /* Service config */ + ctx->flb = flb_create(); + flb_service_set(ctx->flb, + "Flush", "0.200000000", + "Grace", "1", + "Log_Level", "error", + NULL); + + /* Input */ + i_ffd = flb_input(ctx->flb, (char *) "elasticsearch", NULL); + TEST_CHECK(i_ffd >= 0); + ctx->i_ffd = i_ffd; + + /* Output */ + o_ffd = flb_output(ctx->flb, (char *) "lib", (void *) data); + ctx->o_ffd = o_ffd; + + return ctx; +} + +int in_elasticsearch_client_ctx_destroy(struct in_elasticsearch_client_ctx* ctx) +{ + if (!TEST_CHECK(ctx != NULL)) { + return -1; + } + if (ctx->u) { + flb_upstream_destroy(ctx->u); + } + if (ctx->config) { + flb_config_exit(ctx->config); + } + if (ctx->evl) { + mk_event_loop_destroy(ctx->evl); + } + + flb_free(ctx); + return 0; +} + +static void test_ctx_destroy(struct test_ctx *ctx) +{ + TEST_CHECK(ctx != NULL); + if (ctx->httpc) { + in_elasticsearch_client_ctx_destroy(ctx->httpc); + } + + sleep(1); + flb_stop(ctx->flb); + flb_destroy(ctx->flb); + flb_free(ctx); +} + +void flb_test_in_elasticsearch_version() +{ + struct flb_lib_out_cb cb_data; + struct test_ctx *ctx; + struct flb_http_client *c; + int ret; + size_t b_sent; + char *expected = "\"version\":{\"number\":\"8.0.0\",\"build_flavor\""; + char *buf = NULL; + int port = 9201; + char sport[16]; + + snprintf(sport, 16, "%d", port); + + ctx = test_ctx_create(&cb_data); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("test_ctx_create failed"); + exit(EXIT_FAILURE); + } + + ret = flb_input_set(ctx->flb, ctx->i_ffd, + "port", sport, + NULL); + TEST_CHECK(ret == 0); + + ret = flb_output_set(ctx->flb, ctx->o_ffd, + "match", "*", + "format", "json", + NULL); + TEST_CHECK(ret == 0); + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + ctx->httpc = in_elasticsearch_client_ctx_create(port); + TEST_CHECK(ctx->httpc != NULL); + + c = flb_http_client(ctx->httpc->u_conn, FLB_HTTP_GET, "/", NULL, 0, + "127.0.0.1", port, NULL, 0); + if (!TEST_CHECK(c != NULL)) { + TEST_MSG("in_elasticsearch_client failed"); + exit(EXIT_FAILURE); + } + + ret = flb_http_do(c, &b_sent); + if (!TEST_CHECK(ret == 0)) { + TEST_MSG("ret error. ret=%d\n", ret); + } + else if (!TEST_CHECK(b_sent > 0)){ + TEST_MSG("b_sent size error. b_sent = %lu\n", b_sent); + } + else if (!TEST_CHECK(c->resp.status == 200)) { + TEST_MSG("http response code error. expect: 200, got: %d\n", c->resp.status); + } + + /* waiting to flush */ + flb_time_msleep(1500); + + buf = strstr(c->resp.payload, expected); + if (!TEST_CHECK(buf != NULL)) { + TEST_MSG("http request for version info failed"); + } + flb_http_client_destroy(c); + flb_upstream_conn_release(ctx->httpc->u_conn); + test_ctx_destroy(ctx); +} + +void flb_test_in_elasticsearch_version_configured() +{ + struct flb_lib_out_cb cb_data; + struct test_ctx *ctx; + struct flb_http_client *c; + int ret; + size_t b_sent; + char *expected = "\"version\":{\"number\":\"8.1.2\",\"build_flavor\""; + char *buf = NULL; + int port = 9202; + char sport[16]; + + snprintf(sport, 16, "%d", port); + + ctx = test_ctx_create(&cb_data); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("test_ctx_create failed"); + exit(EXIT_FAILURE); + } + + ret = flb_input_set(ctx->flb, ctx->i_ffd, + "port", sport, + "version", "8.1.2", + NULL); + TEST_CHECK(ret == 0); + + ret = flb_output_set(ctx->flb, ctx->o_ffd, + "match", "*", + "format", "json", + NULL); + TEST_CHECK(ret == 0); + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + ctx->httpc = in_elasticsearch_client_ctx_create(port); + TEST_CHECK(ctx->httpc != NULL); + + c = flb_http_client(ctx->httpc->u_conn, FLB_HTTP_GET, "/", NULL, 0, + "127.0.0.1", port, NULL, 0); + if (!TEST_CHECK(c != NULL)) { + TEST_MSG("in_elasticsearch_client failed"); + exit(EXIT_FAILURE); + } + + ret = flb_http_do(c, &b_sent); + if (!TEST_CHECK(ret == 0)) { + TEST_MSG("ret error. ret=%d\n", ret); + } + else if (!TEST_CHECK(b_sent > 0)){ + TEST_MSG("b_sent size error. b_sent = %lu\n", b_sent); + } + else if (!TEST_CHECK(c->resp.status == 200)) { + TEST_MSG("http response code error. expect: 200, got: %d\n", c->resp.status); + } + + /* waiting to flush */ + flb_time_msleep(1500); + + buf = strstr(c->resp.payload, expected); + if (!TEST_CHECK(buf != NULL)) { + TEST_MSG("http request for version info failed"); + } + flb_http_client_destroy(c); + flb_upstream_conn_release(ctx->httpc->u_conn); + test_ctx_destroy(ctx); +} + +void flb_test_in_elasticsearch(char *write_op, int port, char *tag) +{ + struct flb_lib_out_cb cb_data; + struct test_ctx *ctx; + struct flb_http_client *c; + int ret; + int num; + size_t b_sent; + char buf[64]; + char expected[64]; + char sport[16]; + + snprintf(buf, 64, "{\"%s\":{\"_index\":\"fluent-bit\",\"_id\":1}}\n{\"test\":\"msg\"}\n", write_op); + snprintf(expected, 64, "\"@meta\":{\"%s\":{\"_index\":\"fluent-bit\",\"_id\":1}},\"test\":\"msg\"", write_op); + + snprintf(sport, 16, "%d", port); + + clear_output_num(); + + cb_data.cb = cb_check_result_json; + cb_data.data = expected; + + ctx = test_ctx_create(&cb_data); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("test_ctx_create failed"); + exit(EXIT_FAILURE); + } + + ret = flb_input_set(ctx->flb, ctx->i_ffd, + "port", sport, + NULL); + TEST_CHECK(ret == 0); + if (tag != NULL) { + ret = flb_input_set(ctx->flb, ctx->i_ffd, + "tag", tag, + NULL); + TEST_CHECK(ret == 0); + } + + if (tag != NULL) { + ret = flb_output_set(ctx->flb, ctx->o_ffd, + "match", tag, + "format", "json", + NULL); + TEST_CHECK(ret == 0); + } + else { + ret = flb_output_set(ctx->flb, ctx->o_ffd, + "match", "*", + "format", "json", + NULL); + TEST_CHECK(ret == 0); + } + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + ctx->httpc = in_elasticsearch_client_ctx_create(port); + TEST_CHECK(ctx->httpc != NULL); + + c = flb_http_client(ctx->httpc->u_conn, FLB_HTTP_POST, "/_bulk", buf, strlen(buf), + "127.0.0.1", port, NULL, 0); + ret = flb_http_add_header(c, FLB_HTTP_HEADER_CONTENT_TYPE, strlen(FLB_HTTP_HEADER_CONTENT_TYPE), + NDJSON_CONTENT_TYPE, strlen(NDJSON_CONTENT_TYPE)); + TEST_CHECK(ret == 0); + if (!TEST_CHECK(c != NULL)) { + TEST_MSG("in_elasticsearch_client failed"); + exit(EXIT_FAILURE); + } + + ret = flb_http_do(c, &b_sent); + if (!TEST_CHECK(ret == 0)) { + TEST_MSG("ret error. ret=%d\n", ret); + } + else if (!TEST_CHECK(b_sent > 0)){ + TEST_MSG("b_sent size error. b_sent = %lu\n", b_sent); + } + else if (!TEST_CHECK(c->resp.status == 200)) { + TEST_MSG("http response code error. expect: 200, got: %d\n", c->resp.status); + } + + /* waiting to flush */ + flb_time_msleep(1500); + + num = get_output_num(); + if (!TEST_CHECK(num > 0)) { + TEST_MSG("no outputs"); + } + flb_http_client_destroy(c); + flb_upstream_conn_release(ctx->httpc->u_conn); + test_ctx_destroy(ctx); +} + +void flb_test_in_elasticsearch_index_op() +{ + flb_test_in_elasticsearch("index", 9203, NULL); +} + +void flb_test_in_elasticsearch_create_op() +{ + flb_test_in_elasticsearch("create", 9204, NULL); +} + +void flb_test_in_elasticsearch_invalid(char *write_op, int status, char *expected_op, int port) +{ + struct flb_lib_out_cb cb_data; + struct test_ctx *ctx; + struct flb_http_client *c; + int ret; + int num; + size_t b_sent; + char buf[64]; + char expected[64]; + char *ret_buf = NULL; + char sport[16]; + + snprintf(buf, 64, "{\"%s\":{\"_index\":\"fluent-bit\",\"_id\":1}}\n{\"test\":\"msg\"}\n", write_op); + snprintf(expected, 64, "{\"%s\":{\"status\":%d", expected_op, status); + + snprintf(sport, 16, "%d", port); + + clear_output_num(); + + cb_data.cb = NULL; + cb_data.data = NULL; + + ctx = test_ctx_create(&cb_data); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("test_ctx_create failed"); + exit(EXIT_FAILURE); + } + + ret = flb_input_set(ctx->flb, ctx->i_ffd, + "port", sport, + NULL); + TEST_CHECK(ret == 0); + + ret = flb_output_set(ctx->flb, ctx->o_ffd, + "match", "*", + "format", "json", + NULL); + TEST_CHECK(ret == 0); + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + ctx->httpc = in_elasticsearch_client_ctx_create(port); + TEST_CHECK(ctx->httpc != NULL); + + c = flb_http_client(ctx->httpc->u_conn, FLB_HTTP_POST, "/_bulk", buf, strlen(buf), + "127.0.0.1", port, NULL, 0); + ret = flb_http_add_header(c, FLB_HTTP_HEADER_CONTENT_TYPE, strlen(FLB_HTTP_HEADER_CONTENT_TYPE), + NDJSON_CONTENT_TYPE, strlen(NDJSON_CONTENT_TYPE)); + TEST_CHECK(ret == 0); + if (!TEST_CHECK(c != NULL)) { + TEST_MSG("in_elasticsearch_client failed"); + exit(EXIT_FAILURE); + } + + ret = flb_http_do(c, &b_sent); + if (!TEST_CHECK(ret == 0)) { + TEST_MSG("ret error. ret=%d\n", ret); + } + else if (!TEST_CHECK(b_sent > 0)){ + TEST_MSG("b_sent size error. b_sent = %lu\n", b_sent); + } + else if (!TEST_CHECK(c->resp.status == 200)) { + TEST_MSG("http response code error. expect: 200, got: %d\n", c->resp.status); + } + + /* waiting to flush */ + flb_time_msleep(1500); + + num = get_output_num(); + if (!TEST_CHECK(num == 0)) { + TEST_MSG("invalid ingested requests"); + } + ret_buf = strstr(c->resp.payload, expected); + if (!TEST_CHECK(ret_buf != NULL)) { + TEST_MSG("http request for bulk failed"); + } + + flb_http_client_destroy(c); + flb_upstream_conn_release(ctx->httpc->u_conn); + test_ctx_destroy(ctx); +} + +void flb_test_in_elasticsearch_update_op() +{ + flb_test_in_elasticsearch_invalid("update", 403, "update", 9205); +} + +void flb_test_in_elasticsearch_delete_op() +{ + flb_test_in_elasticsearch_invalid("delete", 404, "delete", 9206); +} + +void flb_test_in_elasticsearch_nonexistent_op() +{ + flb_test_in_elasticsearch_invalid("nonexistent", 400, "unknown", 9207); +} + +void flb_test_in_elasticsearch_multi_ops() +{ + struct flb_lib_out_cb cb_data; + struct test_ctx *ctx; + struct flb_http_client *c; + int ret; + int num; + int port = 9208; + char sport[16]; + size_t b_sent; + char *buf = NDJSON_BULK; + char *expected = ":{\"_index\":\"test\",\"_id\":"; + char *ret_buf = NULL; + char *ret_expected = "{\"errors\":true,\"items\":["; + + snprintf(sport, 16, "%d", port); + + clear_output_num(); + + cb_data.cb = cb_check_result_json; + cb_data.data = expected; + + ctx = test_ctx_create(&cb_data); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("test_ctx_create failed"); + exit(EXIT_FAILURE); + } + + ret = flb_input_set(ctx->flb, ctx->i_ffd, + "port", sport, + NULL); + TEST_CHECK(ret == 0); + + ret = flb_output_set(ctx->flb, ctx->o_ffd, + "match", "*", + "format", "json", + NULL); + TEST_CHECK(ret == 0); + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + ctx->httpc = in_elasticsearch_client_ctx_create(port); + TEST_CHECK(ctx->httpc != NULL); + + c = flb_http_client(ctx->httpc->u_conn, FLB_HTTP_POST, "/_bulk", buf, strlen(buf), + "127.0.0.1", port, NULL, 0); + ret = flb_http_add_header(c, FLB_HTTP_HEADER_CONTENT_TYPE, strlen(FLB_HTTP_HEADER_CONTENT_TYPE), + NDJSON_CONTENT_TYPE, strlen(NDJSON_CONTENT_TYPE)); + TEST_CHECK(ret == 0); + if (!TEST_CHECK(c != NULL)) { + TEST_MSG("in_elasticsearch_client failed"); + exit(EXIT_FAILURE); + } + + ret = flb_http_do(c, &b_sent); + if (!TEST_CHECK(ret == 0)) { + TEST_MSG("ret error. ret=%d\n", ret); + } + else if (!TEST_CHECK(b_sent > 0)){ + TEST_MSG("b_sent size error. b_sent = %lu\n", b_sent); + } + else if (!TEST_CHECK(c->resp.status == 200)) { + TEST_MSG("http response code error. expect: 200, got: %d\n", c->resp.status); + } + + /* waiting to flush */ + flb_time_msleep(1500); + + num = get_output_num(); + if (!TEST_CHECK(num > 0)) { + TEST_MSG("no outputs"); + } + ret_buf = strstr(c->resp.payload, ret_expected); + if (!TEST_CHECK(ret_buf != NULL)) { + TEST_MSG("bulk request for multi write ops failed"); + } + flb_http_client_destroy(c); + flb_upstream_conn_release(ctx->httpc->u_conn); + test_ctx_destroy(ctx); +} + +void flb_test_in_elasticsearch_multi_ops_gzip() +{ + struct flb_lib_out_cb cb_data; + struct test_ctx *ctx; + struct flb_http_client *c; + int ret; + int num; + int port = 9209; + char sport[16]; + size_t b_sent; + char *buf = NDJSON_BULK; + char *expected = ":{\"_index\":\"test\",\"_id\":"; + char *ret_buf = NULL; + char *ret_expected = "{\"errors\":true,\"items\":["; + void *final_data; + size_t final_bytes; + + snprintf(sport, 16, "%d", port); + + clear_output_num(); + + cb_data.cb = cb_check_result_json; + cb_data.data = expected; + + ctx = test_ctx_create(&cb_data); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("test_ctx_create failed"); + exit(EXIT_FAILURE); + } + + ret = flb_input_set(ctx->flb, ctx->i_ffd, + "port", sport, + NULL); + TEST_CHECK(ret == 0); + + ret = flb_output_set(ctx->flb, ctx->o_ffd, + "match", "*", + "format", "json", + NULL); + TEST_CHECK(ret == 0); + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + ctx->httpc = in_elasticsearch_client_ctx_create(port); + TEST_CHECK(ctx->httpc != NULL); + + ret = flb_gzip_compress((void *) buf, strlen(buf), &final_data, &final_bytes); + TEST_CHECK(ret != -1); + + c = flb_http_client(ctx->httpc->u_conn, FLB_HTTP_POST, "/_bulk", final_data, final_bytes, + "127.0.0.1", port, NULL, 0); + ret = flb_http_add_header(c, FLB_HTTP_HEADER_CONTENT_TYPE, strlen(FLB_HTTP_HEADER_CONTENT_TYPE), + NDJSON_CONTENT_TYPE, strlen(NDJSON_CONTENT_TYPE)); + TEST_CHECK(ret == 0); + /* Add Content-Encoding: gzip */ + ret = flb_http_add_header(c, "Content-Encoding", 16, "gzip", 4); + TEST_CHECK(ret == 0); + + if (!TEST_CHECK(c != NULL)) { + TEST_MSG("in_elasticsearch_client failed"); + exit(EXIT_FAILURE); + } + + ret = flb_http_do(c, &b_sent); + if (!TEST_CHECK(ret == 0)) { + TEST_MSG("ret error. ret=%d\n", ret); + } + else if (!TEST_CHECK(b_sent > 0)){ + TEST_MSG("b_sent size error. b_sent = %lu\n", b_sent); + } + else if (!TEST_CHECK(c->resp.status == 200)) { + TEST_MSG("http response code error. expect: 200, got: %d\n", c->resp.status); + } + flb_free(final_data); + + /* waiting to flush */ + flb_time_msleep(1500); + + num = get_output_num(); + if (!TEST_CHECK(num > 0)) { + TEST_MSG("no outputs"); + } + ret_buf = strstr(c->resp.payload, ret_expected); + if (!TEST_CHECK(ret_buf != NULL)) { + TEST_MSG("bulk request for multi write ops failed"); + } + flb_http_client_destroy(c); + flb_upstream_conn_release(ctx->httpc->u_conn); + test_ctx_destroy(ctx); +} + +void flb_test_in_elasticsearch_node_info() +{ + struct flb_lib_out_cb cb_data; + struct test_ctx *ctx; + struct flb_http_client *c; + int ret; + int port = 9210; + char sport[16]; + size_t b_sent; + char *expected = "{\"_nodes\":{\"total\":1,\"successful\":1,\"failed\":0},\"nodes\":{\""; + char *buf = NULL; + + snprintf(sport, 16, "%d", port); + + ctx = test_ctx_create(&cb_data); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("test_ctx_create failed"); + exit(EXIT_FAILURE); + } + + ret = flb_input_set(ctx->flb, ctx->i_ffd, + "port", sport, + NULL); + TEST_CHECK(ret == 0); + + ret = flb_output_set(ctx->flb, ctx->o_ffd, + "match", "*", + "format", "json", + NULL); + TEST_CHECK(ret == 0); + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + ctx->httpc = in_elasticsearch_client_ctx_create(port); + TEST_CHECK(ctx->httpc != NULL); + + c = flb_http_client(ctx->httpc->u_conn, FLB_HTTP_GET, "/_nodes/http", NULL, 0, + "127.0.0.1", port, NULL, 0); + if (!TEST_CHECK(c != NULL)) { + TEST_MSG("in_elasticsearch_client failed"); + exit(EXIT_FAILURE); + } + + ret = flb_http_do(c, &b_sent); + if (!TEST_CHECK(ret == 0)) { + TEST_MSG("ret error. ret=%d\n", ret); + } + else if (!TEST_CHECK(b_sent > 0)){ + TEST_MSG("b_sent size error. b_sent = %lu\n", b_sent); + } + else if (!TEST_CHECK(c->resp.status == 200)) { + TEST_MSG("http response code error. expect: 200, got: %d\n", c->resp.status); + } + + /* waiting to flush */ + flb_time_msleep(1500); + + buf = strstr(c->resp.payload, expected); + if (!TEST_CHECK(buf != NULL)) { + TEST_MSG("http request for version info failed"); + } + flb_http_client_destroy(c); + flb_upstream_conn_release(ctx->httpc->u_conn); + test_ctx_destroy(ctx); +} + +void flb_test_in_elasticsearch_tag_key() +{ + struct flb_lib_out_cb cb_data; + struct test_ctx *ctx; + struct flb_http_client *c; + int ret; + int num; + size_t b_sent; + int port = 9211; + char sport[16]; + + char *buf = "{\"index\":{\"_index\":\"fluent-bit\"}}\n{\"test\":\"msg\",\"tag\":\"new_tag\"}\n"; + + snprintf(sport, 16, "%d", port); + + clear_output_num(); + + cb_data.cb = cb_check_result_json; + cb_data.data = "\"@meta\":{\"index\":{\"_index\":\"fluent-bit\"}},\"test\":\"msg\",\"tag\":\"new_tag\""; + + ctx = test_ctx_create(&cb_data); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("test_ctx_create failed"); + exit(EXIT_FAILURE); + } + + ret = flb_input_set(ctx->flb, ctx->i_ffd, + "port", sport, + "tag_key", "tag", + NULL); + TEST_CHECK(ret == 0); + + ret = flb_output_set(ctx->flb, ctx->o_ffd, + "match", "new_tag", + "format", "json", + NULL); + TEST_CHECK(ret == 0); + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + ctx->httpc = in_elasticsearch_client_ctx_create(port); + TEST_CHECK(ctx->httpc != NULL); + + c = flb_http_client(ctx->httpc->u_conn, FLB_HTTP_POST, "/_bulk", buf, strlen(buf), + "127.0.0.1", port, NULL, 0); + ret = flb_http_add_header(c, FLB_HTTP_HEADER_CONTENT_TYPE, strlen(FLB_HTTP_HEADER_CONTENT_TYPE), + NDJSON_CONTENT_TYPE, strlen(NDJSON_CONTENT_TYPE)); + TEST_CHECK(ret == 0); + if (!TEST_CHECK(c != NULL)) { + TEST_MSG("in_elasticsearch_client failed"); + exit(EXIT_FAILURE); + } + + ret = flb_http_do(c, &b_sent); + if (!TEST_CHECK(ret == 0)) { + TEST_MSG("ret error. ret=%d\n", ret); + } + else if (!TEST_CHECK(b_sent > 0)){ + TEST_MSG("b_sent size error. b_sent = %lu\n", b_sent); + } + else if (!TEST_CHECK(c->resp.status == 200)) { + TEST_MSG("http response code error. expect: 200, got: %d\n", c->resp.status); + } + + /* waiting to flush */ + flb_time_msleep(1500); + + num = get_output_num(); + if (!TEST_CHECK(num > 0)) { + TEST_MSG("no outputs"); + } + flb_http_client_destroy(c); + flb_upstream_conn_release(ctx->httpc->u_conn); + test_ctx_destroy(ctx); +} + +void flb_test_in_elasticsearch_index_op_with_plugin_tag() +{ + flb_test_in_elasticsearch("index", 9210, "es.index"); +} + +TEST_LIST = { + {"version", flb_test_in_elasticsearch_version}, + {"configured_version", flb_test_in_elasticsearch_version_configured}, + {"index_op", flb_test_in_elasticsearch_index_op}, + {"create_op", flb_test_in_elasticsearch_create_op}, + {"update_op", flb_test_in_elasticsearch_update_op}, + {"delete_op", flb_test_in_elasticsearch_delete_op}, + {"nonexistent_op", flb_test_in_elasticsearch_nonexistent_op}, + {"multi_ops", flb_test_in_elasticsearch_multi_ops}, + {"multi_ops_gzip", flb_test_in_elasticsearch_multi_ops_gzip}, + {"index_op_with_plugin_tag", flb_test_in_elasticsearch_index_op_with_plugin_tag}, + {"node_info", flb_test_in_elasticsearch_node_info}, + {"tag_key", flb_test_in_elasticsearch_tag_key}, + {NULL, NULL} +}; diff --git a/src/fluent-bit/tests/runtime/in_event_test.c b/src/fluent-bit/tests/runtime/in_event_test.c new file mode 100644 index 000000000..1169d8fd8 --- /dev/null +++ b/src/fluent-bit/tests/runtime/in_event_test.c @@ -0,0 +1,35 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +#include <fluent-bit.h> +#include "flb_tests_runtime.h" + +void flb_test_input_event() +{ + int ret; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + ctx = flb_create(); + + in_ffd = flb_input(ctx, (char *) "event_test", NULL); + TEST_CHECK(in_ffd >= 0); + + out_ffd = flb_output(ctx, (char *) "null", NULL); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, "match", "test", NULL); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + sleep(8); + + flb_stop(ctx); + flb_destroy(ctx); +} + +/* Test list */ +TEST_LIST = { + {"event_test", flb_test_input_event}, + {NULL, NULL} +}; diff --git a/src/fluent-bit/tests/runtime/in_fluentbit_metrics.c b/src/fluent-bit/tests/runtime/in_fluentbit_metrics.c new file mode 100644 index 000000000..cb82ec2d8 --- /dev/null +++ b/src/fluent-bit/tests/runtime/in_fluentbit_metrics.c @@ -0,0 +1,226 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2019-2022 The Fluent Bit Authors + * Copyright (C) 2015-2018 Treasure Data Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <fluent-bit.h> +#include <fluent-bit/flb_compat.h> +#include <fluent-bit/flb_time.h> +#include <float.h> +#include "flb_tests_runtime.h" + +struct test_ctx { + flb_ctx_t *flb; /* Fluent Bit library context */ + int i_ffd; /* Input fd */ + int f_ffd; /* Filter fd (unused) */ + int o_ffd; /* Output fd */ +}; + + +pthread_mutex_t result_mutex = PTHREAD_MUTEX_INITIALIZER; +int num_output = 0; +static int get_output_num() +{ + int ret; + pthread_mutex_lock(&result_mutex); + ret = num_output; + pthread_mutex_unlock(&result_mutex); + + return ret; +} + +static void set_output_num(int num) +{ + pthread_mutex_lock(&result_mutex); + num_output = num; + pthread_mutex_unlock(&result_mutex); +} + +static void clear_output_num() +{ + set_output_num(0); +} + +struct str_list { + size_t size; + char **lists; +}; + +void wait_with_timeout(uint32_t timeout_ms, int *output_num) +{ + struct flb_time start_time; + struct flb_time end_time; + struct flb_time diff_time; + uint64_t elapsed_time_flb = 0; + + flb_time_get(&start_time); + + while (true) { + *output_num = get_output_num(); + + if (*output_num > 0) { + break; + } + + flb_time_msleep(100); + flb_time_get(&end_time); + flb_time_diff(&end_time, &start_time, &diff_time); + elapsed_time_flb = flb_time_to_nanosec(&diff_time) / 1000000; + + if (elapsed_time_flb > timeout_ms) { + flb_warn("[timeout] elapsed_time: %ld", elapsed_time_flb); + // Reached timeout. + break; + } + } +} + +/* Callback to check expected results */ +static int cb_check_json_str_list(void *record, size_t size, void *data) +{ + char *p; + char *result; + int num = get_output_num(); + size_t i; + struct str_list *l = (struct str_list*)data; + + if (!TEST_CHECK(l != NULL)) { + TEST_MSG("Data is NULL"); + flb_free(record); + return 0; + } + + set_output_num(num+1); + + result = (char *) record; + + for (i=0; i<l->size; i++) { + p = strstr(result, l->lists[i]); + if(!TEST_CHECK(p != NULL)) { + TEST_MSG("Expected to find: '%s' in result '%s'", + l->lists[i], result); + } + } + + /* + * If you want to debug your test + * + * printf("Expect: '%s' in result '%s'", expected, result); + */ + flb_free(record); + return 0; +} + +static struct test_ctx *test_ctx_create(struct flb_lib_out_cb *data) +{ + int o_ffd; + struct test_ctx *ctx = NULL; + + ctx = flb_malloc(sizeof(struct test_ctx)); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("malloc failed"); + flb_errno(); + return NULL; + } + + /* Service config */ + ctx->flb = flb_create(); + flb_service_set(ctx->flb, + "Flush", "0.200000000", + "Grace", "1", + "Log_Level", "error", + NULL); + + ctx->i_ffd = flb_input(ctx->flb, (char *) "fluentbit_metrics", NULL); + TEST_CHECK(ctx->i_ffd >= 0); + + /* Output */ + o_ffd = flb_output(ctx->flb, (char *) "lib", (void *) data); + ctx->o_ffd = o_ffd; + TEST_CHECK(ctx->o_ffd >= 0); + + return ctx; +} + +static void test_ctx_destroy(struct test_ctx *ctx) +{ + TEST_CHECK(ctx != NULL); + + sleep(1); + flb_stop(ctx->flb); + flb_destroy(ctx->flb); + flb_free(ctx); +} + +#ifdef FLB_HAVE_METRICS +char *basic_expected_strs[] = {"\"uptime\"", "\"records_total\"", "\"bytes_total\"", "\"proc_records_total\"", "\"proc_bytes_total\"", "\"errors_total\"", "\"retries_total\"", "\"retries_failed_total\"", "\"dropped_records_total\"", "\"retried_records_total\""}; + +static void test_basic(void) +{ + struct flb_lib_out_cb cb_data; + struct test_ctx *ctx; + int ret; + int num; + + struct str_list expected = { + .size = sizeof(basic_expected_strs)/sizeof(char*), + .lists = &basic_expected_strs[0], + }; + + clear_output_num(); + + cb_data.cb = cb_check_json_str_list; + cb_data.data = &expected; + + ctx = test_ctx_create(&cb_data); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("test_ctx_create failed"); + exit(EXIT_FAILURE); + } + /* Input */ + ret = flb_input_set(ctx->flb, ctx->i_ffd, + "scrape_interval", "1", + NULL); + TEST_CHECK(ret == 0); + + ret = flb_output_set(ctx->flb, ctx->o_ffd, + "format", "json", + NULL); + TEST_CHECK(ret == 0); + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + /* waiting to flush */ + wait_with_timeout(3000, &num); + + if (!TEST_CHECK(num > 0)) { + TEST_MSG("no outputs"); + } + + test_ctx_destroy(ctx); +} +#endif + +TEST_LIST = { +#ifdef FLB_HAVE_METRICS + {"basic", test_basic}, +#endif + {NULL, NULL} +}; diff --git a/src/fluent-bit/tests/runtime/in_forward.c b/src/fluent-bit/tests/runtime/in_forward.c new file mode 100644 index 000000000..6cabfa94a --- /dev/null +++ b/src/fluent-bit/tests/runtime/in_forward.c @@ -0,0 +1,579 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2019-2022 The Fluent Bit Authors + * Copyright (C) 2015-2018 Treasure Data Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <fluent-bit.h> +#include <fluent-bit/flb_compat.h> +#include <fluent-bit/flb_time.h> +#include <fluent-bit/flb_pack.h> +#include <fluent-bit/flb_socket.h> +#include <sys/types.h> +#include <sys/stat.h> +#ifdef FLB_HAVE_UNIX_SOCKET +#include <sys/socket.h> +#include <sys/un.h> +#endif +#include <fcntl.h> +#include "flb_tests_runtime.h" + +struct test_ctx { + flb_ctx_t *flb; /* Fluent Bit library context */ + int i_ffd; /* Input fd */ + int f_ffd; /* Filter fd (unused) */ + int o_ffd; /* Output fd */ +}; + + +pthread_mutex_t result_mutex = PTHREAD_MUTEX_INITIALIZER; +int num_output = 0; +static int get_output_num() +{ + int ret; + pthread_mutex_lock(&result_mutex); + ret = num_output; + pthread_mutex_unlock(&result_mutex); + + return ret; +} + +static void set_output_num(int num) +{ + pthread_mutex_lock(&result_mutex); + num_output = num; + pthread_mutex_unlock(&result_mutex); +} + +static void clear_output_num() +{ + set_output_num(0); +} + +static int create_simple_json(char **out_buf, size_t *size) +{ + int root_type; + int ret; + char json[] = "[\"test\", 1234567890,{\"test\":\"msg\"} ]"; + + ret = flb_pack_json(&json[0], strlen(json), out_buf, size, &root_type, NULL); + TEST_CHECK(ret==0); + + return ret; +} + + +/* Callback to check expected results */ +static int cb_check_result_json(void *record, size_t size, void *data) +{ + char *p; + char *expected; + char *result; + int num = get_output_num(); + + set_output_num(num+1); + + expected = (char *) data; + result = (char *) record; + + p = strstr(result, expected); + TEST_CHECK(p != NULL); + + if (p==NULL) { + flb_error("Expected to find: '%s' in result '%s'", + expected, result); + } + /* + * If you want to debug your test + * + * printf("Expect: '%s' in result '%s'", expected, result); + */ + flb_free(record); + return 0; +} + +static struct test_ctx *test_ctx_create(struct flb_lib_out_cb *data) +{ + int i_ffd; + int o_ffd; + struct test_ctx *ctx = NULL; + + ctx = flb_malloc(sizeof(struct test_ctx)); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("malloc failed"); + flb_errno(); + return NULL; + } + + /* Service config */ + ctx->flb = flb_create(); + flb_service_set(ctx->flb, + "Flush", "0.200000000", + "Grace", "1", + "Log_Level", "error", + NULL); + + /* Input */ + i_ffd = flb_input(ctx->flb, (char *) "forward", NULL); + TEST_CHECK(i_ffd >= 0); + ctx->i_ffd = i_ffd; + + /* Output */ + o_ffd = flb_output(ctx->flb, (char *) "lib", (void *) data); + ctx->o_ffd = o_ffd; + + return ctx; +} + +static void test_ctx_destroy(struct test_ctx *ctx) +{ + TEST_CHECK(ctx != NULL); + + sleep(1); + flb_stop(ctx->flb); + flb_destroy(ctx->flb); + flb_free(ctx); +} + +#define DEFAULT_HOST "127.0.0.1" +#define DEFAULT_PORT 24224 +static flb_sockfd_t connect_tcp(char *in_host, int in_port) +{ + int port = in_port; + char *host = in_host; + flb_sockfd_t fd; + int ret; + struct sockaddr_in addr; + + if (host == NULL) { + host = DEFAULT_HOST; + } + if (port < 0) { + port = DEFAULT_PORT; + } + + memset(&addr, 0, sizeof(addr)); + fd = socket(PF_INET, SOCK_STREAM, 0); + if (!TEST_CHECK(fd >= 0)) { + TEST_MSG("failed to socket. host=%s port=%d errno=%d", host, port, errno); + return -1; + } + + addr.sin_family = AF_INET; + addr.sin_addr.s_addr = inet_addr(host); + addr.sin_port = htons(port); + + ret = connect(fd, (const struct sockaddr *)&addr, sizeof(addr)); + if (!TEST_CHECK(ret >= 0)) { + TEST_MSG("failed to connect. host=%s port=%d errno=%d", host, port, errno); + flb_socket_close(fd); + return -1; + } + return fd; +} + +void flb_test_forward() +{ + struct flb_lib_out_cb cb_data; + struct test_ctx *ctx; + flb_sockfd_t fd; + int ret; + int num; + ssize_t w_size; + + char *buf; + size_t size; + + clear_output_num(); + + cb_data.cb = cb_check_result_json; + cb_data.data = "\"test\":\"msg\""; + + ctx = test_ctx_create(&cb_data); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("test_ctx_create failed"); + exit(EXIT_FAILURE); + } + + ret = flb_output_set(ctx->flb, ctx->o_ffd, + "match", "test", + "format", "json", + NULL); + TEST_CHECK(ret == 0); + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + /* use default host/port */ + fd = connect_tcp(NULL, -1); + if (!TEST_CHECK(fd >= 0)) { + exit(EXIT_FAILURE); + } + create_simple_json(&buf, &size); + w_size = send(fd, buf, size, 0); + flb_free(buf); + if (!TEST_CHECK(w_size == size)) { + TEST_MSG("failed to send, errno=%d", errno); + flb_socket_close(fd); + exit(EXIT_FAILURE); + } + + /* waiting to flush */ + flb_time_msleep(1500); + + num = get_output_num(); + if (!TEST_CHECK(num > 0)) { + TEST_MSG("no outputs"); + } + + flb_socket_close(fd); + test_ctx_destroy(ctx); +} + +void flb_test_forward_port() +{ + struct flb_lib_out_cb cb_data; + struct test_ctx *ctx; + + flb_sockfd_t fd; + int ret; + int num; + ssize_t w_size; + char *port = "24000"; + + char *buf; + size_t size; + + clear_output_num(); + + cb_data.cb = cb_check_result_json; + cb_data.data = "\"test\":\"msg\""; + + ctx = test_ctx_create(&cb_data); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("test_ctx_create failed"); + exit(EXIT_FAILURE); + } + + ret = flb_input_set(ctx->flb, ctx->i_ffd, + "port", port, + NULL); + TEST_CHECK(ret == 0); + + ret = flb_output_set(ctx->flb, ctx->o_ffd, + "match", "test", + "format", "json", + NULL); + TEST_CHECK(ret == 0); + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + /* use default host */ + fd = connect_tcp(NULL, atoi(port)); + if (!TEST_CHECK(fd >= 0)) { + exit(EXIT_FAILURE); + } + + create_simple_json(&buf, &size); + w_size = send(fd, buf, size, 0); + flb_free(buf); + if (!TEST_CHECK(w_size == size)) { + TEST_MSG("failed to send, errno=%d", errno); + flb_socket_close(fd); + exit(EXIT_FAILURE); + } + + /* waiting to flush */ + flb_time_msleep(1500); + + num = get_output_num(); + if (!TEST_CHECK(num > 0)) { + TEST_MSG("no outputs"); + } + + flb_socket_close(fd); + test_ctx_destroy(ctx); +} + +void flb_test_tag_prefix() +{ + struct flb_lib_out_cb cb_data; + struct test_ctx *ctx; + char *tag_prefix = "tag_"; + flb_sockfd_t fd; + int ret; + int num; + ssize_t w_size; + + char *buf; + size_t size; + + clear_output_num(); + + cb_data.cb = cb_check_result_json; + cb_data.data = "\"test\":\"msg\""; + + ctx = test_ctx_create(&cb_data); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("test_ctx_create failed"); + exit(EXIT_FAILURE); + } + ret = flb_input_set(ctx->flb, ctx->i_ffd, + "tag_prefix", tag_prefix, + NULL); + TEST_CHECK(ret == 0); + + ret = flb_output_set(ctx->flb, ctx->o_ffd, + "match", "tag_test", /*tag_prefix + "test"*/ + "format", "json", + NULL); + TEST_CHECK(ret == 0); + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + /* use default host/port */ + fd = connect_tcp(NULL, -1); + if (!TEST_CHECK(fd >= 0)) { + exit(EXIT_FAILURE); + } + + create_simple_json(&buf, &size); + w_size = send(fd, buf, size, 0); + flb_free(buf); + if (!TEST_CHECK(w_size == size)) { + TEST_MSG("failed to send, errno=%d", errno); + flb_socket_close(fd); + exit(EXIT_FAILURE); + } + + /* waiting to flush */ + flb_time_msleep(1500); + + num = get_output_num(); + if (!TEST_CHECK(num > 0)) { + TEST_MSG("no outputs"); + } + + flb_socket_close(fd); + test_ctx_destroy(ctx); +} + +#ifdef FLB_HAVE_UNIX_SOCKET +void flb_test_unix_path() +{ + struct flb_lib_out_cb cb_data; + struct test_ctx *ctx; + struct sockaddr_un sun; + flb_sockfd_t fd; + int ret; + int num; + ssize_t w_size; + char *unix_path = "in_forward_unix"; + + char *buf; + size_t size; + + clear_output_num(); + + cb_data.cb = cb_check_result_json; + cb_data.data = "\"test\":\"msg\""; + + ctx = test_ctx_create(&cb_data); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("test_ctx_create failed"); + exit(EXIT_FAILURE); + } + + ret = flb_input_set(ctx->flb, ctx->i_ffd, + "unix_path", unix_path, + NULL); + TEST_CHECK(ret == 0); + + ret = flb_output_set(ctx->flb, ctx->o_ffd, + "match", "test", + "format", "json", + NULL); + TEST_CHECK(ret == 0); + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + /* waiting to create socket */ + flb_time_msleep(200); + + memset(&sun, 0, sizeof(sun)); + fd = socket(AF_LOCAL, SOCK_STREAM, 0); + if (!TEST_CHECK(fd >= 0)) { + TEST_MSG("failed to socket %s, errno=%d", unix_path, errno); + unlink(unix_path); + exit(EXIT_FAILURE); + } + + sun.sun_family = AF_LOCAL; + strcpy(sun.sun_path, unix_path); + ret = connect(fd, (const struct sockaddr *)&sun, sizeof(sun)); + if (!TEST_CHECK(ret >= 0)) { + TEST_MSG("failed to connect, errno=%d", errno); + flb_socket_close(fd); + unlink(unix_path); + exit(EXIT_FAILURE); + } + create_simple_json(&buf, &size); + w_size = send(fd, buf, size, 0); + flb_free(buf); + if (!TEST_CHECK(w_size == size)) { + TEST_MSG("failed to write to %s", unix_path); + flb_socket_close(fd); + unlink(unix_path); + exit(EXIT_FAILURE); + } + + /* waiting to flush */ + flb_time_msleep(1500); + + num = get_output_num(); + if (!TEST_CHECK(num > 0)) { + TEST_MSG("no outputs"); + } + + flb_socket_close(fd); + test_ctx_destroy(ctx); +} + + +void flb_test_unix_perm() +{ + struct flb_lib_out_cb cb_data; + struct test_ctx *ctx; + struct sockaddr_un sun; + flb_sockfd_t fd; + int ret; + int num; + ssize_t w_size; + char *unix_path = "in_forward_unix"; + struct stat sb; + + char *buf; + size_t size; + + clear_output_num(); + + cb_data.cb = cb_check_result_json; + cb_data.data = "\"test\":\"msg\""; + + ctx = test_ctx_create(&cb_data); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("test_ctx_create failed"); + exit(EXIT_FAILURE); + } + + ret = flb_input_set(ctx->flb, ctx->i_ffd, + "unix_path", unix_path, + "unix_perm", "0600", + NULL); + TEST_CHECK(ret == 0); + + ret = flb_output_set(ctx->flb, ctx->o_ffd, + "match", "test", + "format", "json", + NULL); + TEST_CHECK(ret == 0); + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + /* waiting to create socket */ + flb_time_msleep(200); + + memset(&sun, 0, sizeof(sun)); + fd = socket(AF_LOCAL, SOCK_STREAM, 0); + if (!TEST_CHECK(fd >= 0)) { + TEST_MSG("failed to socket %s, errno=%d", unix_path, errno); + unlink(unix_path); + exit(EXIT_FAILURE); + } + + sun.sun_family = AF_LOCAL; + strcpy(sun.sun_path, unix_path); + ret = connect(fd, (const struct sockaddr *)&sun, sizeof(sun)); + if (!TEST_CHECK(ret >= 0)) { + TEST_MSG("failed to connect, errno=%d", errno); + flb_socket_close(fd); + unlink(unix_path); + exit(EXIT_FAILURE); + } + create_simple_json(&buf, &size); + w_size = send(fd, buf, size, 0); + flb_free(buf); + if (!TEST_CHECK(w_size == size)) { + TEST_MSG("failed to write to %s", unix_path); + flb_socket_close(fd); + unlink(unix_path); + exit(EXIT_FAILURE); + } + + /* waiting to flush */ + flb_time_msleep(1500); + + num = get_output_num(); + if (!TEST_CHECK(num > 0)) { + TEST_MSG("no outputs"); + } + + + /* File permission */ + ret = stat(unix_path, &sb); + if (!TEST_CHECK(ret == 0)) { + TEST_MSG("stat failed. errno=%d", errno); + test_ctx_destroy(ctx); + exit(EXIT_FAILURE); + } + + if (!TEST_CHECK((sb.st_mode & S_IRWXO) == 0)) { + TEST_MSG("Permssion(others) error. val=0x%x",sb.st_mode & S_IRWXO); + } + if (!TEST_CHECK((sb.st_mode & S_IRWXG) == 0)) { + TEST_MSG("Permssion(group) error. val=0x%x",sb.st_mode & S_IRWXG); + } + if (!TEST_CHECK((sb.st_mode & S_IRWXU) == (S_IRUSR | S_IWUSR))) { + TEST_MSG("Permssion(user) error. val=0x%x",sb.st_mode & S_IRWXU); + } + + flb_socket_close(fd); + test_ctx_destroy(ctx); +} +#endif /* FLB_HAVE_UNIX_SOCKET */ + + +TEST_LIST = { + {"forward", flb_test_forward}, + {"forward_port", flb_test_forward_port}, + {"tag_prefix", flb_test_tag_prefix}, +#ifdef FLB_HAVE_UNIX_SOCKET + {"unix_path", flb_test_unix_path}, + {"unix_perm", flb_test_unix_perm}, +#endif + {NULL, NULL} +}; + diff --git a/src/fluent-bit/tests/runtime/in_head.c b/src/fluent-bit/tests/runtime/in_head.c new file mode 120000 index 000000000..b194a8539 --- /dev/null +++ b/src/fluent-bit/tests/runtime/in_head.c @@ -0,0 +1 @@ +in_simple_systems.c
\ No newline at end of file diff --git a/src/fluent-bit/tests/runtime/in_http.c b/src/fluent-bit/tests/runtime/in_http.c new file mode 100644 index 000000000..700e581f8 --- /dev/null +++ b/src/fluent-bit/tests/runtime/in_http.c @@ -0,0 +1,444 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2019-2022 The Fluent Bit Authors + * Copyright (C) 2015-2018 Treasure Data Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <stdlib.h> +#include <fluent-bit.h> +#include <fluent-bit/flb_compat.h> +#include <fluent-bit/flb_time.h> +#include <fluent-bit/flb_pack.h> +#include <fluent-bit/flb_http_client.h> +#include <monkey/mk_core.h> +#include "flb_tests_runtime.h" + +#define JSON_CONTENT_TYPE "application/json" + +struct http_client_ctx { + struct flb_upstream *u; + struct flb_connection *u_conn; + struct flb_config *config; + struct mk_event_loop *evl; +}; + +struct test_ctx { + flb_ctx_t *flb; /* Fluent Bit library context */ + int i_ffd; /* Input fd */ + int f_ffd; /* Filter fd (unused) */ + int o_ffd; /* Output fd */ + struct http_client_ctx *httpc; +}; + + +pthread_mutex_t result_mutex = PTHREAD_MUTEX_INITIALIZER; +int num_output = 0; +static int get_output_num() +{ + int ret; + pthread_mutex_lock(&result_mutex); + ret = num_output; + pthread_mutex_unlock(&result_mutex); + + return ret; +} + +static void set_output_num(int num) +{ + pthread_mutex_lock(&result_mutex); + num_output = num; + pthread_mutex_unlock(&result_mutex); +} + +static void clear_output_num() +{ + set_output_num(0); +} + +/* Callback to check expected results */ +static int cb_check_result_json(void *record, size_t size, void *data) +{ + char *p; + char *expected; + char *result; + int num = get_output_num(); + + set_output_num(num+1); + + expected = (char *) data; + result = (char *) record; + + p = strstr(result, expected); + TEST_CHECK(p != NULL); + + if (p==NULL) { + flb_error("Expected to find: '%s' in result '%s'", + expected, result); + } + /* + * If you want to debug your test + * + * printf("Expect: '%s' in result '%s'", expected, result); + */ + flb_free(record); + return 0; +} + +struct http_client_ctx* http_client_ctx_create() +{ + struct http_client_ctx *ret_ctx = NULL; + struct mk_event_loop *evl = NULL; + + ret_ctx = flb_calloc(1, sizeof(struct http_client_ctx)); + if (!TEST_CHECK(ret_ctx != NULL)) { + flb_errno(); + TEST_MSG("flb_calloc(http_client_ctx) failed"); + return NULL; + } + + evl = mk_event_loop_create(16); + if (!TEST_CHECK(evl != NULL)) { + TEST_MSG("mk_event_loop failed"); + flb_free(ret_ctx); + return NULL; + } + ret_ctx->evl = evl; + flb_engine_evl_init(); + flb_engine_evl_set(evl); + + ret_ctx->config = flb_config_init(); + if(!TEST_CHECK(ret_ctx->config != NULL)) { + TEST_MSG("flb_config_init failed"); + mk_event_loop_destroy(evl); + flb_free(ret_ctx); + return NULL; + } + + ret_ctx->u = flb_upstream_create(ret_ctx->config, "127.0.0.1", 9880, 0, NULL); + if (!TEST_CHECK(ret_ctx->u != NULL)) { + TEST_MSG("flb_upstream_create failed"); + flb_config_exit(ret_ctx->config); + mk_event_loop_destroy(evl); + flb_free(ret_ctx); + return NULL; + } + + ret_ctx->u_conn = flb_upstream_conn_get(ret_ctx->u); + TEST_CHECK(ret_ctx->u_conn != NULL); + + ret_ctx->u_conn->upstream = ret_ctx->u; + + return ret_ctx; +} + +static struct test_ctx *test_ctx_create(struct flb_lib_out_cb *data) +{ + int i_ffd; + int o_ffd; + struct test_ctx *ctx = NULL; + + ctx = flb_calloc(1, sizeof(struct test_ctx)); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("flb_calloc failed"); + flb_errno(); + return NULL; + } + + /* Service config */ + ctx->flb = flb_create(); + flb_service_set(ctx->flb, + "Flush", "0.200000000", + "Grace", "1", + "Log_Level", "error", + NULL); + + /* Input */ + i_ffd = flb_input(ctx->flb, (char *) "http", NULL); + TEST_CHECK(i_ffd >= 0); + ctx->i_ffd = i_ffd; + + /* Output */ + o_ffd = flb_output(ctx->flb, (char *) "lib", (void *) data); + ctx->o_ffd = o_ffd; + + return ctx; +} + +int http_client_ctx_destroy(struct http_client_ctx* ctx) +{ + if (!TEST_CHECK(ctx != NULL)) { + return -1; + } + if (ctx->u) { + flb_upstream_destroy(ctx->u); + } + if (ctx->config) { + flb_config_exit(ctx->config); + } + if (ctx->evl) { + mk_event_loop_destroy(ctx->evl); + } + + flb_free(ctx); + return 0; +} + +static void test_ctx_destroy(struct test_ctx *ctx) +{ + TEST_CHECK(ctx != NULL); + if (ctx->httpc) { + http_client_ctx_destroy(ctx->httpc); + } + + sleep(1); + flb_stop(ctx->flb); + flb_destroy(ctx->flb); + flb_free(ctx); +} + +void flb_test_http() +{ + struct flb_lib_out_cb cb_data; + struct test_ctx *ctx; + struct flb_http_client *c; + int ret; + int num; + size_t b_sent; + + char *buf = "{\"test\":\"msg\"}"; + + clear_output_num(); + + cb_data.cb = cb_check_result_json; + cb_data.data = "\"test\":\"msg\""; + + ctx = test_ctx_create(&cb_data); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("test_ctx_create failed"); + exit(EXIT_FAILURE); + } + + ret = flb_output_set(ctx->flb, ctx->o_ffd, + "match", "*", + "format", "json", + NULL); + TEST_CHECK(ret == 0); + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + ctx->httpc = http_client_ctx_create(); + TEST_CHECK(ctx->httpc != NULL); + + c = flb_http_client(ctx->httpc->u_conn, FLB_HTTP_POST, "/", buf, strlen(buf), + "127.0.0.1", 9880, NULL, 0); + ret = flb_http_add_header(c, FLB_HTTP_HEADER_CONTENT_TYPE, strlen(FLB_HTTP_HEADER_CONTENT_TYPE), + JSON_CONTENT_TYPE, strlen(JSON_CONTENT_TYPE)); + TEST_CHECK(ret == 0); + if (!TEST_CHECK(c != NULL)) { + TEST_MSG("http_client failed"); + exit(EXIT_FAILURE); + } + + ret = flb_http_do(c, &b_sent); + if (!TEST_CHECK(ret == 0)) { + TEST_MSG("ret error. ret=%d\n", ret); + } + else if (!TEST_CHECK(b_sent > 0)){ + TEST_MSG("b_sent size error. b_sent = %lu\n", b_sent); + } + else if (!TEST_CHECK(c->resp.status == 201)) { + TEST_MSG("http response code error. expect: 201, got: %d\n", c->resp.status); + } + + /* waiting to flush */ + flb_time_msleep(1500); + + num = get_output_num(); + if (!TEST_CHECK(num > 0)) { + TEST_MSG("no outputs"); + } + flb_http_client_destroy(c); + flb_upstream_conn_release(ctx->httpc->u_conn); + test_ctx_destroy(ctx); +} + +void flb_test_http_successful_response_code(char *response_code) +{ + struct flb_lib_out_cb cb_data; + struct test_ctx *ctx; + struct flb_http_client *c; + int ret; + int num; + size_t b_sent; + + char *buf = "{\"test\":\"msg\"}"; + + clear_output_num(); + + cb_data.cb = cb_check_result_json; + cb_data.data = "\"test\":\"msg\""; + + ctx = test_ctx_create(&cb_data); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("test_ctx_create failed"); + exit(EXIT_FAILURE); + } + + ret = flb_input_set(ctx->flb, ctx->i_ffd, + "successful_response_code", response_code, + NULL); + TEST_CHECK(ret == 0); + + ret = flb_output_set(ctx->flb, ctx->o_ffd, + "match", "*", + "format", "json", + NULL); + TEST_CHECK(ret == 0); + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + ctx->httpc = http_client_ctx_create(); + TEST_CHECK(ctx->httpc != NULL); + + c = flb_http_client(ctx->httpc->u_conn, FLB_HTTP_POST, "/", buf, strlen(buf), + "127.0.0.1", 9880, NULL, 0); + ret = flb_http_add_header(c, FLB_HTTP_HEADER_CONTENT_TYPE, strlen(FLB_HTTP_HEADER_CONTENT_TYPE), + JSON_CONTENT_TYPE, strlen(JSON_CONTENT_TYPE)); + TEST_CHECK(ret == 0); + if (!TEST_CHECK(c != NULL)) { + TEST_MSG("http_client failed"); + exit(EXIT_FAILURE); + } + + ret = flb_http_do(c, &b_sent); + if (!TEST_CHECK(ret == 0)) { + TEST_MSG("ret error. ret=%d\n", ret); + } + else if (!TEST_CHECK(b_sent > 0)){ + TEST_MSG("b_sent size error. b_sent = %lu\n", b_sent); + } + else if (!TEST_CHECK(c->resp.status == atoi(response_code))) { + TEST_MSG("http response code error. expect: %d, got: %d\n", atoi(response_code), c->resp.status); + } + + /* waiting to flush */ + flb_time_msleep(1500); + + num = get_output_num(); + if (!TEST_CHECK(num > 0)) { + TEST_MSG("no outputs"); + } + flb_http_client_destroy(c); + flb_upstream_conn_release(ctx->httpc->u_conn); + test_ctx_destroy(ctx); +} + +void flb_test_http_successful_response_code_200() +{ + flb_test_http_successful_response_code("200"); +} + +void flb_test_http_successful_response_code_204() +{ + flb_test_http_successful_response_code("204"); +} + +void flb_test_http_tag_key() +{ + struct flb_lib_out_cb cb_data; + struct test_ctx *ctx; + struct flb_http_client *c; + int ret; + int num; + size_t b_sent; + + char *buf = "{\"test\":\"msg\", \"tag\":\"new_tag\"}"; + + clear_output_num(); + + cb_data.cb = cb_check_result_json; + cb_data.data = "\"test\":\"msg\""; + + ctx = test_ctx_create(&cb_data); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("test_ctx_create failed"); + exit(EXIT_FAILURE); + } + + ret = flb_input_set(ctx->flb, ctx->i_ffd, + "tag_key", "tag", + NULL); + TEST_CHECK(ret == 0); + + ret = flb_output_set(ctx->flb, ctx->o_ffd, + "match", "new_tag", + "format", "json", + NULL); + TEST_CHECK(ret == 0); + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + ctx->httpc = http_client_ctx_create(); + TEST_CHECK(ctx->httpc != NULL); + + c = flb_http_client(ctx->httpc->u_conn, FLB_HTTP_POST, "/", buf, strlen(buf), + "127.0.0.1", 9880, NULL, 0); + ret = flb_http_add_header(c, FLB_HTTP_HEADER_CONTENT_TYPE, strlen(FLB_HTTP_HEADER_CONTENT_TYPE), + JSON_CONTENT_TYPE, strlen(JSON_CONTENT_TYPE)); + TEST_CHECK(ret == 0); + if (!TEST_CHECK(c != NULL)) { + TEST_MSG("http_client failed"); + exit(EXIT_FAILURE); + } + + ret = flb_http_do(c, &b_sent); + if (!TEST_CHECK(ret == 0)) { + TEST_MSG("ret error. ret=%d\n", ret); + } + else if (!TEST_CHECK(b_sent > 0)){ + TEST_MSG("b_sent size error. b_sent = %lu\n", b_sent); + } + else if (!TEST_CHECK(c->resp.status == 201)) { + TEST_MSG("http response code error. expect: 201, got: %d\n", c->resp.status); + } + + /* waiting to flush */ + flb_time_msleep(1500); + + num = get_output_num(); + if (!TEST_CHECK(num > 0)) { + TEST_MSG("no outputs"); + } + flb_http_client_destroy(c); + flb_upstream_conn_release(ctx->httpc->u_conn); + test_ctx_destroy(ctx); +} + +TEST_LIST = { + {"http", flb_test_http}, + {"successful_response_code_200", flb_test_http_successful_response_code_200}, + {"successful_response_code_204", flb_test_http_successful_response_code_204}, + {"tag_key", flb_test_http_tag_key}, + {NULL, NULL} +}; + diff --git a/src/fluent-bit/tests/runtime/in_mem.c b/src/fluent-bit/tests/runtime/in_mem.c new file mode 120000 index 000000000..b194a8539 --- /dev/null +++ b/src/fluent-bit/tests/runtime/in_mem.c @@ -0,0 +1 @@ +in_simple_systems.c
\ No newline at end of file diff --git a/src/fluent-bit/tests/runtime/in_mqtt.c b/src/fluent-bit/tests/runtime/in_mqtt.c new file mode 100644 index 000000000..0e38d0c51 --- /dev/null +++ b/src/fluent-bit/tests/runtime/in_mqtt.c @@ -0,0 +1,396 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2019-2023 The Fluent Bit Authors + * Copyright (C) 2015-2018 Treasure Data Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <fluent-bit.h> +#include <fluent-bit/flb_compat.h> +#include <fluent-bit/flb_time.h> +#include <fluent-bit/flb_socket.h> +#include "flb_tests_runtime.h" + +struct test_ctx { + flb_ctx_t *flb; /* Fluent Bit library context */ + int i_ffd; /* Input fd */ + int f_ffd; /* Filter fd (unused) */ + int o_ffd; /* Output fd */ +}; + +pthread_mutex_t result_mutex = PTHREAD_MUTEX_INITIALIZER; +int num_output = 0; +static int get_output_num() +{ + int ret; + pthread_mutex_lock(&result_mutex); + ret = num_output; + pthread_mutex_unlock(&result_mutex); + + return ret; +} + +static void set_output_num(int num) +{ + pthread_mutex_lock(&result_mutex); + num_output = num; + pthread_mutex_unlock(&result_mutex); +} + +static void clear_output_num() +{ + set_output_num(0); +} + +struct str_list { + size_t size; + char **lists; +}; + +/* Callback to check expected results */ +static int cb_check_json_str_list(void *record, size_t size, void *data) +{ + char *p; + char *result; + int num = get_output_num(); + size_t i; + struct str_list *l = (struct str_list*)data; + + if (!TEST_CHECK(l != NULL)) { + flb_error("Data is NULL"); + flb_free(record); + return 0; + } + set_output_num(num+1); + + result = (char *) record; + + for (i=0; i<l->size; i++) { + p = strstr(result, l->lists[i]); + if(!TEST_CHECK(p != NULL)) { + flb_error("Expected to find: '%s' in result '%s'", + l->lists[i], result); + } + } + flb_free(record); + return 0; +} + +static struct test_ctx *test_ctx_create(struct flb_lib_out_cb *data) +{ + int i_ffd; + int o_ffd; + int ret; + struct test_ctx *ctx = NULL; + + ctx = flb_malloc(sizeof(struct test_ctx)); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("malloc failed"); + flb_errno(); + return NULL; + } + + /* Service config */ + ctx->flb = flb_create(); + flb_service_set(ctx->flb, + "Flush", "0.200000000", + "Grace", "1", + "Log_Level", "error", + NULL); + + /* Input */ + i_ffd = flb_input(ctx->flb, (char *) "mqtt", NULL); + TEST_CHECK(i_ffd >= 0); + ctx->i_ffd = i_ffd; + + /* Output */ + o_ffd = flb_output(ctx->flb, (char *) "lib", (void *) data); + ctx->o_ffd = o_ffd; + ret = flb_output_set(ctx->flb, ctx->o_ffd, + "format", "json", + NULL); + TEST_CHECK(ret == 0); + + return ctx; +} + +static void test_ctx_destroy(struct test_ctx *ctx) +{ + TEST_CHECK(ctx != NULL); + + sleep(1); + flb_stop(ctx->flb); + flb_destroy(ctx->flb); + flb_free(ctx); +} + +#define DEFAULT_HOST "127.0.0.1" +#define DEFAULT_PORT 1883 +static flb_sockfd_t connect_tcp(char *in_host, int in_port) +{ + int port = in_port; + char *host = in_host; + flb_sockfd_t fd; + int ret; + struct sockaddr_in addr; + + if (host == NULL) { + host = DEFAULT_HOST; + } + if (port < 0) { + port = DEFAULT_PORT; + } + + memset(&addr, 0, sizeof(addr)); + fd = socket(PF_INET, SOCK_STREAM, 0); + if (!TEST_CHECK(fd >= 0)) { + TEST_MSG("failed to socket. host=%s port=%d errno=%d", host, port, errno); + return -1; + } + + addr.sin_family = AF_INET; + addr.sin_addr.s_addr = inet_addr(host); + addr.sin_port = htons(port); + + ret = connect(fd, (const struct sockaddr *)&addr, sizeof(addr)); + if (!TEST_CHECK(ret >= 0)) { + TEST_MSG("failed to connect. host=%s port=%d errno=%d", host, port, errno); + flb_socket_close(fd); + return -1; + } + return fd; +} + +/* http://docs.oasis-open.org/mqtt/mqtt/v3.1.1/os/mqtt-v3.1.1-os.html#_Toc398718028 */ +int send_CONNECT(flb_sockfd_t fd) +{ + ssize_t w_size; + char buf[] = {0x10, 0xa, + 0x0,0x4, 'M', 'Q', 'T', 'T', 0x4, 0xce, 0x0, 0xa + }; + + w_size = send(fd, buf, sizeof(buf), 0); + if (!TEST_CHECK(w_size == sizeof(buf))) { + TEST_MSG("failed to send CONNECT, errno=%d", errno); + return -1; + } + return 0; +} + +/* http://docs.oasis-open.org/mqtt/mqtt/v3.1.1/os/mqtt-v3.1.1-os.html#_Toc398718033 */ +int recv_CONNACK(flb_sockfd_t fd) +{ + ssize_t r_size; + char buf[1024] = {0}; + + r_size = recv(fd, &buf[0], sizeof(buf), 0); + if (!TEST_CHECK(r_size != -1)) { + TEST_MSG("failed to recv CONNACK, errno=%d", errno); + return -1; + } + return 0; +} + +/* http://docs.oasis-open.org/mqtt/mqtt/v3.1.1/os/mqtt-v3.1.1-os.html#_Toc398718037 */ +int send_PUBLISH(flb_sockfd_t fd, const char *payload, size_t payload_size) +{ + ssize_t w_size; + char buf[1024] = {0}; + + buf[0] = 0x30; + + buf[2] = 0x0; + buf[3] = 0x3; + buf[4] = 'a'; + buf[5] = '/'; + buf[6] = 'b'; + /* No packet id since QoS Level is 0 */ + buf[1] = 5 + (char)payload_size; + + if (!TEST_CHECK(sizeof(buf) >= buf[1])) { + TEST_MSG("payload is too long"); + return -1; + } + + strncpy(&buf[7], payload, payload_size); + + w_size = send(fd, buf, sizeof(buf), 0); + if (!TEST_CHECK(w_size == sizeof(buf))) { + TEST_MSG("failed to send PUBLISH, errno=%d", errno); + return -1; + } + return 0; +} + +void flb_test_mqtt() +{ + struct flb_lib_out_cb cb_data; + struct test_ctx *ctx; + flb_sockfd_t fd; + int ret; + int num; + + char *expected_strs[] = {"\"key\":\"val\""}; + struct str_list expected = { + .size = sizeof(expected_strs)/sizeof(char*), + .lists = &expected_strs[0], + }; + const char *payload = "{\"key\":\"val\"}"; + size_t payload_size = strlen(payload); + + clear_output_num(); + + cb_data.cb = cb_check_json_str_list; + cb_data.data = &expected; + + ctx = test_ctx_create(&cb_data); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("test_ctx_create failed"); + exit(EXIT_FAILURE); + } + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + /* use default host/port */ + fd = connect_tcp(NULL, -1); + if (!TEST_CHECK(fd >= 0)) { + test_ctx_destroy(ctx); + exit(EXIT_FAILURE); + } + + ret = send_CONNECT(fd); + if (!TEST_CHECK(ret == 0)) { + TEST_MSG("failed to send, errno=%d", errno); + flb_socket_close(fd); + test_ctx_destroy(ctx); + exit(EXIT_FAILURE); + } + + ret = recv_CONNACK(fd); + if (!TEST_CHECK(ret == 0)) { + TEST_MSG("failed to recv, errno=%d", errno); + flb_socket_close(fd); + test_ctx_destroy(ctx); + exit(EXIT_FAILURE); + } + + ret = send_PUBLISH(fd, payload, payload_size); + if (!TEST_CHECK(ret == 0)) { + TEST_MSG("failed to send, errno=%d", errno); + flb_socket_close(fd); + test_ctx_destroy(ctx); + exit(EXIT_FAILURE); + } + + /* waiting to flush */ + flb_time_msleep(500); + + num = get_output_num(); + if (!TEST_CHECK(num > 0)) { + TEST_MSG("no outputs"); + } + + flb_socket_close(fd); + test_ctx_destroy(ctx); +} + +void flb_test_payload_key() +{ + struct flb_lib_out_cb cb_data; + struct test_ctx *ctx; + flb_sockfd_t fd; + int ret; + int num; + + char *expected_strs[] = {"\"payload_k\":{\"key\":\"val\"}"}; + struct str_list expected = { + .size = sizeof(expected_strs)/sizeof(char*), + .lists = &expected_strs[0], + }; + const char *payload = "{\"key\":\"val\"}"; + size_t payload_size = strlen(payload); + + clear_output_num(); + + cb_data.cb = cb_check_json_str_list; + cb_data.data = &expected; + + ctx = test_ctx_create(&cb_data); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("test_ctx_create failed"); + exit(EXIT_FAILURE); + } + + ret = flb_input_set(ctx->flb, ctx->i_ffd, + "payload_key", "payload_k", + NULL); + TEST_CHECK(ret == 0); + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + /* use default host/port */ + fd = connect_tcp(NULL, -1); + if (!TEST_CHECK(fd >= 0)) { + test_ctx_destroy(ctx); + exit(EXIT_FAILURE); + } + + ret = send_CONNECT(fd); + if (!TEST_CHECK(ret == 0)) { + TEST_MSG("failed to send, errno=%d", errno); + flb_socket_close(fd); + test_ctx_destroy(ctx); + exit(EXIT_FAILURE); + } + + ret = recv_CONNACK(fd); + if (!TEST_CHECK(ret == 0)) { + TEST_MSG("failed to recv, errno=%d", errno); + flb_socket_close(fd); + test_ctx_destroy(ctx); + exit(EXIT_FAILURE); + } + + ret = send_PUBLISH(fd, payload, payload_size); + if (!TEST_CHECK(ret == 0)) { + TEST_MSG("failed to send, errno=%d", errno); + flb_socket_close(fd); + test_ctx_destroy(ctx); + exit(EXIT_FAILURE); + } + + /* waiting to flush */ + flb_time_msleep(500); + + num = get_output_num(); + if (!TEST_CHECK(num > 0)) { + TEST_MSG("no outputs"); + } + + flb_socket_close(fd); + test_ctx_destroy(ctx); +} + +TEST_LIST = { + {"mqtt", flb_test_mqtt}, + {"payload_key", flb_test_payload_key}, + {NULL, NULL} +}; + diff --git a/src/fluent-bit/tests/runtime/in_netif.c b/src/fluent-bit/tests/runtime/in_netif.c new file mode 100644 index 000000000..ff66d9e13 --- /dev/null +++ b/src/fluent-bit/tests/runtime/in_netif.c @@ -0,0 +1,350 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2019-2022 The Fluent Bit Authors + * Copyright (C) 2015-2018 Treasure Data Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <fluent-bit.h> +#include <fluent-bit/flb_sds.h> +#include <fluent-bit/flb_time.h> +#include <sys/types.h> +#include <ifaddrs.h> +#include <errno.h> +#include "flb_tests_runtime.h" + +struct test_ctx { + flb_ctx_t *flb; /* Fluent Bit library context */ + int i_ffd; /* Input fd */ + int f_ffd; /* Filter fd (unused) */ + int o_ffd; /* Output fd */ +}; + +pthread_mutex_t result_mutex = PTHREAD_MUTEX_INITIALIZER; +int num_output = 0; + +static struct test_ctx *test_ctx_create(struct flb_lib_out_cb *data) +{ + int i_ffd; + int o_ffd; + struct test_ctx *ctx = NULL; + + ctx = flb_malloc(sizeof(struct test_ctx)); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("malloc failed"); + flb_errno(); + return NULL; + } + + /* Service config */ + ctx->flb = flb_create(); + flb_service_set(ctx->flb, + "Flush", "0.200000000", + "Grace", "1", + "Log_Level", "error", + NULL); + + /* Input */ + i_ffd = flb_input(ctx->flb, (char *) "netif", NULL); + TEST_CHECK(i_ffd >= 0); + ctx->i_ffd = i_ffd; + + /* Output */ + o_ffd = flb_output(ctx->flb, (char *) "lib", (void *) data); + ctx->o_ffd = o_ffd; + + return ctx; +} + +static void test_ctx_destroy(struct test_ctx *ctx) +{ + TEST_CHECK(ctx != NULL); + + sleep(1); + flb_stop(ctx->flb); + flb_destroy(ctx->flb); + flb_free(ctx); +} + +static void clear_output_num() +{ + pthread_mutex_lock(&result_mutex); + num_output = 0; + pthread_mutex_unlock(&result_mutex); +} + +static int get_output_num() +{ + int ret; + pthread_mutex_lock(&result_mutex); + ret = num_output; + pthread_mutex_unlock(&result_mutex); + + return ret; +} + +static flb_sds_t get_interface_str() +{ + struct ifaddrs *ifap = NULL; + flb_sds_t ret_str; + int ret = 0; + + ret = getifaddrs(&ifap); + if (!TEST_CHECK(ret == 0)) { + TEST_MSG("getifaddrs failed errno=%d", errno); + return NULL; + } + + if (!TEST_CHECK(ifap != NULL)) { + TEST_MSG("failed to get ifaddrs"); + return NULL; + } + ret_str = flb_sds_create(ifap->ifa_name); + freeifaddrs(ifap); + + /* printf("ret:%s\n", ret_str); */ + + return ret_str; +} + +static int cb_count_msgpack_map_size(void *record, size_t size, void *data) +{ + msgpack_unpacked result; + msgpack_object *obj; + struct flb_time tm; + size_t off = 0; + int map_size; + + if (!TEST_CHECK(data != NULL)) { + flb_error("data is NULL"); + } + + /* Iterate each item array and apply rules */ + msgpack_unpacked_init(&result); + while (msgpack_unpack_next(&result, record, size, &off) == MSGPACK_UNPACK_SUCCESS) { + if (!TEST_CHECK(result.data.type == MSGPACK_OBJECT_ARRAY)) { + continue; + } + flb_time_pop_from_msgpack(&tm, &result, &obj); + + if (!TEST_CHECK(obj->type == MSGPACK_OBJECT_MAP)) { + continue; + } + map_size = get_output_num(); + if (obj->via.map.size > map_size) { + pthread_mutex_lock(&result_mutex); + num_output = obj->via.map.size; + pthread_mutex_unlock(&result_mutex); + } + } + msgpack_unpacked_destroy(&result); + + flb_free(record); + return 0; +} + +static int cb_count_msgpack_events(void *record, size_t size, void *data) +{ + msgpack_unpacked result; + size_t off = 0; + + if (!TEST_CHECK(data != NULL)) { + flb_error("data is NULL"); + } + + /* Iterate each item array and apply rules */ + msgpack_unpacked_init(&result); + while (msgpack_unpack_next(&result, record, size, &off) == MSGPACK_UNPACK_SUCCESS) { + pthread_mutex_lock(&result_mutex); + num_output++; + pthread_mutex_unlock(&result_mutex); + } + msgpack_unpacked_destroy(&result); + + flb_free(record); + return 0; +} + +static void flb_test_normal() +{ + int ret; + int got; + int unused = 0; + flb_sds_t ifname; + struct test_ctx *ctx; + struct flb_lib_out_cb cb_data; + + cb_data.cb = cb_count_msgpack_events; + cb_data.data = &unused; + + ctx = test_ctx_create(&cb_data); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("test_ctx_create failed"); + exit(EXIT_FAILURE); + } + + ifname = get_interface_str(); + if (!TEST_CHECK(ifname != NULL)) { + TEST_MSG("can't get interface name"); + flb_free(ctx); + exit(EXIT_FAILURE); + } + + ret = flb_input_set(ctx->flb, ctx->i_ffd, + "interface", ifname, + "interval_sec", "0", + "interval_nsec", "500000000", /* 500 ms */ + "test_at_init", "true", + NULL); + TEST_CHECK(ret==0); + + flb_sds_destroy(ifname); + + clear_output_num(); + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + flb_time_msleep(1500); /* waiting flush */ + got = get_output_num(); + + if (!TEST_CHECK(got != 0)) { + TEST_MSG("expect: >=1 got: %d", got); + } + + test_ctx_destroy(ctx); +} + +static void flb_test_no_interface() +{ + int ret; + int unused = 0; + struct test_ctx *ctx; + struct flb_lib_out_cb cb_data; + + cb_data.cb = cb_count_msgpack_events; + cb_data.data = &unused; + + ctx = test_ctx_create(&cb_data); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("test_ctx_create failed"); + exit(EXIT_FAILURE); + } + + clear_output_num(); + + /* It should be error */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret != 0); + + test_ctx_destroy(ctx); +} + +static void flb_test_invalid_interface() +{ + int ret; + int unused = 0; + char *ifname = "\t\n"; + struct test_ctx *ctx; + struct flb_lib_out_cb cb_data; + + + cb_data.cb = cb_count_msgpack_events; + cb_data.data = &unused; + ctx = test_ctx_create(&cb_data); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("test_ctx_create failed"); + exit(EXIT_FAILURE); + } + + ret = flb_input_set(ctx->flb, ctx->i_ffd, + "interface", ifname, + "interval_sec", "0", + "interval_nsec", "500000000", /* 500 ms */ + "test_at_init", "true", + NULL); + TEST_CHECK(ret==0); + + clear_output_num(); + + /* It should be error */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret != 0); + + test_ctx_destroy(ctx); +} + +static void flb_test_verbose() +{ + int ret; + int got; + int unused = 0; + int expect = 10; + flb_sds_t ifname; + struct test_ctx *ctx; + struct flb_lib_out_cb cb_data; + + cb_data.cb = cb_count_msgpack_map_size; + cb_data.data = &unused; + + ctx = test_ctx_create(&cb_data); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("test_ctx_create failed"); + exit(EXIT_FAILURE); + } + + ifname = get_interface_str(); + if (!TEST_CHECK(ifname != NULL)) { + TEST_MSG("can't get interface name"); + flb_free(ctx); + exit(EXIT_FAILURE); + } + + ret = flb_input_set(ctx->flb, ctx->i_ffd, + "interface", ifname, + "interval_sec", "0", + "interval_nsec", "500000000", /* 500 ms */ + "test_at_init", "true", + "verbose", "true", + NULL); + TEST_CHECK(ret==0); + + flb_sds_destroy(ifname); + + clear_output_num(); + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + flb_time_msleep(1500); /* waiting flush */ + got = get_output_num(); + + if (!TEST_CHECK(got > expect)) { + TEST_MSG("expect: >10 got: %d", got); + } + + test_ctx_destroy(ctx); +} + +TEST_LIST = { + {"normal case", flb_test_normal}, + {"no interface", flb_test_no_interface}, + {"invalid interface", flb_test_invalid_interface}, + {"verbose", flb_test_verbose}, + {NULL, NULL} +}; diff --git a/src/fluent-bit/tests/runtime/in_opentelemetry.c b/src/fluent-bit/tests/runtime/in_opentelemetry.c new file mode 100644 index 000000000..6f721cbec --- /dev/null +++ b/src/fluent-bit/tests/runtime/in_opentelemetry.c @@ -0,0 +1,376 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2019-2022 The Fluent Bit Authors + * Copyright (C) 2015-2018 Treasure Data Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <stdlib.h> +#include <fluent-bit.h> +#include <fluent-bit/flb_compat.h> +#include <fluent-bit/flb_time.h> +#include <fluent-bit/flb_pack.h> +#include <fluent-bit/flb_http_client.h> +#include <monkey/mk_core.h> +#include "flb_tests_runtime.h" + +#define JSON_CONTENT_TYPE "application/json" + +#define PORT_OTEL 4318 +#define TEST_MSG_OTEL_LOGS "{\"resourceLogs\":[{\"resource\":{},\"scopeLogs\":[{\"scope\":{},\"logRecords\":[{\"timeUnixNano\":\"1660296023390371588\",\"body\":{\"stringValue\":\"{\\\"message\\\":\\\"test\\\"}\"}}]}]}]}" +#define TEST_CB_MSG_OTEL_LOGS "[1660296024.698112,{\"log\":\"{\\\"message\\\":\\\"test\\\"}\"}]" + +#define V1_ENDPOINT_LOGS "/v1/logs" + +struct http_client_ctx { + struct flb_upstream *u; + struct flb_connection *u_conn; + struct flb_config *config; + struct mk_event_loop *evl; +}; + +struct test_ctx { + flb_ctx_t *flb; /* Fluent Bit library context */ + int i_ffd; /* Input fd */ + int f_ffd; /* Filter fd (unused) */ + int o_ffd; /* Output fd */ + struct http_client_ctx *httpc; +}; + + +pthread_mutex_t result_mutex = PTHREAD_MUTEX_INITIALIZER; +int num_output = 0; +static int get_output_num() +{ + int ret; + pthread_mutex_lock(&result_mutex); + ret = num_output; + pthread_mutex_unlock(&result_mutex); + + return ret; +} + +static void set_output_num(int num) +{ + pthread_mutex_lock(&result_mutex); + num_output = num; + pthread_mutex_unlock(&result_mutex); +} + +static void clear_output_num() +{ + set_output_num(0); +} + +/* Callback to check expected results */ +static int cb_check_result_json(void *record, size_t size, void *data) +{ + char *p; + char *expected; + char *result; + int num = get_output_num(); + + set_output_num(num+1); + + expected = (char *) data; + result = (char *) record; + + p = strstr(result, expected); + TEST_CHECK(p != NULL); + + if (p==NULL) { + flb_error("Expected to find: '%s' in result '%s'", + expected, result); + } + /* + * If you want to debug your test + * + * printf("Expect: '%s' in result '%s'", expected, result); + */ + flb_free(record); + return 0; +} + +struct http_client_ctx* http_client_ctx_create() +{ + struct http_client_ctx *ret_ctx = NULL; + struct mk_event_loop *evl = NULL; + + ret_ctx = flb_calloc(1, sizeof(struct http_client_ctx)); + if (!TEST_CHECK(ret_ctx != NULL)) { + flb_errno(); + TEST_MSG("flb_calloc(http_client_ctx) failed"); + return NULL; + } + + evl = mk_event_loop_create(16); + if (!TEST_CHECK(evl != NULL)) { + TEST_MSG("mk_event_loop failed"); + flb_free(ret_ctx); + return NULL; + } + ret_ctx->evl = evl; + flb_engine_evl_init(); + flb_engine_evl_set(evl); + + ret_ctx->config = flb_config_init(); + if(!TEST_CHECK(ret_ctx->config != NULL)) { + TEST_MSG("flb_config_init failed"); + mk_event_loop_destroy(evl); + flb_free(ret_ctx); + return NULL; + } + + ret_ctx->u = flb_upstream_create(ret_ctx->config, "127.0.0.1", PORT_OTEL, 0, NULL); + if (!TEST_CHECK(ret_ctx->u != NULL)) { + TEST_MSG("flb_upstream_create failed"); + flb_config_exit(ret_ctx->config); + mk_event_loop_destroy(evl); + flb_free(ret_ctx); + return NULL; + } + + ret_ctx->u_conn = flb_upstream_conn_get(ret_ctx->u); + TEST_CHECK(ret_ctx->u_conn != NULL); + + ret_ctx->u_conn->upstream = ret_ctx->u; + + return ret_ctx; +} + +static struct test_ctx *test_ctx_create(struct flb_lib_out_cb *data) +{ + int i_ffd; + int o_ffd; + struct test_ctx *ctx = NULL; + + ctx = flb_calloc(1, sizeof(struct test_ctx)); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("flb_calloc failed"); + flb_errno(); + return NULL; + } + + /* Service config */ + ctx->flb = flb_create(); + flb_service_set(ctx->flb, + "Flush", "0.200000000", + "Grace", "1", + "Log_Level", "error", + NULL); + + /* Input */ + i_ffd = flb_input(ctx->flb, (char *) "opentelemetry", NULL); + TEST_CHECK(i_ffd >= 0); + ctx->i_ffd = i_ffd; + + /* Output */ + o_ffd = flb_output(ctx->flb, (char *) "lib", (void *) data); + ctx->o_ffd = o_ffd; + + return ctx; +} + +int http_client_ctx_destroy(struct http_client_ctx* ctx) +{ + if (!TEST_CHECK(ctx != NULL)) { + return -1; + } + if (ctx->u) { + flb_upstream_destroy(ctx->u); + } + if (ctx->config) { + flb_config_exit(ctx->config); + } + if (ctx->evl) { + mk_event_loop_destroy(ctx->evl); + } + + flb_free(ctx); + return 0; +} + +static void test_ctx_destroy(struct test_ctx *ctx) +{ + TEST_CHECK(ctx != NULL); + if (ctx->httpc) { + http_client_ctx_destroy(ctx->httpc); + } + + sleep(1); + flb_stop(ctx->flb); + flb_destroy(ctx->flb); + flb_free(ctx); +} + +void flb_test_otel_logs() +{ + struct flb_lib_out_cb cb_data; + struct test_ctx *ctx; + struct flb_http_client *c; + int ret; + int num; + size_t b_sent; + + char *buf = TEST_MSG_OTEL_LOGS; + + clear_output_num(); + + cb_data.cb = cb_check_result_json; + cb_data.data = TEST_CB_MSG_OTEL_LOGS; + + ctx = test_ctx_create(&cb_data); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("test_ctx_create failed"); + exit(EXIT_FAILURE); + } + + ret = flb_output_set(ctx->flb, ctx->o_ffd, + "match", "*", + "format", "json", + NULL); + TEST_CHECK(ret == 0); + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + ctx->httpc = http_client_ctx_create(); + TEST_CHECK(ctx->httpc != NULL); + + c = flb_http_client(ctx->httpc->u_conn, FLB_HTTP_POST, V1_ENDPOINT_LOGS, buf, strlen(buf), + "127.0.0.1", PORT_OTEL, NULL, 0); + ret = flb_http_add_header(c, FLB_HTTP_HEADER_CONTENT_TYPE, strlen(FLB_HTTP_HEADER_CONTENT_TYPE), + JSON_CONTENT_TYPE, strlen(JSON_CONTENT_TYPE)); + TEST_CHECK(ret == 0); + if (!TEST_CHECK(c != NULL)) { + TEST_MSG("http_client failed"); + exit(EXIT_FAILURE); + } + + ret = flb_http_do(c, &b_sent); + if (!TEST_CHECK(ret == 0)) { + TEST_MSG("ret error. ret=%d\n", ret); + } + else if (!TEST_CHECK(b_sent > 0)){ + TEST_MSG("b_sent size error. b_sent = %lu\n", b_sent); + } + else if (!TEST_CHECK(c->resp.status == 201)) { + TEST_MSG("http response code error. expect: 201, got: %d\n", c->resp.status); + } + + /* waiting to flush */ + flb_time_msleep(1500); + + num = get_output_num(); + if (!TEST_CHECK(num > 0)) { + TEST_MSG("no outputs"); + } + flb_http_client_destroy(c); + flb_upstream_conn_release(ctx->httpc->u_conn); + test_ctx_destroy(ctx); +} + +void flb_test_otel_successful_response_code(char *response_code) +{ + struct flb_lib_out_cb cb_data; + struct test_ctx *ctx; + struct flb_http_client *c; + int ret; + int num; + size_t b_sent; + + char *buf = TEST_MSG_OTEL_LOGS; + + clear_output_num(); + + cb_data.cb = cb_check_result_json; + cb_data.data = TEST_CB_MSG_OTEL_LOGS; + + ctx = test_ctx_create(&cb_data); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("test_ctx_create failed"); + exit(EXIT_FAILURE); + } + + ret = flb_input_set(ctx->flb, ctx->i_ffd, + "successful_response_code", response_code, + NULL); + TEST_CHECK(ret == 0); + + ret = flb_output_set(ctx->flb, ctx->o_ffd, + "match", "*", + "format", "json", + NULL); + TEST_CHECK(ret == 0); + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + ctx->httpc = http_client_ctx_create(); + TEST_CHECK(ctx->httpc != NULL); + + c = flb_http_client(ctx->httpc->u_conn, FLB_HTTP_POST, V1_ENDPOINT_LOGS, buf, strlen(buf), + "127.0.0.1", PORT_OTEL, NULL, 0); + ret = flb_http_add_header(c, FLB_HTTP_HEADER_CONTENT_TYPE, strlen(FLB_HTTP_HEADER_CONTENT_TYPE), + JSON_CONTENT_TYPE, strlen(JSON_CONTENT_TYPE)); + TEST_CHECK(ret == 0); + if (!TEST_CHECK(c != NULL)) { + TEST_MSG("http_client failed"); + exit(EXIT_FAILURE); + } + + ret = flb_http_do(c, &b_sent); + if (!TEST_CHECK(ret == 0)) { + TEST_MSG("ret error. ret=%d\n", ret); + } + else if (!TEST_CHECK(b_sent > 0)){ + TEST_MSG("b_sent size error. b_sent = %lu\n", b_sent); + } + else if (!TEST_CHECK(c->resp.status == atoi(response_code))) { + TEST_MSG("http response code error. expect: %d, got: %d\n", atoi(response_code), c->resp.status); + } + + /* waiting to flush */ + flb_time_msleep(1500); + + num = get_output_num(); + if (!TEST_CHECK(num > 0)) { + TEST_MSG("no outputs"); + } + flb_http_client_destroy(c); + flb_upstream_conn_release(ctx->httpc->u_conn); + test_ctx_destroy(ctx); +} + +void flb_test_otel_successful_response_code_200() +{ + flb_test_otel_successful_response_code("200"); +} + +void flb_test_otel_successful_response_code_204() +{ + flb_test_otel_successful_response_code("204"); +} + +TEST_LIST = { + {"otel_logs", flb_test_otel_logs}, + {"successful_response_code_200", flb_test_otel_successful_response_code_200}, + {"successful_response_code_204", flb_test_otel_successful_response_code_204}, + {NULL, NULL} +}; + diff --git a/src/fluent-bit/tests/runtime/in_podman_metrics.c b/src/fluent-bit/tests/runtime/in_podman_metrics.c new file mode 100644 index 000000000..5ea0f852b --- /dev/null +++ b/src/fluent-bit/tests/runtime/in_podman_metrics.c @@ -0,0 +1,222 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2016 Treasure Data Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <fluent-bit.h> +#include "flb_tests_runtime.h" + +#define DPATH_PODMAN_REGULAR FLB_TESTS_DATA_PATH "/data/podman/regular" +#define DPATH_PODMAN_REVERSED FLB_TESTS_DATA_PATH "/data/podman/reversed" +#define DPATH_PODMAN_NO_CONFIG FLB_TESTS_DATA_PATH "/data/podman/no_config" +#define DPATH_PODMAN_GARBAGE_CONFIG FLB_TESTS_DATA_PATH "/data/podman/garbage_config" +#define DPATH_PODMAN_NO_SYSFS FLB_TESTS_DATA_PATH "/data/podman/no_sysfs" +#define DPATH_PODMAN_NO_PROC FLB_TESTS_DATA_PATH "/data/podman/no_proc" +#define DPATH_PODMAN_GARBAGE FLB_TESTS_DATA_PATH "/data/podman/garbage" +#define DPATH_PODMAN_CGROUP_V2 FLB_TESTS_DATA_PATH "/data/podman/cgroupv2" + + +int check_metric(flb_ctx_t *ctx, flb_sds_t *name) { + struct mk_list *tmp; + struct mk_list *head; + struct mk_list *inner_tmp; + struct mk_list *inner_head; + + struct flb_input_instance *i_ins; + struct cmt_counter *counter; + + int number_of_metrics=0; + + mk_list_foreach_safe(head, tmp, &ctx->config->inputs) { + i_ins = mk_list_entry(head, struct flb_input_instance, _head); + mk_list_foreach_safe(inner_head, inner_tmp, &i_ins->cmt->counters) { + counter = mk_list_entry(inner_head, struct cmt_counter, _head); + + if (strlen(name) != 0 && strcmp(name, counter->opts.name) == 0) + { + return 0; + } + number_of_metrics++; + } + } + return number_of_metrics; + +} + +void do_create(flb_ctx_t *ctx, char *system, ...) +{ + int64_t ret; + int in_ffd; + int out_ffd; + va_list va; + char *key; + char *value; + struct flb_lib_out_cb cb; + cb.data = NULL; + + in_ffd = flb_input(ctx, (char *) system, NULL); + + va_start(va, system); + while ((key = va_arg(va, char *))) { + value = va_arg(va, char *); + TEST_CHECK(value != NULL); + TEST_CHECK(flb_input_set(ctx, in_ffd, key, value, NULL) == 0); + } + va_end(va); + + TEST_CHECK(flb_service_set(ctx, "Flush", "0.5", + "Grace", "1", + NULL) == 0); +} + +void do_destroy(flb_ctx_t *ctx) { + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_ipm_regular() { + flb_ctx_t *ctx = flb_create(); + do_create(ctx, + "podman_metrics", + "path.config", DPATH_PODMAN_REGULAR "/config.json", + "scrape_on_start", "true", + "path.sysfs", DPATH_PODMAN_REGULAR, + "path.procfs", DPATH_PODMAN_REGULAR, + NULL); + TEST_CHECK(flb_start(ctx) == 0); + sleep(1); + TEST_CHECK(check_metric(ctx, "usage_bytes") == 0); + TEST_CHECK(check_metric(ctx, "receive_bytes_total") == 0); + do_destroy(ctx); +} + +void flb_test_ipm_reversed() { + flb_ctx_t *ctx = flb_create(); + do_create(ctx, + "podman_metrics", + "path.config", DPATH_PODMAN_REVERSED "/config.json", + "scrape_on_start", "true", + "path.sysfs", DPATH_PODMAN_REVERSED, + "path.procfs", DPATH_PODMAN_REVERSED, + NULL); + TEST_CHECK(flb_start(ctx) == 0); + sleep(1); + TEST_CHECK(check_metric(ctx, "usage_bytes") == 0); + TEST_CHECK(check_metric(ctx, "receive_bytes_total") == 0); + do_destroy(ctx); +} + +void flb_test_ipm_garbage_config() { + flb_ctx_t *ctx = flb_create(); + do_create(ctx, + "podman_metrics", + "path.config", DPATH_PODMAN_GARBAGE_CONFIG "/config.json", + "scrape_on_start", "true", + "path.sysfs", DPATH_PODMAN_GARBAGE_CONFIG, + "path.procfs", DPATH_PODMAN_GARBAGE_CONFIG, + NULL); + TEST_CHECK(flb_start(ctx) != 0); + do_destroy(ctx); +} + +void flb_test_ipm_no_config() { + flb_ctx_t *ctx = flb_create(); + do_create(ctx, + "podman_metrics", + "path.config", DPATH_PODMAN_NO_CONFIG "/config.json", + "scrape_on_start", "true", + "path.sysfs", DPATH_PODMAN_NO_CONFIG, + "path.procfs", DPATH_PODMAN_NO_CONFIG, + NULL); + TEST_CHECK(flb_start(ctx) != 0); + do_destroy(ctx); +} + +void flb_test_ipm_no_sysfs() { + flb_ctx_t *ctx = flb_create(); + do_create(ctx, + "podman_metrics", + "path.config", DPATH_PODMAN_NO_SYSFS "/config.json", + "scrape_on_start", "true", + "path.sysfs", DPATH_PODMAN_NO_SYSFS, + "path.procfs", DPATH_PODMAN_NO_SYSFS, + NULL); + TEST_CHECK(flb_start(ctx) == 0); + sleep(1); + TEST_CHECK(check_metric(ctx, "usage_bytes") != 0); + TEST_CHECK(check_metric(ctx, "receive_bytes_total") != 0); + do_destroy(ctx); +} + +void flb_test_ipm_no_proc() { + flb_ctx_t *ctx = flb_create(); + do_create(ctx, + "podman_metrics", + "path.config", DPATH_PODMAN_NO_PROC "/config.json", + "scrape_on_start", "true", + "path.sysfs", DPATH_PODMAN_NO_PROC, + "path.procfs", DPATH_PODMAN_NO_PROC, + NULL); + TEST_CHECK(flb_start(ctx) == 0); + sleep(1); + TEST_CHECK(check_metric(ctx, "usage_bytes") == 0); + TEST_CHECK(check_metric(ctx, "receive_bytes_total") != 0); + do_destroy(ctx); +} + +void flb_test_ipm_garbage() { + flb_ctx_t *ctx = flb_create(); + do_create(ctx, + "podman_metrics", + "path.config", DPATH_PODMAN_GARBAGE "/config.json", + "scrape_on_start", "true", + "path.sysfs", DPATH_PODMAN_GARBAGE, + "path.procfs", DPATH_PODMAN_GARBAGE, + NULL); + TEST_CHECK(flb_start(ctx) == 0); + sleep(1); + TEST_CHECK(check_metric(ctx, "usage_bytes") != 0); + TEST_CHECK(check_metric(ctx, "receive_bytes_total") != 0); + do_destroy(ctx); +} + +void flb_test_ipm_cgroupv2() { + flb_ctx_t *ctx = flb_create(); + do_create(ctx, + "podman_metrics", + "path.config", DPATH_PODMAN_CGROUP_V2 "/config.json", + "scrape_on_start", "true", + "path.sysfs", DPATH_PODMAN_CGROUP_V2, + "path.procfs", DPATH_PODMAN_CGROUP_V2, + NULL); + TEST_CHECK(flb_start(ctx) == 0); + sleep(1); + TEST_CHECK(check_metric(ctx, "usage_bytes") == 0); + TEST_CHECK(check_metric(ctx, "receive_bytes_total") == 0); + do_destroy(ctx); +} + +TEST_LIST = { + {"regular", flb_test_ipm_regular}, + {"reversed", flb_test_ipm_reversed}, + {"no_config", flb_test_ipm_no_config}, + {"garbage_config", flb_test_ipm_garbage_config}, + {"no_sysfs_data", flb_test_ipm_no_sysfs}, + {"no_proc_data", flb_test_ipm_no_proc}, + {"garbage_data", flb_test_ipm_garbage}, + {"cgroup_v2", flb_test_ipm_cgroupv2}, + {NULL, NULL}}; diff --git a/src/fluent-bit/tests/runtime/in_proc.c b/src/fluent-bit/tests/runtime/in_proc.c new file mode 120000 index 000000000..b194a8539 --- /dev/null +++ b/src/fluent-bit/tests/runtime/in_proc.c @@ -0,0 +1 @@ +in_simple_systems.c
\ No newline at end of file diff --git a/src/fluent-bit/tests/runtime/in_random.c b/src/fluent-bit/tests/runtime/in_random.c new file mode 120000 index 000000000..b194a8539 --- /dev/null +++ b/src/fluent-bit/tests/runtime/in_random.c @@ -0,0 +1 @@ +in_simple_systems.c
\ No newline at end of file diff --git a/src/fluent-bit/tests/runtime/in_simple_systems.c b/src/fluent-bit/tests/runtime/in_simple_systems.c new file mode 100644 index 000000000..7020c4f48 --- /dev/null +++ b/src/fluent-bit/tests/runtime/in_simple_systems.c @@ -0,0 +1,576 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2016 Treasure Data Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <fluent-bit.h> +#include <fluent-bit/flb_time.h> +#include <pthread.h> +#include <stdlib.h> +#include <unistd.h> +#include "flb_tests_runtime.h" + + +int64_t result_time; +static inline int64_t set_result(int64_t v) +{ + int64_t old = __sync_lock_test_and_set(&result_time, v); + return old; +} + +static inline int64_t get_result(void) +{ + int64_t old = __sync_fetch_and_add(&result_time, 0); + + return old; +} + +static inline int64_t time_in_ms() +{ + int ms; + struct timespec s; + TEST_CHECK(clock_gettime(CLOCK_MONOTONIC, &s) == 0); + ms = s.tv_nsec / 1.0e6; + if (ms >= 1000) { + ms = 0; + } + return 1000 * s.tv_sec + ms; +} + +int callback_test(void* data, size_t size, void* cb_data) +{ + if (size > 0) { + flb_info("[test] flush triggered"); + flb_lib_free(data); + set_result(time_in_ms()); /* success */ + } + return 0; +} + +struct callback_record { + void *data; + size_t size; +}; + +struct callback_records { + int num_records; + struct callback_record *records; +}; + +int callback_add_record(void* data, size_t size, void* cb_data) +{ + struct callback_records *ctx = (struct callback_records *)cb_data; + + if (size > 0) { + flb_info("[test] flush record"); + if (ctx->records == NULL) { + ctx->records = (struct callback_record *) + flb_calloc(1, sizeof(struct callback_record)); + } else { + ctx->records = (struct callback_record *) + flb_realloc(ctx->records, + (ctx->num_records+1)*sizeof(struct callback_record)); + } + if (ctx->records == NULL) { + return -1; + } + ctx->records[ctx->num_records].size = size; + ctx->records[ctx->num_records].data = data; + ctx->num_records++; + } + return 0; +} + +void do_test(char *system, ...) +{ + int64_t ret; + flb_ctx_t *ctx = NULL; + int in_ffd; + int out_ffd; + va_list va; + char *key; + char *value; + struct flb_lib_out_cb cb; + + cb.cb = callback_test; + cb.data = NULL; + + /* initialize */ + set_result(0); + + ctx = flb_create(); + + in_ffd = flb_input(ctx, (char *) system, NULL); + TEST_CHECK(in_ffd >= 0); + TEST_CHECK(flb_input_set(ctx, in_ffd, "tag", "test", NULL) == 0); + + va_start(va, system); + while ((key = va_arg(va, char *))) { + value = va_arg(va, char *); + TEST_CHECK(value != NULL); + TEST_CHECK(flb_input_set(ctx, in_ffd, key, value, NULL) == 0); + } + va_end(va); + + out_ffd = flb_output(ctx, (char *) "lib", &cb); + TEST_CHECK(out_ffd >= 0); + TEST_CHECK(flb_output_set(ctx, out_ffd, "match", "test", NULL) == 0); + + TEST_CHECK(flb_service_set(ctx, "Flush", "0.5", + "Grace", "1", + NULL) == 0); + + /* The following test tries to check if an input plugin generates + * data in a timely manner. + * + * 0 1 2 3 4 (sec) + * |--F--F--F--C--F--F--F--C--| + * + * F ... Flush (0.5 sec interval) + * C ... Condition checks + * + * Since CI servers can be sometimes very slow, we wait slightly a + * little more before checking the condition. + */ + + /* Start test */ + TEST_CHECK(flb_start(ctx) == 0); + + /* 2 sec passed. It must have flushed */ + sleep(2); + flb_info("[test] check status 1"); + ret = get_result(); + TEST_CHECK(ret > 0); + + /* 4 sec passed. It must have flushed */ + sleep(2); + flb_info("[test] check status 2"); + ret = get_result(); + TEST_CHECK(ret > 0); + + flb_stop(ctx); + flb_destroy(ctx); +} + +void do_test_records(char *system, void (*records_cb)(struct callback_records *), ...) +{ + flb_ctx_t *ctx = NULL; + int in_ffd; + int out_ffd; + va_list va; + char *key; + char *value; + int i; + struct flb_lib_out_cb cb; + struct callback_records *records; + + records = flb_calloc(1, sizeof(struct callback_records)); + records->num_records = 0; + records->records = NULL; + cb.cb = callback_add_record; + cb.data = (void *)records; + + /* initialize */ + set_result(0); + + ctx = flb_create(); + + in_ffd = flb_input(ctx, (char *) system, NULL); + TEST_CHECK(in_ffd >= 0); + TEST_CHECK(flb_input_set(ctx, in_ffd, "tag", "test", NULL) == 0); + + va_start(va, records_cb); + while ((key = va_arg(va, char *))) { + value = va_arg(va, char *); + TEST_CHECK(value != NULL); + TEST_CHECK(flb_input_set(ctx, in_ffd, key, value, NULL) == 0); + } + va_end(va); + + out_ffd = flb_output(ctx, (char *) "lib", &cb); + TEST_CHECK(out_ffd >= 0); + TEST_CHECK(flb_output_set(ctx, out_ffd, "match", "test", NULL) == 0); + + TEST_CHECK(flb_service_set(ctx, "Flush", "1", + "Grace", "1", + NULL) == 0); + + /* Start test */ + TEST_CHECK(flb_start(ctx) == 0); + + /* 4 sec passed. It must have flushed */ + sleep(5); + + records_cb(records); + + flb_stop(ctx); + + for (i = 0; i < records->num_records; i++) { + flb_lib_free(records->records[i].data); + } + flb_free(records->records); + flb_free(records); + + flb_destroy(ctx); +} + +void do_test_records_single(char *system, void (*records_cb)(struct callback_records *), ...) +{ + flb_ctx_t *ctx = NULL; + int in_ffd; + int out_ffd; + int exit_ffd; + va_list va; + char *key; + char *value; + int i; + struct flb_lib_out_cb cb; + struct callback_records *records; + + records = flb_calloc(1, sizeof(struct callback_records)); + records->num_records = 0; + records->records = NULL; + cb.cb = callback_add_record; + cb.data = (void *)records; + + /* initialize */ + set_result(0); + + ctx = flb_create(); + + in_ffd = flb_input(ctx, (char *) system, NULL); + TEST_CHECK(in_ffd >= 0); + TEST_CHECK(flb_input_set(ctx, in_ffd, "tag", "test", NULL) == 0); + + va_start(va, records_cb); + while ((key = va_arg(va, char *))) { + value = va_arg(va, char *); + TEST_CHECK(value != NULL); + TEST_CHECK(flb_input_set(ctx, in_ffd, key, value, NULL) == 0); + } + va_end(va); + + out_ffd = flb_output(ctx, (char *) "lib", &cb); + TEST_CHECK(out_ffd >= 0); + TEST_CHECK(flb_output_set(ctx, out_ffd, "match", "test", NULL) == 0); + + exit_ffd = flb_output(ctx, (char *)"exit", &cb); + TEST_CHECK(flb_output_set(ctx, exit_ffd, "match", "test", NULL) == 0); + + TEST_CHECK(flb_service_set(ctx, "Flush", "1", + "Grace", "1", + NULL) == 0); + + /* Start test */ + TEST_CHECK(flb_start(ctx) == 0); + + /* 4 sec passed. It must have flushed */ + sleep(5); + + records_cb(records); + + flb_stop(ctx); + + for (i = 0; i < records->num_records; i++) { + flb_lib_free(records->records[i].data); + } + flb_free(records->records); + flb_free(records); + + flb_destroy(ctx); +} + +void flb_test_in_disk_flush() +{ + do_test("disk", + "interval_sec", "0", + "interval_nsec", "500000000", + NULL); +} +void flb_test_in_proc_flush() +{ + do_test("proc", + "interval_sec", "0", + "interval_nsec", "500000000", + "proc_name", "flb_test_in_proc", + "alert", "true", + "mem", "on", + "fd", "on", + NULL); +} +void flb_test_in_head_flush() +{ + do_test("head", + "interval_sec", "0", + "interval_nsec", "500000000", + "File", "/dev/urandom", + NULL); +} +void flb_test_in_cpu_flush() +{ + do_test("cpu", NULL); +} +void flb_test_in_random_flush() +{ + do_test("random", NULL); +} + +void flb_test_dummy_records_1234(struct callback_records *records) +{ + int i; + msgpack_unpacked result; + msgpack_object *obj; + size_t off = 0; + struct flb_time ftm; + + TEST_CHECK(records->num_records > 0); + for (i = 0; i < records->num_records; i++) { + msgpack_unpacked_init(&result); + + while (msgpack_unpack_next(&result, records->records[i].data, + records->records[i].size, &off) == MSGPACK_UNPACK_SUCCESS) { + flb_time_pop_from_msgpack(&ftm, &result, &obj); + + TEST_CHECK(ftm.tm.tv_sec == 1234); + TEST_CHECK(ftm.tm.tv_nsec == 1234); + } + msgpack_unpacked_destroy(&result); + } +} + +void flb_test_dummy_records_1999(struct callback_records *records) +{ + int i; + msgpack_unpacked result; + msgpack_object *obj; + size_t off = 0; + struct flb_time ftm; + + TEST_CHECK(records->num_records > 0); + for (i = 0; i < records->num_records; i++) { + msgpack_unpacked_init(&result); + + while (msgpack_unpack_next(&result, records->records[i].data, + records->records[i].size, &off) == MSGPACK_UNPACK_SUCCESS) { + flb_time_pop_from_msgpack(&ftm, &result, &obj); + TEST_CHECK(ftm.tm.tv_sec == 1999); + TEST_CHECK(ftm.tm.tv_nsec == 1999); + } + msgpack_unpacked_destroy(&result); + } +} + +void flb_test_dummy_records_today(struct callback_records *records) +{ + int i; + msgpack_unpacked result; + msgpack_object *obj; + size_t off = 0; + struct flb_time ftm; + struct flb_time now; + + flb_time_get(&now); + /* set 5 minutes in the past since this is invoked after the test began */ + now.tm.tv_sec -= (5 * 60); + + TEST_CHECK(records->num_records > 0); + for (i = 0; i < records->num_records; i++) { + msgpack_unpacked_init(&result); + + while (msgpack_unpack_next(&result, records->records[i].data, + records->records[i].size, &off) == MSGPACK_UNPACK_SUCCESS) { + flb_time_pop_from_msgpack(&ftm, &result, &obj); + TEST_CHECK(ftm.tm.tv_sec >= now.tm.tv_sec); + } + msgpack_unpacked_destroy(&result); + } +} + +void flb_test_dummy_records_message(struct callback_records *records) +{ + int i; + msgpack_unpacked result; + msgpack_object *obj; + size_t off = 0; + struct flb_time ftm; + + TEST_CHECK(records->num_records > 0); + for (i = 0; i < records->num_records; i++) { + msgpack_unpacked_init(&result); + + while (msgpack_unpack_next(&result, records->records[i].data, + records->records[i].size, &off) == MSGPACK_UNPACK_SUCCESS) { + flb_time_pop_from_msgpack(&ftm, &result, &obj); + TEST_CHECK(obj->type == MSGPACK_OBJECT_MAP); + TEST_CHECK(strncmp("new_key", + obj->via.map.ptr[0].key.via.str.ptr, + obj->via.map.ptr[0].key.via.str.size) == 0); + TEST_CHECK(strncmp("new_value", + obj->via.map.ptr[0].val.via.str.ptr, + obj->via.map.ptr[0].val.via.str.size) == 0); + } + msgpack_unpacked_destroy(&result); + } +} + +void flb_test_dummy_records_message_default(struct callback_records *records) +{ + int i; + msgpack_unpacked result; + msgpack_object *obj; + size_t off = 0; + struct flb_time ftm; + + TEST_CHECK(records->num_records > 0); + for (i = 0; i < records->num_records; i++) { + msgpack_unpacked_init(&result); + + while (msgpack_unpack_next(&result, records->records[i].data, + records->records[i].size, &off) == MSGPACK_UNPACK_SUCCESS) { + flb_time_pop_from_msgpack(&ftm, &result, &obj); + TEST_CHECK(obj->type == MSGPACK_OBJECT_MAP); + TEST_CHECK(strncmp("message", + obj->via.map.ptr[0].key.via.str.ptr, + obj->via.map.ptr[0].key.via.str.size) == 0); + TEST_CHECK(strncmp("dummy", + obj->via.map.ptr[0].val.via.str.ptr, + obj->via.map.ptr[0].val.via.str.size) == 0); + } + msgpack_unpacked_destroy(&result); + } +} + +// We check for a minimum of messages because of the non-deterministic +// nature of flushes as well as chunking. +void flb_test_dummy_records_message_copies_1(struct callback_records *records) +{ + TEST_CHECK(records->num_records >= 1); +} + +void flb_test_dummy_records_message_copies_5(struct callback_records *records) +{ + TEST_CHECK(records->num_records >= 5); +} + +void flb_test_dummy_records_message_copies_100(struct callback_records *records) +{ + TEST_CHECK(records->num_records >= 100); +} + +void flb_test_in_dummy_flush() +{ + do_test("dummy", NULL); + do_test_records("dummy", flb_test_dummy_records_message_default, NULL); + do_test_records("dummy", flb_test_dummy_records_today, NULL); + do_test_records("dummy", flb_test_dummy_records_message, + "dummy", "{\"new_key\": \"new_value\"}", + NULL); + do_test_records("dummy", flb_test_dummy_records_message_default, + "dummy", "{\"bad_json}", + NULL); + do_test_records("dummy", flb_test_dummy_records_1234, + "start_time_sec", "1234", + "start_time_nsec", "1234", + "fixed_timestamp", "on", + NULL); + do_test_records("dummy", flb_test_dummy_records_1999, + "start_time_sec", "1999", + "start_time_nsec", "1999", + "fixed_timestamp", "on", + NULL); + do_test_records_single("dummy", flb_test_dummy_records_message_copies_1, + "copies", "1", + NULL); + do_test_records_single("dummy", flb_test_dummy_records_message_copies_5, + "copies", "5", + NULL); + do_test_records_single("dummy", flb_test_dummy_records_message_copies_100, + "copies", "100", + NULL); +} + +void flb_test_in_dummy_thread_flush() +{ + do_test("dummy_thread", NULL); +} + +void flb_test_in_mem_flush() +{ + do_test("mem", NULL); +} + +#ifdef in_proc +void flb_test_in_proc_absent_process(void) +{ + int ret; + flb_ctx_t *ctx = NULL; + int in_ffd; + int out_ffd; + + struct flb_lib_out_cb cb; + cb.cb = callback_test; + cb.data = NULL; + + ctx = flb_create(); + + in_ffd = flb_input(ctx, (char *) "proc", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx, in_ffd, "tag", "test", + "interval_sec", "1", "proc_name", "-", + "alert", "true", "mem", "on", "fd", "on", NULL); + + out_ffd = flb_output(ctx, (char *) "lib", &cb); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, "match", "test", NULL); + + flb_service_set(ctx, "Flush", "2", "Grace", "1", NULL); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); // error occurs but return value is true + + flb_stop(ctx); + flb_destroy(ctx); +} +#endif + +/* Test list */ +TEST_LIST = { +#ifdef in_disk + {"disk_flush", flb_test_in_disk_flush}, +#endif +#ifdef in_proc + {"proc_flush", flb_test_in_proc_flush}, + {"proc_absent_process", flb_test_in_proc_absent_process}, +#endif +#ifdef in_head + {"head_flush", flb_test_in_head_flush}, +#endif +#ifdef in_cpu + {"cpu_flush", flb_test_in_cpu_flush}, +#endif +#ifdef in_random + {"random_flush", flb_test_in_random_flush}, +#endif +#ifdef in_dummy + {"dummy_flush", flb_test_in_dummy_flush}, +#endif +#ifdef in_mem + {"mem_flush", flb_test_in_mem_flush}, +#endif + {NULL, NULL} +}; + diff --git a/src/fluent-bit/tests/runtime/in_splunk.c b/src/fluent-bit/tests/runtime/in_splunk.c new file mode 100644 index 000000000..bba4977c4 --- /dev/null +++ b/src/fluent-bit/tests/runtime/in_splunk.c @@ -0,0 +1,824 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2019-2023 The Fluent Bit Authors + * Copyright (C) 2015-2018 Treasure Data Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <stdlib.h> +#include <fluent-bit.h> +#include <fluent-bit/flb_compat.h> +#include <fluent-bit/flb_time.h> +#include <fluent-bit/flb_pack.h> +#include <fluent-bit/flb_http_client.h> +#include <fluent-bit/flb_gzip.h> +#include <monkey/mk_core.h> +#include "flb_tests_runtime.h" + +#define JSON_CONTENT_TYPE "application/json" + +struct in_splunk_client_ctx { + struct flb_upstream *u; + struct flb_connection *u_conn; + struct flb_config *config; + struct mk_event_loop *evl; +}; + +struct test_ctx { + flb_ctx_t *flb; /* Fluent Bit library context */ + int i_ffd; /* Input fd */ + int f_ffd; /* Filter fd (unused) */ + int o_ffd; /* Output fd */ + struct in_splunk_client_ctx *httpc; +}; + + +pthread_mutex_t result_mutex = PTHREAD_MUTEX_INITIALIZER; +int num_output = 0; +static int get_output_num() +{ + int ret; + pthread_mutex_lock(&result_mutex); + ret = num_output; + pthread_mutex_unlock(&result_mutex); + + return ret; +} + +static void set_output_num(int num) +{ + pthread_mutex_lock(&result_mutex); + num_output = num; + pthread_mutex_unlock(&result_mutex); +} + +static void clear_output_num() +{ + set_output_num(0); +} + +/* Callback to check expected results */ +static int cb_check_result_json(void *record, size_t size, void *data) +{ + char *p; + char *expected; + char *result; + int num = get_output_num(); + + set_output_num(num+1); + + expected = (char *) data; + result = (char *) record; + + p = strstr(result, expected); + TEST_CHECK(p != NULL); + + if (p==NULL) { + flb_error("Expected to find: '%s' in result '%s'", + expected, result); + } + /* + * If you want to debug your test + * + * printf("Expect: '%s' in result '%s'", expected, result); + */ + flb_free(record); + return 0; +} + +struct in_splunk_client_ctx* splunk_client_ctx_create(int port) +{ + struct in_splunk_client_ctx *ret_ctx = NULL; + struct mk_event_loop *evl = NULL; + + ret_ctx = flb_calloc(1, sizeof(struct in_splunk_client_ctx)); + if (!TEST_CHECK(ret_ctx != NULL)) { + flb_errno(); + TEST_MSG("flb_calloc(splunk_client_ctx) failed"); + return NULL; + } + + evl = mk_event_loop_create(16); + if (!TEST_CHECK(evl != NULL)) { + TEST_MSG("mk_event_loop failed"); + flb_free(ret_ctx); + return NULL; + } + ret_ctx->evl = evl; + flb_engine_evl_init(); + flb_engine_evl_set(evl); + + ret_ctx->config = flb_config_init(); + if(!TEST_CHECK(ret_ctx->config != NULL)) { + TEST_MSG("flb_config_init failed"); + mk_event_loop_destroy(evl); + flb_free(ret_ctx); + return NULL; + } + + ret_ctx->u = flb_upstream_create(ret_ctx->config, "127.0.0.1", port, 0, NULL); + if (!TEST_CHECK(ret_ctx->u != NULL)) { + TEST_MSG("flb_upstream_create failed"); + flb_config_exit(ret_ctx->config); + mk_event_loop_destroy(evl); + flb_free(ret_ctx); + return NULL; + } + + ret_ctx->u_conn = flb_upstream_conn_get(ret_ctx->u); + TEST_CHECK(ret_ctx->u_conn != NULL); + + ret_ctx->u_conn->upstream = ret_ctx->u; + + return ret_ctx; +} + +static struct test_ctx *test_ctx_create(struct flb_lib_out_cb *data) +{ + int i_ffd; + int o_ffd; + struct test_ctx *ctx = NULL; + + ctx = flb_calloc(1, sizeof(struct test_ctx)); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("flb_calloc failed"); + flb_errno(); + return NULL; + } + + /* Service config */ + ctx->flb = flb_create(); + flb_service_set(ctx->flb, + "Flush", "0.200000000", + "Grace", "1", + "Log_Level", "error", + NULL); + + /* Input */ + i_ffd = flb_input(ctx->flb, (char *) "splunk", NULL); + TEST_CHECK(i_ffd >= 0); + ctx->i_ffd = i_ffd; + + /* Output */ + o_ffd = flb_output(ctx->flb, (char *) "lib", (void *) data); + ctx->o_ffd = o_ffd; + + return ctx; +} + +int splunk_client_ctx_destroy(struct in_splunk_client_ctx* ctx) +{ + if (!TEST_CHECK(ctx != NULL)) { + return -1; + } + if (ctx->u) { + flb_upstream_destroy(ctx->u); + } + if (ctx->config) { + flb_config_exit(ctx->config); + } + if (ctx->evl) { + mk_event_loop_destroy(ctx->evl); + } + + flb_free(ctx); + return 0; +} + +static void test_ctx_destroy(struct test_ctx *ctx) +{ + TEST_CHECK(ctx != NULL); + if (ctx->httpc) { + splunk_client_ctx_destroy(ctx->httpc); + } + + sleep(1); + flb_stop(ctx->flb); + flb_destroy(ctx->flb); + flb_free(ctx); +} + +void flb_test_splunk_health() +{ + struct flb_lib_out_cb cb_data; + struct test_ctx *ctx; + struct flb_http_client *c; + int ret; + size_t b_sent; + char *expected = "{\"text\":\"Success\",\"code\":200}"; + char *buf = NULL; + int port = 8808; + char sport[16]; + + snprintf(sport, 16, "%d", port); + + ctx = test_ctx_create(&cb_data); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("test_ctx_create failed"); + exit(EXIT_FAILURE); + } + + ret = flb_input_set(ctx->flb, ctx->i_ffd, + "port", sport, + NULL); + TEST_CHECK(ret == 0); + + ret = flb_output_set(ctx->flb, ctx->o_ffd, + "match", "*", + "format", "json", + NULL); + TEST_CHECK(ret == 0); + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + ctx->httpc = splunk_client_ctx_create(port); + TEST_CHECK(ctx->httpc != NULL); + + c = flb_http_client(ctx->httpc->u_conn, FLB_HTTP_GET, "/services/collector/health", NULL, 0, + "127.0.0.1", port, NULL, 0); + if (!TEST_CHECK(c != NULL)) { + TEST_MSG("splunk_client failed"); + exit(EXIT_FAILURE); + } + + ret = flb_http_do(c, &b_sent); + if (!TEST_CHECK(ret == 0)) { + TEST_MSG("ret error. ret=%d\n", ret); + } + else if (!TEST_CHECK(b_sent > 0)){ + TEST_MSG("b_sent size error. b_sent = %lu\n", b_sent); + } + else if (!TEST_CHECK(c->resp.status == 200)) { + TEST_MSG("http response code error. expect: 200, got: %d\n", c->resp.status); + } + + /* waiting to flush */ + flb_time_msleep(1500); + + buf = strstr(c->resp.payload, expected); + if (!TEST_CHECK(buf != NULL)) { + TEST_MSG("http request for version info failed"); + } + flb_http_client_destroy(c); + flb_upstream_conn_release(ctx->httpc->u_conn); + test_ctx_destroy(ctx); +} + +void flb_test_splunk(int port, char *endpoint) +{ + struct flb_lib_out_cb cb_data; + struct test_ctx *ctx; + struct flb_http_client *c; + int ret; + int num; + size_t b_sent; + char *buf = "{\"event\": \"Pony 1 has left the barn\"}{\"event\": \"Pony 2 has left the barn\"}{\"event\": \"Pony 3 has left the barn\", \"nested\": {\"key1\": \"value1\"}}"; + char *expected = "\"event\":"; + char sport[16]; + flb_sds_t target; + + target = flb_sds_create_size(64); + flb_sds_cat(target, endpoint, strlen(endpoint)); + + snprintf(sport, 16, "%d", port); + + clear_output_num(); + + cb_data.cb = cb_check_result_json; + cb_data.data = expected; + + ctx = test_ctx_create(&cb_data); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("test_ctx_create failed"); + exit(EXIT_FAILURE); + } + + ret = flb_input_set(ctx->flb, ctx->i_ffd, + "port", sport, + NULL); + TEST_CHECK(ret == 0); + + ret = flb_output_set(ctx->flb, ctx->o_ffd, + "match", "*", + "format", "json", + NULL); + TEST_CHECK(ret == 0); + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + ctx->httpc = splunk_client_ctx_create(port); + TEST_CHECK(ctx->httpc != NULL); + + c = flb_http_client(ctx->httpc->u_conn, FLB_HTTP_POST, target, buf, strlen(buf), + "127.0.0.1", port, NULL, 0); + ret = flb_http_add_header(c, FLB_HTTP_HEADER_CONTENT_TYPE, strlen(FLB_HTTP_HEADER_CONTENT_TYPE), + JSON_CONTENT_TYPE, strlen(JSON_CONTENT_TYPE)); + TEST_CHECK(ret == 0); + if (!TEST_CHECK(c != NULL)) { + TEST_MSG("splunk_client failed"); + exit(EXIT_FAILURE); + } + + ret = flb_http_do(c, &b_sent); + if (!TEST_CHECK(ret == 0)) { + TEST_MSG("ret error. ret=%d\n", ret); + } + else if (!TEST_CHECK(b_sent > 0)){ + TEST_MSG("b_sent size error. b_sent = %lu\n", b_sent); + } + else if (!TEST_CHECK(c->resp.status == 200)) { + TEST_MSG("http response code error. expect: 200, got: %d\n", c->resp.status); + } + + /* waiting to flush */ + flb_time_msleep(1500); + + num = get_output_num(); + if (!TEST_CHECK(num > 0)) { + TEST_MSG("no outputs"); + } + flb_sds_destroy(target); + flb_http_client_destroy(c); + flb_upstream_conn_release(ctx->httpc->u_conn); + test_ctx_destroy(ctx); +} + +void flb_test_splunk_collector() +{ + flb_test_splunk(8809, "/services/collector"); +} + +void flb_test_splunk_collector_event() +{ + flb_test_splunk(8810, "/services/collector/event"); +} + +void flb_test_splunk_raw(int port) +{ + struct flb_lib_out_cb cb_data; + struct test_ctx *ctx; + struct flb_http_client *c; + int ret; + int num; + size_t b_sent; + char *buf = "1, 2, 3... Hello, world!"; + char *expected = "\"log\":"; + char sport[16]; + + snprintf(sport, 16, "%d", port); + + clear_output_num(); + + cb_data.cb = cb_check_result_json; + cb_data.data = expected; + + ctx = test_ctx_create(&cb_data); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("test_ctx_create failed"); + exit(EXIT_FAILURE); + } + + ret = flb_input_set(ctx->flb, ctx->i_ffd, + "port", sport, + NULL); + TEST_CHECK(ret == 0); + + ret = flb_output_set(ctx->flb, ctx->o_ffd, + "match", "*", + "format", "json", + NULL); + TEST_CHECK(ret == 0); + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + ctx->httpc = splunk_client_ctx_create(port); + TEST_CHECK(ctx->httpc != NULL); + + c = flb_http_client(ctx->httpc->u_conn, FLB_HTTP_POST, "/services/collector/raw", buf, strlen(buf), + "127.0.0.1", port, NULL, 0); + ret = flb_http_add_header(c, FLB_HTTP_HEADER_CONTENT_TYPE, strlen(FLB_HTTP_HEADER_CONTENT_TYPE), + JSON_CONTENT_TYPE, strlen(JSON_CONTENT_TYPE)); + TEST_CHECK(ret == 0); + if (!TEST_CHECK(c != NULL)) { + TEST_MSG("splunk_client failed"); + exit(EXIT_FAILURE); + } + + ret = flb_http_do(c, &b_sent); + if (!TEST_CHECK(ret == 0)) { + TEST_MSG("ret error. ret=%d\n", ret); + } + else if (!TEST_CHECK(b_sent > 0)){ + TEST_MSG("b_sent size error. b_sent = %lu\n", b_sent); + } + else if (!TEST_CHECK(c->resp.status == 200)) { + TEST_MSG("http response code error. expect: 200, got: %d\n", c->resp.status); + } + + /* waiting to flush */ + flb_time_msleep(1500); + + num = get_output_num(); + if (!TEST_CHECK(num > 0)) { + TEST_MSG("no outputs"); + } + flb_http_client_destroy(c); + flb_upstream_conn_release(ctx->httpc->u_conn); + test_ctx_destroy(ctx); +} + +void flb_test_splunk_collector_raw() +{ + flb_test_splunk_raw(8811); +} + +void flb_test_splunk_raw_multilines(int port) +{ + struct flb_lib_out_cb cb_data; + struct test_ctx *ctx; + struct flb_http_client *c; + int ret; + int num; + size_t b_sent; + char *buf = "127.0.0.1 - admin [28/Sep/2016:09:05:26.875 -0700] \"GET /servicesNS/admin/launcher/data/ui/views?count=-1 HTTP/1.0\" 200 126721 - - - 6ms" \ + "127.0.0.1 - admin [28/Sep/2016:09:05:26.917 -0700] \"GET /servicesNS/admin/launcher/data/ui/nav/default HTTP/1.0\" 200 4367 - - - 6ms" \ + "127.0.0.1 - admin [28/Sep/2016:09:05:26.941 -0700] \"GET /services/apps/local?search=disabled%3Dfalse&count=-1 HTTP/1.0\" 200 31930 - - - 4ms" \ + "127.0.0.1 - admin [28/Sep/2016:09:05:26.954 -0700] \"GET /services/apps/local?search=disabled%3Dfalse&count=-1 HTTP/1.0\" 200 31930 - - - 3ms" \ + "127.0.0.1 - admin [28/Sep/2016:09:05:26.968 -0700] \"GET /servicesNS/admin/launcher/data/ui/views?digest=1&count=-1 HTTP/1.0\" 200 58672 - - - 5ms"; + char *expected = "\"log\":"; + char sport[16]; + + snprintf(sport, 16, "%d", port); + + clear_output_num(); + + cb_data.cb = cb_check_result_json; + cb_data.data = expected; + + ctx = test_ctx_create(&cb_data); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("test_ctx_create failed"); + exit(EXIT_FAILURE); + } + + ret = flb_input_set(ctx->flb, ctx->i_ffd, + "port", sport, + NULL); + TEST_CHECK(ret == 0); + + ret = flb_output_set(ctx->flb, ctx->o_ffd, + "match", "*", + "format", "json", + NULL); + TEST_CHECK(ret == 0); + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + ctx->httpc = splunk_client_ctx_create(port); + TEST_CHECK(ctx->httpc != NULL); + + c = flb_http_client(ctx->httpc->u_conn, FLB_HTTP_POST, "/services/collector/raw", buf, strlen(buf), + "127.0.0.1", port, NULL, 0); + ret = flb_http_add_header(c, FLB_HTTP_HEADER_CONTENT_TYPE, strlen(FLB_HTTP_HEADER_CONTENT_TYPE), + JSON_CONTENT_TYPE, strlen(JSON_CONTENT_TYPE)); + TEST_CHECK(ret == 0); + if (!TEST_CHECK(c != NULL)) { + TEST_MSG("splunk_client failed"); + exit(EXIT_FAILURE); + } + + ret = flb_http_do(c, &b_sent); + if (!TEST_CHECK(ret == 0)) { + TEST_MSG("ret error. ret=%d\n", ret); + } + else if (!TEST_CHECK(b_sent > 0)){ + TEST_MSG("b_sent size error. b_sent = %lu\n", b_sent); + } + else if (!TEST_CHECK(c->resp.status == 200)) { + TEST_MSG("http response code error. expect: 200, got: %d\n", c->resp.status); + } + + /* waiting to flush */ + flb_time_msleep(1500); + + num = get_output_num(); + if (!TEST_CHECK(num > 0)) { + TEST_MSG("no outputs"); + } + flb_http_client_destroy(c); + flb_upstream_conn_release(ctx->httpc->u_conn); + test_ctx_destroy(ctx); +} + +void flb_test_splunk_collector_raw_multilines() +{ + flb_test_splunk_raw_multilines(8812); +} + +void flb_test_splunk_tag_key() +{ + struct flb_lib_out_cb cb_data; + struct test_ctx *ctx; + struct flb_http_client *c; + int ret; + int num; + size_t b_sent; + int port = 8812; + char sport[16]; + + char *buf = "{\"event\": \"Pony 1 has left the barn\",\"tag\":\"new_tag\"}"; + + snprintf(sport, 16, "%d", port); + + clear_output_num(); + + cb_data.cb = cb_check_result_json; + cb_data.data = "\"event\":\"Pony 1 has left the barn\",\"tag\":\"new_tag\""; + + ctx = test_ctx_create(&cb_data); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("test_ctx_create failed"); + exit(EXIT_FAILURE); + } + + ret = flb_input_set(ctx->flb, ctx->i_ffd, + "port", sport, + "tag_key", "tag", + NULL); + TEST_CHECK(ret == 0); + + ret = flb_output_set(ctx->flb, ctx->o_ffd, + "match", "new_tag", + "format", "json", + NULL); + TEST_CHECK(ret == 0); + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + ctx->httpc = splunk_client_ctx_create(port); + TEST_CHECK(ctx->httpc != NULL); + + c = flb_http_client(ctx->httpc->u_conn, FLB_HTTP_POST, "/services/collector", buf, strlen(buf), + "127.0.0.1", port, NULL, 0); + ret = flb_http_add_header(c, FLB_HTTP_HEADER_CONTENT_TYPE, strlen(FLB_HTTP_HEADER_CONTENT_TYPE), + JSON_CONTENT_TYPE, strlen(JSON_CONTENT_TYPE)); + TEST_CHECK(ret == 0); + if (!TEST_CHECK(c != NULL)) { + TEST_MSG("splunk_client failed"); + exit(EXIT_FAILURE); + } + + ret = flb_http_do(c, &b_sent); + if (!TEST_CHECK(ret == 0)) { + TEST_MSG("ret error. ret=%d\n", ret); + } + else if (!TEST_CHECK(b_sent > 0)){ + TEST_MSG("b_sent size error. b_sent = %lu\n", b_sent); + } + else if (!TEST_CHECK(c->resp.status == 200)) { + TEST_MSG("http response code error. expect: 200, got: %d\n", c->resp.status); + } + + /* waiting to flush */ + flb_time_msleep(1500); + + num = get_output_num(); + if (!TEST_CHECK(num > 0)) { + TEST_MSG("no outputs"); + } + flb_http_client_destroy(c); + flb_upstream_conn_release(ctx->httpc->u_conn); + test_ctx_destroy(ctx); +} + +void flb_test_splunk_gzip(int port, char *endpoint) +{ + struct flb_lib_out_cb cb_data; + struct test_ctx *ctx; + struct flb_http_client *c; + int ret; + int num; + size_t b_sent; + char *buf = "{\"event\": \"Pony 1 has left the barn\"}{\"event\": \"Pony 2 has left the barn\"}{\"event\": \"Pony 3 has left the barn\", \"nested\": {\"key1\": \"value1\"}}"; + char *expected = "\"event\":"; + char sport[16]; + flb_sds_t target; + void *final_data; + size_t final_bytes; + + target = flb_sds_create_size(64); + flb_sds_cat(target, endpoint, strlen(endpoint)); + + snprintf(sport, 16, "%d", port); + + clear_output_num(); + + cb_data.cb = cb_check_result_json; + cb_data.data = expected; + + ctx = test_ctx_create(&cb_data); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("test_ctx_create failed"); + exit(EXIT_FAILURE); + } + + ret = flb_input_set(ctx->flb, ctx->i_ffd, + "port", sport, + NULL); + TEST_CHECK(ret == 0); + + ret = flb_output_set(ctx->flb, ctx->o_ffd, + "match", "*", + "format", "json", + NULL); + TEST_CHECK(ret == 0); + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + ctx->httpc = splunk_client_ctx_create(port); + TEST_CHECK(ctx->httpc != NULL); + + ret = flb_gzip_compress((void *) buf, strlen(buf), &final_data, &final_bytes); + TEST_CHECK(ret != -1); + + c = flb_http_client(ctx->httpc->u_conn, FLB_HTTP_POST, target, final_data, final_bytes, + "127.0.0.1", port, NULL, 0); + ret = flb_http_add_header(c, FLB_HTTP_HEADER_CONTENT_TYPE, strlen(FLB_HTTP_HEADER_CONTENT_TYPE), + JSON_CONTENT_TYPE, strlen(JSON_CONTENT_TYPE)); + TEST_CHECK(ret == 0); + /* Add Content-Encoding: gzip */ + ret = flb_http_add_header(c, "Content-Encoding", 16, "gzip", 4); + TEST_CHECK(ret == 0); + + if (!TEST_CHECK(c != NULL)) { + TEST_MSG("splunk_client failed"); + exit(EXIT_FAILURE); + } + + ret = flb_http_do(c, &b_sent); + if (!TEST_CHECK(ret == 0)) { + TEST_MSG("ret error. ret=%d\n", ret); + } + else if (!TEST_CHECK(b_sent > 0)){ + TEST_MSG("b_sent size error. b_sent = %lu\n", b_sent); + } + else if (!TEST_CHECK(c->resp.status == 200)) { + TEST_MSG("http response code error. expect: 200, got: %d\n", c->resp.status); + } + flb_free(final_data); + + /* waiting to flush */ + flb_time_msleep(1500); + + num = get_output_num(); + if (!TEST_CHECK(num > 0)) { + TEST_MSG("no outputs"); + } + flb_sds_destroy(target); + flb_http_client_destroy(c); + flb_upstream_conn_release(ctx->httpc->u_conn); + test_ctx_destroy(ctx); +} + +void flb_test_splunk_collector_gzip() +{ + flb_test_splunk_gzip(8813, "/services/collector"); +} + +void flb_test_splunk_collector_event_gzip() +{ + flb_test_splunk_gzip(8814, "/services/collector/event"); +} + +void flb_test_splunk_raw_multilines_gzip(int port) +{ + struct flb_lib_out_cb cb_data; + struct test_ctx *ctx; + struct flb_http_client *c; + int ret; + int num; + size_t b_sent; + char *buf = "127.0.0.1 - admin [28/Sep/2016:09:05:26.875 -0700] \"GET /servicesNS/admin/launcher/data/ui/views?count=-1 HTTP/1.0\" 200 126721 - - - 6ms" \ + "127.0.0.1 - admin [28/Sep/2016:09:05:26.917 -0700] \"GET /servicesNS/admin/launcher/data/ui/nav/default HTTP/1.0\" 200 4367 - - - 6ms" \ + "127.0.0.1 - admin [28/Sep/2016:09:05:26.941 -0700] \"GET /services/apps/local?search=disabled%3Dfalse&count=-1 HTTP/1.0\" 200 31930 - - - 4ms" \ + "127.0.0.1 - admin [28/Sep/2016:09:05:26.954 -0700] \"GET /services/apps/local?search=disabled%3Dfalse&count=-1 HTTP/1.0\" 200 31930 - - - 3ms" \ + "127.0.0.1 - admin [28/Sep/2016:09:05:26.968 -0700] \"GET /servicesNS/admin/launcher/data/ui/views?digest=1&count=-1 HTTP/1.0\" 200 58672 - - - 5ms"; + char *expected = "\"log\":"; + char sport[16]; + void *final_data; + size_t final_bytes; + + snprintf(sport, 16, "%d", port); + + clear_output_num(); + + cb_data.cb = cb_check_result_json; + cb_data.data = expected; + + ctx = test_ctx_create(&cb_data); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("test_ctx_create failed"); + exit(EXIT_FAILURE); + } + + ret = flb_input_set(ctx->flb, ctx->i_ffd, + "port", sport, + NULL); + TEST_CHECK(ret == 0); + + ret = flb_output_set(ctx->flb, ctx->o_ffd, + "match", "*", + "format", "json", + NULL); + TEST_CHECK(ret == 0); + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + ctx->httpc = splunk_client_ctx_create(port); + TEST_CHECK(ctx->httpc != NULL); + + ret = flb_gzip_compress((void *) buf, strlen(buf), &final_data, &final_bytes); + TEST_CHECK(ret != -1); + + c = flb_http_client(ctx->httpc->u_conn, FLB_HTTP_POST, "/services/collector/raw", final_data, final_bytes, + "127.0.0.1", port, NULL, 0); + ret = flb_http_add_header(c, FLB_HTTP_HEADER_CONTENT_TYPE, strlen(FLB_HTTP_HEADER_CONTENT_TYPE), + JSON_CONTENT_TYPE, strlen(JSON_CONTENT_TYPE)); + /* Add Content-Encoding: gzip */ + ret = flb_http_add_header(c, "Content-Encoding", 16, "gzip", 4); + TEST_CHECK(ret == 0); + + TEST_CHECK(ret == 0); + if (!TEST_CHECK(c != NULL)) { + TEST_MSG("splunk_client failed"); + exit(EXIT_FAILURE); + } + + ret = flb_http_do(c, &b_sent); + if (!TEST_CHECK(ret == 0)) { + TEST_MSG("ret error. ret=%d\n", ret); + } + else if (!TEST_CHECK(b_sent > 0)){ + TEST_MSG("b_sent size error. b_sent = %lu\n", b_sent); + } + else if (!TEST_CHECK(c->resp.status == 200)) { + TEST_MSG("http response code error. expect: 200, got: %d\n", c->resp.status); + } + flb_free(final_data); + + /* waiting to flush */ + flb_time_msleep(1500); + + num = get_output_num(); + if (!TEST_CHECK(num > 0)) { + TEST_MSG("no outputs"); + } + flb_http_client_destroy(c); + flb_upstream_conn_release(ctx->httpc->u_conn); + test_ctx_destroy(ctx); +} + +void flb_test_splunk_collector_raw_multilines_gzip() +{ + flb_test_splunk_raw_multilines_gzip(8815); +} + +TEST_LIST = { + {"health", flb_test_splunk_health}, + {"collector", flb_test_splunk_collector}, + {"collector_event", flb_test_splunk_collector_event}, + {"collector_raw", flb_test_splunk_collector_raw}, + {"collector_raw_multilines", flb_test_splunk_collector_raw_multilines}, + {"collector_gzip", flb_test_splunk_collector_gzip}, + {"collector_event_gzip", flb_test_splunk_collector_event_gzip}, + {"collector_raw_multilines_gzip", flb_test_splunk_collector_raw_multilines_gzip}, + {"tag_key", flb_test_splunk_tag_key}, + {NULL, NULL} +}; diff --git a/src/fluent-bit/tests/runtime/in_statsd.c b/src/fluent-bit/tests/runtime/in_statsd.c new file mode 100644 index 000000000..1ae9d6cf3 --- /dev/null +++ b/src/fluent-bit/tests/runtime/in_statsd.c @@ -0,0 +1,324 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2019-2022 The Fluent Bit Authors + * Copyright (C) 2015-2018 Treasure Data Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <fluent-bit.h> +#include <fluent-bit/flb_compat.h> +#include <fluent-bit/flb_time.h> +#include <fluent-bit/flb_socket.h> +#include <sys/types.h> +#include <sys/stat.h> +#include <fcntl.h> +#include "flb_tests_runtime.h" + +struct test_ctx { + flb_ctx_t *flb; /* Fluent Bit library context */ + int i_ffd; /* Input fd */ + int f_ffd; /* Filter fd (unused) */ + int o_ffd; /* Output fd */ +}; + + +pthread_mutex_t result_mutex = PTHREAD_MUTEX_INITIALIZER; +int num_output = 0; +static int get_output_num() +{ + int ret; + pthread_mutex_lock(&result_mutex); + ret = num_output; + pthread_mutex_unlock(&result_mutex); + + return ret; +} + +static void set_output_num(int num) +{ + pthread_mutex_lock(&result_mutex); + num_output = num; + pthread_mutex_unlock(&result_mutex); +} + +static void clear_output_num() +{ + set_output_num(0); +} + +struct str_list { + size_t size; + char **lists; +}; + +/* Callback to check expected results */ +static int cb_check_json_str_list(void *record, size_t size, void *data) +{ + char *p; + char *result; + int num = get_output_num(); + size_t i; + struct str_list *l = (struct str_list*)data; + + if (!TEST_CHECK(l != NULL)) { + flb_error("Data is NULL"); + flb_free(record); + return 0; + } + + + set_output_num(num+1); + + result = (char *) record; + + for (i=0; i<l->size; i++) { + p = strstr(result, l->lists[i]); + if(!TEST_CHECK(p != NULL)) { + flb_error("Expected to find: '%s' in result '%s'", + l->lists[i], result); + } + } + + /* + * If you want to debug your test + * + * printf("Expect: '%s' in result '%s'", expected, result); + */ + flb_free(record); + return 0; +} + +static struct test_ctx *test_ctx_create(struct flb_lib_out_cb *data) +{ + int i_ffd; + int o_ffd; + struct test_ctx *ctx = NULL; + + ctx = flb_malloc(sizeof(struct test_ctx)); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("malloc failed"); + flb_errno(); + return NULL; + } + + /* Service config */ + ctx->flb = flb_create(); + flb_service_set(ctx->flb, + "Flush", "0.200000000", + "Grace", "1", + "Log_Level", "error", + NULL); + + /* Input */ + i_ffd = flb_input(ctx->flb, (char *) "statsd", NULL); + TEST_CHECK(i_ffd >= 0); + ctx->i_ffd = i_ffd; + + /* Output */ + o_ffd = flb_output(ctx->flb, (char *) "lib", (void *) data); + ctx->o_ffd = o_ffd; + + return ctx; +} + +static void test_ctx_destroy(struct test_ctx *ctx) +{ + TEST_CHECK(ctx != NULL); + + sleep(1); + flb_stop(ctx->flb); + flb_destroy(ctx->flb); + flb_free(ctx); +} + +#define DEFAULT_HOST "127.0.0.1" +#define DEFAULT_PORT 8125 +static int init_udp(char *in_host, int in_port, struct sockaddr_in *addr) +{ + int port = in_port; + char *host = in_host; + flb_sockfd_t fd; + + if (host == NULL) { + host = DEFAULT_HOST; + } + if (port < 0) { + port = DEFAULT_PORT; + } + + memset(addr, 0, sizeof(struct sockaddr_in)); + fd = socket(AF_INET, SOCK_DGRAM, 0); + if (!TEST_CHECK(fd >= 0)) { + TEST_MSG("failed to socket. host=%s port=%d errno=%d", host, port, errno); + return -1; + } + + addr->sin_family = AF_INET; + addr->sin_addr.s_addr = inet_addr(host); + addr->sin_port = htons(port); + + return fd; +} + +static int test_normal(char *payload, struct str_list *expected) +{ + struct flb_lib_out_cb cb_data; + struct test_ctx *ctx; + struct sockaddr_in addr; + int fd; + int ret; + int num; + ssize_t w_size; + + size_t size; + + if (!TEST_CHECK(payload != NULL && expected != NULL)) { + TEST_MSG("input is NULL"); + return -1; + } + size = strlen(payload); + + clear_output_num(); + + cb_data.cb = cb_check_json_str_list; + cb_data.data = expected; + + ctx = test_ctx_create(&cb_data); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("test_ctx_create failed"); + exit(EXIT_FAILURE); + } + + ret = flb_output_set(ctx->flb, ctx->o_ffd, + "format", "json", + NULL); + TEST_CHECK(ret == 0); + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + /* use default host/port */ + fd = init_udp(NULL, -1, &addr); + if (!TEST_CHECK(fd >= 0)) { + exit(EXIT_FAILURE); + } + + w_size = sendto(fd, payload, size, 0, (const struct sockaddr *)&addr, sizeof(addr)); + if (!TEST_CHECK(w_size == size)) { + TEST_MSG("failed to send, errno=%d", errno); + flb_socket_close(fd); + exit(EXIT_FAILURE); + } + + /* waiting to flush */ + flb_time_msleep(500); + + num = get_output_num(); + if (!TEST_CHECK(num > 0)) { + TEST_MSG("no outputs"); + } + + flb_socket_close(fd); + test_ctx_destroy(ctx); + + return 0; +} + +void flb_test_statsd_count() +{ + char *expected_strs[] = {"\"bucket\":\"gorets\"", "\"type\":\"counter\"", "\"value\":1"}; + struct str_list expected = { + .size = sizeof(expected_strs)/sizeof(char*), + .lists = &expected_strs[0], + }; + + char *buf = "gorets:1|c"; + int ret; + + ret = test_normal(buf, &expected); + if (!TEST_CHECK(ret == 0)) { + TEST_MSG("test failed"); + exit(EXIT_FAILURE); + } + +} + +void flb_test_statsd_sample() +{ + char *expected_strs[] = {"\"bucket\":\"gorets\"", "\"type\":\"counter\"", + "\"value\":1", "\"sample_rate\":0.1"}; + struct str_list expected = { + .size = sizeof(expected_strs)/sizeof(char*), + .lists = &expected_strs[0], + }; + + char *buf = "gorets:1|c|@0.1"; + int ret; + + ret = test_normal(buf, &expected); + if (!TEST_CHECK(ret == 0)) { + TEST_MSG("test failed"); + exit(EXIT_FAILURE); + } +} + +void flb_test_statsd_gauge() +{ + char *expected_strs[] = {"\"type\":\"gauge\"","\"bucket\":\"gaugor\"", + "\"value\":333"}; + struct str_list expected = { + .size = sizeof(expected_strs)/sizeof(char*), + .lists = &expected_strs[0], + }; + + char *buf = "gaugor:333|g"; + int ret; + + ret = test_normal(buf, &expected); + if (!TEST_CHECK(ret == 0)) { + TEST_MSG("test failed"); + exit(EXIT_FAILURE); + } +} + +void flb_test_statsd_set() +{ + char *expected_strs[] = {"\"bucket\":\"uniques\"", "\"type\":\"set\"", + "\"value\":\"765\""}; + struct str_list expected = { + .size = sizeof(expected_strs)/sizeof(char*), + .lists = &expected_strs[0], + }; + + char *buf = "uniques:765|s"; + int ret; + + ret = test_normal(buf, &expected); + if (!TEST_CHECK(ret == 0)) { + TEST_MSG("test failed"); + exit(EXIT_FAILURE); + } +} + +TEST_LIST = { + {"count", flb_test_statsd_count}, + {"sample", flb_test_statsd_sample}, + {"gauge", flb_test_statsd_gauge}, + {"set", flb_test_statsd_set}, + {NULL, NULL} +}; + diff --git a/src/fluent-bit/tests/runtime/in_syslog.c b/src/fluent-bit/tests/runtime/in_syslog.c new file mode 100644 index 000000000..6935c7a3b --- /dev/null +++ b/src/fluent-bit/tests/runtime/in_syslog.c @@ -0,0 +1,1023 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2019-2022 The Fluent Bit Authors + * Copyright (C) 2015-2018 Treasure Data Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <fluent-bit.h> +#include <fluent-bit/flb_compat.h> +#include <fluent-bit/flb_time.h> +#include <fluent-bit/flb_socket.h> +#include <fluent-bit/flb_parser.h> +#include <sys/types.h> +#include <sys/stat.h> +#ifdef FLB_HAVE_UNIX_SOCKET +#include <sys/socket.h> +#include <sys/un.h> +#endif +#include <fcntl.h> +#include "flb_tests_runtime.h" + +#define DPATH FLB_TESTS_DATA_PATH "/data/common" + +/* Examples from RFCs */ +#define RFC5424_EXAMPLE_1 "<34>1 2003-10-11T22:14:15.003Z mymachine.example.com su - ID47 - 'su root' failed for lonvick on /dev/pts/8\n" +#define RFC3164_EXAMPLE_1 "<34>Oct 11 22:14:15 mymachine su: 'su root' failed for lonvick on /dev/pts/8\n" + +char *RFC5424_EXPECTED_STRS_1[] = {"\"pri\":\"34\"", "\"message\":\"'su root' failed for lonvick on /dev/pts/8\"", + "\"host\":\"mymachine.example.com\"", "\"msgid\":\"ID47\"","\"time\":\"2003-10-11T22:14:15.003Z\"", + "\"ident\":\"su\"" +}; + +char *RFC5424_EXPECTED_STRS_TCP[] = {"\"pri\":\"34\"", "\"message\":\"'su root' failed for lonvick on /dev/pts/8\"", + "\"host\":\"mymachine.example.com\"", "\"msgid\":\"ID47\"","\"time\":\"2003-10-11T22:14:15.003Z\"", + "\"ident\":\"su\"", + "\"source_host\":\"tcp://" +}; + +char *RFC5424_EXPECTED_STRS_UDP[] = {"\"pri\":\"34\"", "\"message\":\"'su root' failed for lonvick on /dev/pts/8\"", + "\"host\":\"mymachine.example.com\"", "\"msgid\":\"ID47\"","\"time\":\"2003-10-11T22:14:15.003Z\"", + "\"ident\":\"su\"", + "\"source_host\":\"udp://" +}; + +char *RFC3164_EXPECTED_STRS_1[] = {"\"pri\":\"34\"", "\"message\":\"'su root' failed for lonvick on /dev/pts/8\"", + "\"host\":\"mymachine\"", "\"time\":\"Oct 11 22:14:15\"", "\"ident\":\"su\"" +}; + + +struct test_ctx { + flb_ctx_t *flb; /* Fluent Bit library context */ + int i_ffd; /* Input fd */ + int f_ffd; /* Filter fd (unused) */ + int o_ffd; /* Output fd */ +}; + + +pthread_mutex_t result_mutex = PTHREAD_MUTEX_INITIALIZER; +int num_output = 0; +static int get_output_num() +{ + int ret; + pthread_mutex_lock(&result_mutex); + ret = num_output; + pthread_mutex_unlock(&result_mutex); + + return ret; +} + +static void set_output_num(int num) +{ + pthread_mutex_lock(&result_mutex); + num_output = num; + pthread_mutex_unlock(&result_mutex); +} + +static void clear_output_num() +{ + set_output_num(0); +} + +struct str_list { + size_t size; + char **lists; +}; + +/* Callback to check expected results */ +static int cb_check_json_str_list(void *record, size_t size, void *data) +{ + char *p; + char *result; + int num = get_output_num(); + size_t i; + struct str_list *l = (struct str_list*)data; + + if (!TEST_CHECK(l != NULL)) { + flb_error("Data is NULL"); + flb_free(record); + return 0; + } + set_output_num(num+1); + + result = (char *) record; + + for (i=0; i<l->size; i++) { + p = strstr(result, l->lists[i]); + if(!TEST_CHECK(p != NULL)) { + flb_error("Expected to find: '%s' in result '%s'", + l->lists[i], result); + } + } + flb_free(record); + return 0; +} + +static struct test_ctx *test_ctx_create(struct flb_lib_out_cb *data) +{ + int i_ffd; + int o_ffd; + int ret; + struct test_ctx *ctx = NULL; + + ctx = flb_malloc(sizeof(struct test_ctx)); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("malloc failed"); + flb_errno(); + return NULL; + } + + /* Service config */ + ctx->flb = flb_create(); + flb_service_set(ctx->flb, + "Flush", "0.200000000", + "Grace", "1", + "Log_Level", "error", + "Parsers_File", DPATH "/parsers.conf", + NULL); + + /* Input */ + i_ffd = flb_input(ctx->flb, (char *) "syslog", NULL); + TEST_CHECK(i_ffd >= 0); + ctx->i_ffd = i_ffd; + + /* Output */ + o_ffd = flb_output(ctx->flb, (char *) "lib", (void *) data); + ctx->o_ffd = o_ffd; + ret = flb_output_set(ctx->flb, ctx->o_ffd, + "format", "json", + NULL); + TEST_CHECK(ret == 0); + + return ctx; +} + +#define PARSER_NAME_RFC5424 "syslog-rfc5424" +#define PARSER_NAME_RFC3164 "syslog-rfc3164" + +static void test_ctx_destroy(struct test_ctx *ctx) +{ + TEST_CHECK(ctx != NULL); + + sleep(1); + flb_stop(ctx->flb); + flb_destroy(ctx->flb); + flb_free(ctx); +} + +#define DEFAULT_HOST "127.0.0.1" +#define DEFAULT_PORT 5140 +static flb_sockfd_t connect_tcp(char *in_host, int in_port) +{ + int port = in_port; + char *host = in_host; + flb_sockfd_t fd; + int ret; + struct sockaddr_in addr; + + if (host == NULL) { + host = DEFAULT_HOST; + } + if (port < 0) { + port = DEFAULT_PORT; + } + + memset(&addr, 0, sizeof(addr)); + fd = socket(PF_INET, SOCK_STREAM, 0); + if (!TEST_CHECK(fd >= 0)) { + TEST_MSG("failed to socket. host=%s port=%d errno=%d", host, port, errno); + return -1; + } + + addr.sin_family = AF_INET; + addr.sin_addr.s_addr = inet_addr(host); + addr.sin_port = htons(port); + + ret = connect(fd, (const struct sockaddr *)&addr, sizeof(addr)); + if (!TEST_CHECK(ret >= 0)) { + TEST_MSG("failed to connect. host=%s port=%d errno=%d", host, port, errno); + flb_socket_close(fd); + return -1; + } + return fd; +} + +#ifdef FLB_HAVE_UNIX_SOCKET +static flb_sockfd_t connect_tcp_unix(char *path) +{ + flb_sockfd_t fd; + struct sockaddr_un sun; + int ret; + + if (!TEST_CHECK(path != NULL)) { + TEST_MSG("path is NULL"); + return -1; + } + + memset(&sun, 0, sizeof(sun)); + fd = socket(AF_LOCAL, SOCK_STREAM, 0); + if (!TEST_CHECK(fd >= 0)) { + TEST_MSG("failed to socket. path=%s errno=%d", path, errno); + return -1; + } + + sun.sun_family = AF_LOCAL; + strcpy(sun.sun_path, path); + ret = connect(fd, (const struct sockaddr *)&sun, sizeof(sun)); + if (!TEST_CHECK(ret >= 0)) { + TEST_MSG("failed to connect. path=%s errno=%d", path, errno); + flb_socket_close(fd); + return -1; + } + return fd; +} + +static flb_sockfd_t init_udp_unix(char *path, struct sockaddr_un *sun) +{ + flb_sockfd_t fd; + int ret; + + if (!TEST_CHECK(path != NULL)) { + TEST_MSG("path is NULL"); + return -1; + } + if (!TEST_CHECK(sun != NULL)) { + TEST_MSG("sun is NULL"); + return -1; + } + + memset(sun, 0, sizeof(struct sockaddr_un)); + fd = socket(AF_LOCAL, SOCK_DGRAM, 0); + if (!TEST_CHECK(fd >= 0)) { + TEST_MSG("failed to socket. path=%s errno=%d", path, errno); + return -1; + } + + sun->sun_family = AF_LOCAL; + strcpy(sun->sun_path, path); + ret = connect(fd, (const struct sockaddr *)sun, sizeof(struct sockaddr_un)); + if (!TEST_CHECK(ret >= 0)) { + TEST_MSG("failed to connect. path=%s errno=%d", path, errno); + flb_socket_close(fd); + return -1; + } + return fd; +} +#endif + +static int init_udp(char *in_host, int in_port, struct sockaddr_in *addr) +{ + int port = in_port; + char *host = in_host; + flb_sockfd_t fd; + + if (host == NULL) { + host = DEFAULT_HOST; + } + if (port < 0) { + port = DEFAULT_PORT; + } + + memset(addr, 0, sizeof(struct sockaddr_in)); + fd = socket(AF_INET, SOCK_DGRAM, 0); + if (!TEST_CHECK(fd >= 0)) { + TEST_MSG("failed to socket. host=%s port=%d errno=%d", host, port, errno); + return -1; + } + + addr->sin_family = AF_INET; + addr->sin_addr.s_addr = inet_addr(host); + addr->sin_port = htons(port); + + return fd; +} + +void flb_test_syslog_tcp() +{ + struct flb_lib_out_cb cb_data; + struct test_ctx *ctx; + flb_sockfd_t fd; + int ret; + int num; + ssize_t w_size; + + struct str_list expected = { + .size = sizeof(RFC5424_EXPECTED_STRS_1)/sizeof(char*), + .lists = &RFC5424_EXPECTED_STRS_1[0], + }; + + char *buf = RFC5424_EXAMPLE_1; + size_t size = strlen(buf); + + clear_output_num(); + + cb_data.cb = cb_check_json_str_list; + cb_data.data = &expected; + + ctx = test_ctx_create(&cb_data); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("test_ctx_create failed"); + exit(EXIT_FAILURE); + } + + ret = flb_input_set(ctx->flb, ctx->i_ffd, + "mode", "tcp", + "parser", PARSER_NAME_RFC5424, + NULL); + TEST_CHECK(ret == 0); + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + /* use default host/port */ + fd = connect_tcp(NULL, -1); + if (!TEST_CHECK(fd >= 0)) { + test_ctx_destroy(ctx); + exit(EXIT_FAILURE); + } + + w_size = send(fd, buf, size, 0); + if (!TEST_CHECK(w_size == size)) { + TEST_MSG("failed to send, errno=%d", errno); + flb_socket_close(fd); + test_ctx_destroy(ctx); + exit(EXIT_FAILURE); + } + + /* waiting to flush */ + flb_time_msleep(500); + + num = get_output_num(); + if (!TEST_CHECK(num > 0)) { + TEST_MSG("no outputs"); + } + + flb_socket_close(fd); + test_ctx_destroy(ctx); +} + +void flb_test_syslog_tcp_port() +{ + struct flb_lib_out_cb cb_data; + struct test_ctx *ctx; + flb_sockfd_t fd; + int ret; + int num; + ssize_t w_size; + char *port = "15140"; + + struct str_list expected = { + .size = sizeof(RFC5424_EXPECTED_STRS_1)/sizeof(char*), + .lists = &RFC5424_EXPECTED_STRS_1[0], + }; + + char *buf = RFC5424_EXAMPLE_1; + size_t size = strlen(buf); + + clear_output_num(); + + cb_data.cb = cb_check_json_str_list; + cb_data.data = &expected; + + ctx = test_ctx_create(&cb_data); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("test_ctx_create failed"); + exit(EXIT_FAILURE); + } + + ret = flb_input_set(ctx->flb, ctx->i_ffd, + "mode", "tcp", + "Port", port, + "parser", PARSER_NAME_RFC5424, + NULL); + TEST_CHECK(ret == 0); + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + /* use default host/port */ + fd = connect_tcp(NULL, atoi(port)); + if (!TEST_CHECK(fd >= 0)) { + test_ctx_destroy(ctx); + exit(EXIT_FAILURE); + } + + w_size = send(fd, buf, size, 0); + if (!TEST_CHECK(w_size == size)) { + TEST_MSG("failed to send, errno=%d", errno); + flb_socket_close(fd); + test_ctx_destroy(ctx); + exit(EXIT_FAILURE); + } + + /* waiting to flush */ + flb_time_msleep(500); + + num = get_output_num(); + if (!TEST_CHECK(num > 0)) { + TEST_MSG("no outputs"); + } + + flb_socket_close(fd); + test_ctx_destroy(ctx); +} + +void flb_test_syslog_tcp_source_address() +{ + struct flb_lib_out_cb cb_data; + struct test_ctx *ctx; + flb_sockfd_t fd; + int ret; + int num; + ssize_t w_size; + + struct str_list expected = { + .size = sizeof(RFC5424_EXPECTED_STRS_TCP)/sizeof(char*), + .lists = &RFC5424_EXPECTED_STRS_TCP[0], + }; + + char *buf = RFC5424_EXAMPLE_1; + size_t size = strlen(buf); + + clear_output_num(); + + cb_data.cb = cb_check_json_str_list; + cb_data.data = &expected; + + ctx = test_ctx_create(&cb_data); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("test_ctx_create failed"); + exit(EXIT_FAILURE); + } + + ret = flb_input_set(ctx->flb, ctx->i_ffd, + "mode", "tcp", + "source_address_key", "source_host", + "parser", PARSER_NAME_RFC5424, + NULL); + TEST_CHECK(ret == 0); + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + /* use default host/port */ + fd = connect_tcp(NULL, -1); + if (!TEST_CHECK(fd >= 0)) { + test_ctx_destroy(ctx); + exit(EXIT_FAILURE); + } + + w_size = send(fd, buf, size, 0); + if (!TEST_CHECK(w_size == size)) { + TEST_MSG("failed to send, errno=%d", errno); + flb_socket_close(fd); + test_ctx_destroy(ctx); + exit(EXIT_FAILURE); + } + + /* waiting to flush */ + flb_time_msleep(500); + + num = get_output_num(); + if (!TEST_CHECK(num > 0)) { + TEST_MSG("no outputs"); + } + + flb_socket_close(fd); + test_ctx_destroy(ctx); +} + +void flb_test_syslog_unknown_mode() +{ + struct flb_lib_out_cb cb_data; + struct test_ctx *ctx; + int ret; + + struct str_list expected = { + .size = sizeof(RFC5424_EXPECTED_STRS_1)/sizeof(char*), + .lists = &RFC5424_EXPECTED_STRS_1[0], + }; + + + clear_output_num(); + + cb_data.cb = cb_check_json_str_list; + cb_data.data = &expected; + + ctx = test_ctx_create(&cb_data); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("test_ctx_create failed"); + exit(EXIT_FAILURE); + } + + ret = flb_input_set(ctx->flb, ctx->i_ffd, + "mode", "UNKNOWN", + "parser", PARSER_NAME_RFC5424, + NULL); + TEST_CHECK(ret == 0); + + /* Start the engine */ + ret = flb_start(ctx->flb); + if(!TEST_CHECK(ret != 0)) { + TEST_MSG("flb_start should be failed"); + test_ctx_destroy(ctx); + exit(EXIT_FAILURE); + } + + test_ctx_destroy(ctx); +} + +void flb_test_syslog_unix_perm() +{ + struct flb_lib_out_cb cb_data; + struct test_ctx *ctx; + int ret; + char *unix_path = "in_syslog_unix"; + struct stat sb; + struct str_list expected = { + .size = sizeof(RFC5424_EXPECTED_STRS_1)/sizeof(char*), + .lists = &RFC5424_EXPECTED_STRS_1[0], + }; + + + clear_output_num(); + + cb_data.cb = cb_check_json_str_list; + cb_data.data = &expected; + + ctx = test_ctx_create(&cb_data); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("test_ctx_create failed"); + exit(EXIT_FAILURE); + } + + ret = flb_input_set(ctx->flb, ctx->i_ffd, + "path", unix_path, + "unix_perm", "0600", + "parser", PARSER_NAME_RFC5424, + NULL); + TEST_CHECK(ret == 0); + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + ret = stat(unix_path, &sb); + if (!TEST_CHECK(ret == 0)) { + TEST_MSG("stat failed. errno=%d", errno); + test_ctx_destroy(ctx); + exit(EXIT_FAILURE); + } + + if (!TEST_CHECK((sb.st_mode & S_IRWXO) == 0)) { + TEST_MSG("Permssion(others) error. val=0x%x",sb.st_mode & S_IRWXO); + } + if (!TEST_CHECK((sb.st_mode & S_IRWXG) == 0)) { + TEST_MSG("Permssion(group) error. val=0x%x",sb.st_mode & S_IRWXG); + } + if (!TEST_CHECK((sb.st_mode & S_IRWXU) == (S_IRUSR | S_IWUSR))) { + TEST_MSG("Permssion(user) error. val=0x%x",sb.st_mode & S_IRWXU); + } + + test_ctx_destroy(ctx); +} + +void flb_test_syslog_udp() +{ + struct flb_lib_out_cb cb_data; + struct test_ctx *ctx; + struct sockaddr_in addr; + flb_sockfd_t fd; + int ret; + int num; + ssize_t w_size; + + struct str_list expected = { + .size = sizeof(RFC5424_EXPECTED_STRS_1)/sizeof(char*), + .lists = &RFC5424_EXPECTED_STRS_1[0], + }; + + char *buf = RFC5424_EXAMPLE_1; + size_t size = strlen(buf); + + clear_output_num(); + + cb_data.cb = cb_check_json_str_list; + cb_data.data = &expected; + + ctx = test_ctx_create(&cb_data); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("test_ctx_create failed"); + exit(EXIT_FAILURE); + } + + ret = flb_input_set(ctx->flb, ctx->i_ffd, + "mode", "udp", + "parser", PARSER_NAME_RFC5424, + NULL); + TEST_CHECK(ret == 0); + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + /* use default host/port */ + fd = init_udp(NULL, -1, &addr); + if (!TEST_CHECK(fd >= 0)) { + test_ctx_destroy(ctx); + exit(EXIT_FAILURE); + } + + w_size = sendto(fd, buf, size, 0, (const struct sockaddr *)&addr, sizeof(addr)); + if (!TEST_CHECK(w_size == size)) { + TEST_MSG("failed to send, errno=%d", errno); + flb_socket_close(fd); + test_ctx_destroy(ctx); + exit(EXIT_FAILURE); + } + + /* waiting to flush */ + flb_time_msleep(500); + + num = get_output_num(); + if (!TEST_CHECK(num > 0)) { + TEST_MSG("no outputs"); + } + + flb_socket_close(fd); + test_ctx_destroy(ctx); +} + +void flb_test_syslog_udp_port() +{ + struct flb_lib_out_cb cb_data; + struct test_ctx *ctx; + struct sockaddr_in addr; + flb_sockfd_t fd; + int ret; + int num; + ssize_t w_size; + char *port = "15140"; + + struct str_list expected = { + .size = sizeof(RFC5424_EXPECTED_STRS_1)/sizeof(char*), + .lists = &RFC5424_EXPECTED_STRS_1[0], + }; + + char *buf = RFC5424_EXAMPLE_1; + size_t size = strlen(buf); + + clear_output_num(); + + cb_data.cb = cb_check_json_str_list; + cb_data.data = &expected; + + ctx = test_ctx_create(&cb_data); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("test_ctx_create failed"); + exit(EXIT_FAILURE); + } + + ret = flb_input_set(ctx->flb, ctx->i_ffd, + "mode", "udp", + "Port", port, + "parser", PARSER_NAME_RFC5424, + NULL); + TEST_CHECK(ret == 0); + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + /* use default host/port */ + fd = init_udp(NULL, atoi(port), &addr); + if (!TEST_CHECK(fd >= 0)) { + test_ctx_destroy(ctx); + exit(EXIT_FAILURE); + } + + w_size = sendto(fd, buf, size, 0, (const struct sockaddr *)&addr, sizeof(addr)); + if (!TEST_CHECK(w_size == size)) { + TEST_MSG("failed to send, errno=%d", errno); + flb_socket_close(fd); + test_ctx_destroy(ctx); + exit(EXIT_FAILURE); + } + + /* waiting to flush */ + flb_time_msleep(500); + + num = get_output_num(); + if (!TEST_CHECK(num > 0)) { + TEST_MSG("no outputs"); + } + + flb_socket_close(fd); + test_ctx_destroy(ctx); +} + +void flb_test_syslog_udp_source_address() +{ + struct flb_lib_out_cb cb_data; + struct test_ctx *ctx; + struct sockaddr_in addr; + flb_sockfd_t fd; + int ret; + int num; + ssize_t w_size; + + struct str_list expected = { + .size = sizeof(RFC5424_EXPECTED_STRS_UDP)/sizeof(char*), + .lists = &RFC5424_EXPECTED_STRS_UDP[0], + }; + + char *buf = RFC5424_EXAMPLE_1; + size_t size = strlen(buf); + + clear_output_num(); + + cb_data.cb = cb_check_json_str_list; + cb_data.data = &expected; + + ctx = test_ctx_create(&cb_data); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("test_ctx_create failed"); + exit(EXIT_FAILURE); + } + + ret = flb_input_set(ctx->flb, ctx->i_ffd, + "mode", "udp", + "source_address_key", "source_host", + "parser", PARSER_NAME_RFC5424, + NULL); + TEST_CHECK(ret == 0); + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + /* use default host/port */ + fd = init_udp(NULL, -1, &addr); + if (!TEST_CHECK(fd >= 0)) { + test_ctx_destroy(ctx); + exit(EXIT_FAILURE); + } + + w_size = sendto(fd, buf, size, 0, (const struct sockaddr *)&addr, sizeof(addr)); + if (!TEST_CHECK(w_size == size)) { + TEST_MSG("failed to send, errno=%d", errno); + flb_socket_close(fd); + test_ctx_destroy(ctx); + exit(EXIT_FAILURE); + } + + /* waiting to flush */ + flb_time_msleep(500); + + num = get_output_num(); + if (!TEST_CHECK(num > 0)) { + TEST_MSG("no outputs"); + } + + flb_socket_close(fd); + test_ctx_destroy(ctx); +} + +#ifdef FLB_HAVE_UNIX_SOCKET +void flb_test_syslog_tcp_unix() +{ + struct flb_lib_out_cb cb_data; + struct test_ctx *ctx; + flb_sockfd_t fd; + int ret; + int num; + ssize_t w_size; + char *unix_path = "in_syslog_unix"; + + struct str_list expected = { + .size = sizeof(RFC5424_EXPECTED_STRS_1)/sizeof(char*), + .lists = &RFC5424_EXPECTED_STRS_1[0], + }; + + char *buf = RFC5424_EXAMPLE_1; + size_t size = strlen(buf); + + clear_output_num(); + + cb_data.cb = cb_check_json_str_list; + cb_data.data = &expected; + + ctx = test_ctx_create(&cb_data); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("test_ctx_create failed"); + exit(EXIT_FAILURE); + } + + ret = flb_input_set(ctx->flb, ctx->i_ffd, + "mode", "unix_tcp", + "path", unix_path, + "parser", PARSER_NAME_RFC5424, + NULL); + TEST_CHECK(ret == 0); + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + /* use default host/port */ + fd = connect_tcp_unix(unix_path); + if (!TEST_CHECK(fd >= 0)) { + test_ctx_destroy(ctx); + exit(EXIT_FAILURE); + } + + w_size = send(fd, buf, size, 0); + if (!TEST_CHECK(w_size == size)) { + TEST_MSG("failed to send, errno=%d", errno); + flb_socket_close(fd); + test_ctx_destroy(ctx); + exit(EXIT_FAILURE); + } + + /* waiting to flush */ + flb_time_msleep(500); + + num = get_output_num(); + if (!TEST_CHECK(num > 0)) { + TEST_MSG("no outputs"); + } + + flb_socket_close(fd); + test_ctx_destroy(ctx); +} + +void flb_test_syslog_udp_unix() +{ + struct flb_lib_out_cb cb_data; + struct test_ctx *ctx; + struct sockaddr_un sun; + flb_sockfd_t fd; + int ret; + int num; + ssize_t w_size; + char *unix_path = "in_syslog_unix"; + + struct str_list expected = { + .size = sizeof(RFC5424_EXPECTED_STRS_1)/sizeof(char*), + .lists = &RFC5424_EXPECTED_STRS_1[0], + }; + + char *buf = RFC5424_EXAMPLE_1; + size_t size = strlen(buf); + + clear_output_num(); + + cb_data.cb = cb_check_json_str_list; + cb_data.data = &expected; + + ctx = test_ctx_create(&cb_data); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("test_ctx_create failed"); + exit(EXIT_FAILURE); + } + + ret = flb_input_set(ctx->flb, ctx->i_ffd, + "mode", "unix_udp", + "path", unix_path, + "parser", PARSER_NAME_RFC5424, + NULL); + TEST_CHECK(ret == 0); + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + fd = init_udp_unix(unix_path, &sun); + if (!TEST_CHECK(fd >= 0)) { + test_ctx_destroy(ctx); + exit(EXIT_FAILURE); + } + + w_size = sendto(fd, buf, size, 0, (const struct sockaddr *)&sun, sizeof(sun)); + if (!TEST_CHECK(w_size == size)) { + TEST_MSG("failed to send, errno=%d", errno); + flb_socket_close(fd); + test_ctx_destroy(ctx); + exit(EXIT_FAILURE); + } + + /* waiting to flush */ + flb_time_msleep(500); + + num = get_output_num(); + if (!TEST_CHECK(num > 0)) { + TEST_MSG("no outputs"); + } + + flb_socket_close(fd); + test_ctx_destroy(ctx); +} +#endif + +void flb_test_syslog_rfc3164() +{ + struct flb_lib_out_cb cb_data; + struct test_ctx *ctx; + flb_sockfd_t fd; + int ret; + int num; + ssize_t w_size; + + struct str_list expected = { + .size = sizeof(RFC3164_EXPECTED_STRS_1)/sizeof(char*), + .lists = &RFC3164_EXPECTED_STRS_1[0], + }; + + char *buf = RFC3164_EXAMPLE_1; + size_t size = strlen(buf); + + clear_output_num(); + + cb_data.cb = cb_check_json_str_list; + cb_data.data = &expected; + + ctx = test_ctx_create(&cb_data); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("test_ctx_create failed"); + exit(EXIT_FAILURE); + } + + ret = flb_input_set(ctx->flb, ctx->i_ffd, + "mode", "tcp", + "parser", PARSER_NAME_RFC3164, + NULL); + TEST_CHECK(ret == 0); + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + /* use default host/port */ + fd = connect_tcp(NULL, -1); + if (!TEST_CHECK(fd >= 0)) { + test_ctx_destroy(ctx); + exit(EXIT_FAILURE); + } + + w_size = send(fd, buf, size, 0); + if (!TEST_CHECK(w_size == size)) { + TEST_MSG("failed to send, errno=%d", errno); + flb_socket_close(fd); + test_ctx_destroy(ctx); + exit(EXIT_FAILURE); + } + + /* waiting to flush */ + flb_time_msleep(500); + + num = get_output_num(); + if (!TEST_CHECK(num > 0)) { + TEST_MSG("no outputs"); + } + + flb_socket_close(fd); + test_ctx_destroy(ctx); +} + +TEST_LIST = { + {"syslog_tcp", flb_test_syslog_tcp}, + {"syslog_udp", flb_test_syslog_udp}, + {"syslog_tcp_port", flb_test_syslog_tcp_port}, + {"syslog_tcp_source_address", flb_test_syslog_tcp_source_address}, + {"syslog_udp_port", flb_test_syslog_udp_port}, + {"syslog_udp_source_address", flb_test_syslog_udp_source_address}, + {"syslog_unknown_mode", flb_test_syslog_unknown_mode}, +#ifdef FLB_HAVE_UNIX_SOCKET + {"syslog_unix_perm", flb_test_syslog_unix_perm}, +#endif + {"syslog_rfc3164", flb_test_syslog_rfc3164}, +#ifdef FLB_HAVE_UNIX_SOCKET + {"syslog_tcp_unix", flb_test_syslog_tcp_unix}, +#ifndef FLB_SYSTEM_MACOS + {"syslog_udp_unix", flb_test_syslog_udp_unix}, +#endif +#endif + {NULL, NULL} +}; + diff --git a/src/fluent-bit/tests/runtime/in_tail.c b/src/fluent-bit/tests/runtime/in_tail.c new file mode 100644 index 000000000..ee5fba887 --- /dev/null +++ b/src/fluent-bit/tests/runtime/in_tail.c @@ -0,0 +1,1583 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2019-2022 The Fluent Bit Authors + * Copyright (C) 2015-2018 Treasure Data Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* +Approach for this tests is basing on filter_kubernetes tests +*/ + +#include <fluent-bit.h> +#include <fluent-bit/flb_time.h> +#include <fluent-bit/flb_pthread.h> +#include <fluent-bit/flb_compat.h> +#include <stdlib.h> +#include <sys/stat.h> +#include <sys/types.h> +#include <fcntl.h> +#include <string.h> +#include "flb_tests_runtime.h" + +#define NEW_LINE "\n" +#define PATH_SEPARATOR "/" + +#define DPATH_COMMON FLB_TESTS_DATA_PATH "/data/common" + +#ifdef _WIN32 + #define TIME_EPSILON_MS 30 +#else + #define TIME_EPSILON_MS 10 +#endif + +struct test_tail_ctx { + flb_ctx_t *flb; /* Fluent Bit library context */ + int i_ffd; /* Input fd */ + int o_ffd; /* Output fd */ + char **filepaths; + int *fds; + int fd_num; +}; + +pthread_mutex_t result_mutex = PTHREAD_MUTEX_INITIALIZER; +int num_output = 0; +static int get_output_num() +{ + int ret; + pthread_mutex_lock(&result_mutex); + ret = num_output; + pthread_mutex_unlock(&result_mutex); + + return ret; +} + +static void set_output_num(int num) +{ + pthread_mutex_lock(&result_mutex); + num_output = num; + pthread_mutex_unlock(&result_mutex); +} + +static void clear_output_num() +{ + set_output_num(0); +} + +static int cb_count_msgpack(void *record, size_t size, void *data) +{ + msgpack_unpacked result; + size_t off = 0; + + /* Iterate each item array and apply rules */ + msgpack_unpacked_init(&result); + while (msgpack_unpack_next(&result, record, size, &off) == MSGPACK_UNPACK_SUCCESS) { + pthread_mutex_lock(&result_mutex); + num_output++; + /* + msgpack_object_print(stdout, result.data); + puts(NEW_LINE); + */ + pthread_mutex_unlock(&result_mutex); + } + msgpack_unpacked_destroy(&result); + + flb_free(record); + return 0; +} + +struct str_list { + size_t size; + char **lists; +}; + +/* Callback to check expected results */ +static int cb_check_json_str_list(void *record, size_t size, void *data) +{ + char *p; + char *result; + int num = get_output_num(); + size_t i; + struct str_list *l = (struct str_list*)data; + + if (!TEST_CHECK(l != NULL)) { + TEST_MSG("Data is NULL"); + flb_free(record); + return 0; + } + + + set_output_num(num+1); + + result = (char *) record; + + for (i=0; i<l->size; i++) { + p = strstr(result, l->lists[i]); + if(!TEST_CHECK(p != NULL)) { + TEST_MSG("Expected to find: '%s' in result '%s'", + l->lists[i], result); + } + } + + flb_free(record); + return 0; +} + +static struct test_tail_ctx *test_tail_ctx_create(struct flb_lib_out_cb *data, + char **paths, int path_num, int override) +{ + int i_ffd; + int o_ffd; + int i; + int j; + int fd; + int o_flags; + struct test_tail_ctx *ctx = NULL; + + if (!TEST_CHECK(data != NULL)){ + TEST_MSG("data is NULL"); + return NULL; + } + + ctx = flb_malloc(sizeof(struct test_tail_ctx)); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("malloc failed"); + flb_errno(); + return NULL; + } + ctx->fds = NULL; + ctx->filepaths = NULL; + ctx->fd_num = path_num; + + /* Service config */ + ctx->flb = flb_create(); + flb_service_set(ctx->flb, + "Flush", "0.200000000", + "Grace", "1", + "Log_Level", "info", + "Parsers_File", DPATH_COMMON "/parsers.conf", + NULL); + + /* Input */ + i_ffd = flb_input(ctx->flb, (char *) "tail", NULL); + TEST_CHECK(i_ffd >= 0); + ctx->i_ffd = i_ffd; + + /* Output */ + o_ffd = flb_output(ctx->flb, (char *) "lib", (void *) data); + ctx->o_ffd = o_ffd; + + /* open() flags */ + o_flags = O_RDWR | O_CREAT; + + if (paths != NULL) { + ctx->fds = flb_malloc(sizeof(int) * path_num); + ctx->filepaths = paths; + if (!TEST_CHECK(ctx->fds != NULL)) { + TEST_MSG("malloc failed"); + flb_destroy(ctx->flb); + flb_free(ctx); + flb_errno(); + return NULL; + } + + for (i=0; i<path_num; i++) { + if (override) { + unlink(paths[i]); + } + else { + o_flags |= O_APPEND; + } + + fd = open(paths[i], o_flags, S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP); + if (!TEST_CHECK(fd >= 0)) { + TEST_MSG("open failed. errno=%d path[%d]=%s", errno, i, paths[i]); + flb_destroy(ctx->flb); + for (j=0; j<i; j++) { + close(ctx->fds[j]); + } + flb_free(ctx->fds); + flb_free(ctx); + flb_errno(); + return NULL; + } + ctx->fds[i] = fd; + } + } + + return ctx; +} + +static void test_tail_ctx_destroy(struct test_tail_ctx *ctx) +{ + int i; + TEST_CHECK(ctx != NULL); + + if (ctx->fds != NULL) { + for (i=0; i <ctx->fd_num; i++) { + close(ctx->fds[i]); + unlink(ctx->filepaths[i]); + } + flb_free(ctx->fds); + } + + sleep(1); + flb_stop(ctx->flb); + flb_destroy(ctx->flb); + flb_free(ctx); +} + +static ssize_t write_msg(struct test_tail_ctx *ctx, char *msg, size_t msg_len) +{ + int i; + ssize_t w_byte; + + for (i = 0; i <ctx->fd_num; i++) { + flb_time_msleep(100); + w_byte = write(ctx->fds[i], msg, msg_len); + if (!TEST_CHECK(w_byte == msg_len)) { + TEST_MSG("write failed ret=%ld", w_byte); + return -1; + } + /* new line */ + w_byte = write(ctx->fds[i], NEW_LINE, strlen(NEW_LINE)); + if (!TEST_CHECK(w_byte == strlen(NEW_LINE))) { + TEST_MSG("write failed ret=%ld", w_byte); + return -1; + } + fsync(ctx->fds[i]); + flb_time_msleep(100); + } + return w_byte; +} + + +#define DPATH FLB_TESTS_DATA_PATH "/data/tail" +#define MAX_LINES 32 + +int64_t result_time; +struct tail_test_result { + const char *target; + int nMatched; + int nNotMatched; + int nLines; +}; + +struct tail_file_lines { + char *lines[MAX_LINES]; + int lines_c; +}; + +void wait_with_timeout(uint32_t timeout_ms, struct tail_test_result *result, int nExpected) +{ + struct flb_time start_time; + struct flb_time end_time; + struct flb_time diff_time; + uint64_t elapsed_time_flb = 0; + + flb_time_get(&start_time); + + while (true) { + if (result->nMatched == nExpected) { + break; + } + + flb_time_msleep(100); + flb_time_get(&end_time); + flb_time_diff(&end_time, &start_time, &diff_time); + elapsed_time_flb = flb_time_to_nanosec(&diff_time) / 1000000; + + if (elapsed_time_flb > timeout_ms - TIME_EPSILON_MS) { + flb_warn("[timeout] elapsed_time: %ld", elapsed_time_flb); + // Reached timeout. + break; + } + } +} + +static inline int64_t set_result(int64_t v) +{ + int64_t old = __sync_lock_test_and_set(&result_time, v); + return old; +} + + +static int file_to_buf(const char *path, char **out_buf, size_t *out_size) +{ + int ret; + long bytes; + char *buf; + FILE *fp; + struct stat st; + + ret = stat(path, &st); + if (ret == -1) { + return -1; + } + + fp = fopen(path, "r"); + if (!fp) { + return -1; + } + + buf = flb_malloc(st.st_size+1); + if (!buf) { + flb_errno(); + fclose(fp); + return -1; + } + + bytes = fread(buf, st.st_size, 1, fp); + if (bytes != 1) { + flb_errno(); + flb_free(buf); + fclose(fp); + return -1; + } + + fclose(fp); + buf[st.st_size] = '\0'; + *out_buf = buf; + *out_size = st.st_size; + + return 0; +} + +/* Given a target, lookup the .out file and return it content in a tail_file_lines structure */ +static struct tail_file_lines *get_out_file_content(const char *target) +{ + int ret; + char file[PATH_MAX]; + char *p; + char *out_buf; + size_t out_size; + struct tail_file_lines *file_lines = flb_malloc(sizeof (struct tail_file_lines)); + file_lines->lines_c = 0; + + snprintf(file, sizeof(file) - 1, DPATH "/out/%s.out", target); + + ret = file_to_buf(file, &out_buf, &out_size); + TEST_CHECK_(ret == 0, "getting output file content: %s", file); + if (ret != 0) { + file_lines->lines_c = 0; + return file_lines; + } + + file_lines->lines[file_lines->lines_c++] = out_buf; + + int i; + for (i=0; i<out_size; i++) { + // Nullify \n and \r characters + p = (char *)(out_buf + i); + if (*p == '\n' || *p == '\r') { + *p = '\0'; + + if (i == out_size - 1) { + break; + } + + if (*++p != '\0' && *p != '\n' && *p != '\r' && file_lines->lines_c < MAX_LINES) { + file_lines->lines[file_lines->lines_c++] = p; + } + } + } + + return file_lines; +} + +static int cb_check_result(void *record, size_t size, void *data) +{ + struct tail_test_result *result; + struct tail_file_lines *out; + + result = (struct tail_test_result *) data; + + char *check; + + out = get_out_file_content(result->target); + if (!out->lines_c) { + goto exit; + } + /* + * Our validation is: check that the one of the output lines + * in the output record. + */ + int i; + result->nLines = out->lines_c; + for (i=0; i<out->lines_c; i++) { + check = strstr(record, out->lines[i]); + if (check != NULL) { + result->nMatched++; + goto exit; + } + } + result->nNotMatched++; +exit: + if (size > 0) { + flb_free(record); + } + if (out->lines_c) { + flb_free(out->lines[0]); + flb_free(out); + } + return 0; +} + +void do_test(char *system, const char *target, int tExpected, int nExpected, ...) +{ + int64_t ret; + flb_ctx_t *ctx = NULL; + int in_ffd; + int out_ffd; + va_list va; + char *key; + char *value; + char path[PATH_MAX]; + struct tail_test_result result = {0}; + + result.nMatched = 0; + result.target = target; + + struct flb_lib_out_cb cb; + cb.cb = cb_check_result; + cb.data = &result; + + /* initialize */ + set_result(0); + + ctx = flb_create(); + + ret = flb_service_set(ctx, + "Log_Level", "error", + "Parsers_File", DPATH "/parsers.conf", + NULL); + TEST_CHECK_(ret == 0, "setting service options"); + + in_ffd = flb_input(ctx, (char *) system, NULL); + TEST_CHECK(in_ffd >= 0); + TEST_CHECK(flb_input_set(ctx, in_ffd, "tag", "test", NULL) == 0); + + /* Compose path based on target */ + snprintf(path, sizeof(path) - 1, DPATH "/log/%s.log", target); + TEST_CHECK_(access(path, R_OK) == 0, "accessing log file: %s", path); + + TEST_CHECK(flb_input_set(ctx, in_ffd, + "path" , path, + "docker_mode" , "on", + "parser" , "docker", + "read_from_head", "true", + NULL) == 0); + + va_start(va, nExpected); + while ((key = va_arg(va, char *))) { + value = va_arg(va, char *); + TEST_CHECK(value != NULL); + TEST_CHECK(flb_input_set(ctx, in_ffd, key, value, NULL) == 0); + } + va_end(va); + + out_ffd = flb_output(ctx, (char *) "lib", &cb); + TEST_CHECK(out_ffd >= 0); + TEST_CHECK(flb_output_set(ctx, out_ffd, + "match", "test", + "format", "json", + NULL) == 0); + + TEST_CHECK(flb_service_set(ctx, "Flush", "0.5", + "Grace", "1", + NULL) == 0); + + /* Start test */ + /* Start the engine */ + ret = flb_start(ctx); + TEST_CHECK_(ret == 0, "starting engine"); + + /* Poll for up to 5 seconds or until we got a match */ + for (ret = 0; ret < tExpected && result.nMatched < nExpected; ret++) { + usleep(1000); + } + + /* Wait until matching nExpected results */ + wait_with_timeout(5000, &result, nExpected); + + TEST_CHECK(result.nMatched == nExpected); + TEST_MSG("result.nMatched: %i\nnExpected: %i", result.nMatched, nExpected); + + ret = flb_stop(ctx); + TEST_CHECK_(ret == 0, "stopping engine"); + + if (ctx) { + flb_destroy(ctx); + } +} + +void flb_test_in_tail_dockermode() +{ + do_test("tail", "dockermode", 20000, 3, + NULL); +} + +void flb_test_in_tail_dockermode_splitted_line() +{ + do_test("tail", "dockermode_splitted_line", 20000, 2, + NULL); +} + +void flb_test_in_tail_dockermode_multiple_lines() +{ + do_test("tail", "dockermode_multiple_lines", 20000, 2, + "Docker_Mode_Parser", "docker_multiline", + NULL); +} + +void flb_test_in_tail_dockermode_splitted_multiple_lines() +{ + do_test("tail", "dockermode_splitted_multiple_lines", 20000, 2, + "Docker_Mode_Parser", "docker_multiline", + NULL); +} + +void flb_test_in_tail_dockermode_firstline_detection() +{ + do_test("tail", "dockermode_firstline_detection", 20000, 5, + "Docker_Mode_Parser", "docker_multiline", + NULL); +} + +int write_long_lines(int fd) { + ssize_t ret; + int i; + const char* data = "0123456789abcdef" "0123456789abcdef"; + size_t len = strlen(data); + + for (i=0; i<1024; i++) { + ret = write(fd, data, strlen(data)); + if (ret < 0) { + flb_errno(); + return -1; + } + else if(ret != len) { + write(fd, &data[ret], len-ret); + } + } + + write(fd, "\n", 1); + return 0; +} + +void flb_test_in_tail_skip_long_lines() +{ + int64_t ret; + flb_ctx_t *ctx = NULL; + int in_ffd; + int out_ffd; + char path[PATH_MAX]; + struct tail_test_result result = {0}; + int fd; + + char *target = "skip_long_lines"; + int nExpected = 2; + int nExpectedNotMatched = 0; + int nExpectedLines = 2; + + result.nMatched = 0; + result.target = target; + + struct flb_lib_out_cb cb; + cb.cb = cb_check_result; + cb.data = &result; + + /* initialize */ + set_result(0); + + ctx = flb_create(); + + ret = flb_service_set(ctx, + "Log_Level", "error", + NULL); + TEST_CHECK_(ret == 0, "setting service options"); + + in_ffd = flb_input(ctx, "tail", NULL); + TEST_CHECK(in_ffd >= 0); + TEST_CHECK(flb_input_set(ctx, in_ffd, "tag", "test", NULL) == 0); + + /* Compose path based on target */ + snprintf(path, sizeof(path) - 1, DPATH "/log/%s.log", target); + fd = creat(path, S_IRWXU | S_IRGRP); + TEST_CHECK(fd >= 0); + + /* Write log + ======= + before_long_line + (long line which should be skipped) + after_long_line + ======= + + Output should be "before_long_line" and "after_long_line" + */ + write(fd, "before_long_line\n", strlen("before_long_line\n")); + write_long_lines(fd); + write(fd, "after_long_line\n", strlen("after_long_line\n")); + close(fd); + + TEST_CHECK_(access(path, R_OK) == 0, "accessing log file: %s", path); + + TEST_CHECK(flb_input_set(ctx, in_ffd, + "path" , path, + "read_from_head", "true", + "skip_long_lines", "on", + NULL) == 0); + + out_ffd = flb_output(ctx, (char *) "lib", &cb); + TEST_CHECK(out_ffd >= 0); + TEST_CHECK(flb_output_set(ctx, out_ffd, + "match", "test", + "format", "json", + NULL) == 0); + + TEST_CHECK(flb_service_set(ctx, "Flush", "0.5", + "Grace", "1", + NULL) == 0); + + /* Start test */ + /* Start the engine */ + ret = flb_start(ctx); + TEST_CHECK_(ret == 0, "starting engine"); + + wait_with_timeout(5000, &result, nExpected); + + TEST_CHECK(result.nMatched == nExpected); + TEST_MSG("result.nMatched: %i\nnExpected: %i", result.nMatched, nExpected); + TEST_CHECK(result.nNotMatched == nExpectedNotMatched); + TEST_MSG("result.nNotMatched: %i\nnExpectedNotMatched: %i", result.nNotMatched, nExpectedNotMatched); + TEST_CHECK(result.nLines == nExpectedLines); + TEST_MSG("result.nLines: %i\nnExpectedLines: %i", result.nLines, nExpectedLines); + + ret = flb_stop(ctx); + TEST_CHECK_(ret == 0, "stopping engine"); + + if (ctx) { + flb_destroy(ctx); + } + + unlink(path); +} + +/* + * test case for https://github.com/fluent/fluent-bit/issues/3943 + * + * test to read the lines "CRLF + empty_line + LF" + */ +void flb_test_in_tail_issue_3943() +{ + int64_t ret; + flb_ctx_t *ctx = NULL; + int in_ffd; + int out_ffd; + char path[PATH_MAX]; + struct tail_test_result result = {0}; + + char *target = "3943"; + int nExpected = 2; + int nExpectedNotMatched = 0; + int nExpectedLines = 2; + + result.nMatched = 0; + result.target = target; + + struct flb_lib_out_cb cb; + cb.cb = cb_check_result; + cb.data = &result; + + /* initialize */ + set_result(0); + + ctx = flb_create(); + + ret = flb_service_set(ctx, + "Log_Level", "error", + NULL); + TEST_CHECK_(ret == 0, "setting service options"); + + in_ffd = flb_input(ctx, "tail", NULL); + TEST_CHECK(in_ffd >= 0); + TEST_CHECK(flb_input_set(ctx, in_ffd, "tag", "test", NULL) == 0); + + snprintf(path, sizeof(path) - 1, DPATH "/log/%s.log", target); + TEST_CHECK_(access(path, R_OK) == 0, "accessing log file: %s", path); + + TEST_CHECK(flb_input_set(ctx, in_ffd, + "path" , path, + "read_from_head", "true", + NULL) == 0); + + out_ffd = flb_output(ctx, (char *) "lib", &cb); + TEST_CHECK(out_ffd >= 0); + TEST_CHECK(flb_output_set(ctx, out_ffd, + "match", "test", + "format", "json", + NULL) == 0); + + TEST_CHECK(flb_service_set(ctx, "Flush", "0.5", + "Grace", "1", + NULL) == 0); + + /* Start test */ + /* Start the engine */ + ret = flb_start(ctx); + TEST_CHECK_(ret == 0, "starting engine"); + + wait_with_timeout(3000, &result, nExpected); + + TEST_CHECK(result.nMatched == nExpected); + TEST_MSG("result.nMatched: %i\nnExpected: %i", result.nMatched, nExpected); + TEST_CHECK(result.nNotMatched == nExpectedNotMatched); + TEST_MSG("result.nNotMatched: %i\nnExpectedNotMatched: %i", result.nNotMatched, nExpectedNotMatched); + TEST_CHECK(result.nLines == nExpectedLines); + TEST_MSG("result.nLines: %i\nnExpectedLines: %i", result.nLines, nExpectedLines); + + ret = flb_stop(ctx); + TEST_CHECK_(ret == 0, "stopping engine"); + + if (ctx) { + flb_destroy(ctx); + } +} + +void flb_test_in_tail_multiline_json_and_regex() +{ + int64_t ret; + int in_ffd; + int out_ffd; + int n_expected; + int t_expected; + char *target; + char path[PATH_MAX]; + struct tail_test_result result = {0}; + flb_ctx_t *ctx; + + target = "multiline_001"; + result.nMatched = 0; + result.target = target; + + struct flb_lib_out_cb cb; + cb.cb = cb_check_result; + cb.data = &result; + + /* initialize */ + set_result(0); + + ctx = flb_create(); + + TEST_CHECK(flb_service_set(ctx, "Flush", "0.5", + "Grace", "5", + NULL) == 0); + + ret = flb_service_set(ctx, + "Log_Level", "info", + "Parsers_File", DPATH "/parsers_multiline_json.conf", + NULL); + TEST_CHECK_(ret == 0, "setting service options"); + + in_ffd = flb_input(ctx, (char *) "tail", NULL); + TEST_CHECK(in_ffd >= 0); + TEST_CHECK(flb_input_set(ctx, in_ffd, "tag", "test", NULL) == 0); + + /* Compose path based on target */ + snprintf(path, sizeof(path) - 1, DPATH "/log/%s.log", target); + TEST_CHECK_(access(path, R_OK) == 0, "accessing log file: %s", path); + + TEST_CHECK(flb_input_set(ctx, in_ffd, + "path" , path, + "read_from_head" , "true", + "multiline.parser", "multiline-json-regex", + NULL) == 0); + + + out_ffd = flb_output(ctx, (char *) "lib", &cb); + TEST_CHECK(out_ffd >= 0); + TEST_CHECK(flb_output_set(ctx, out_ffd, + "match", "test", + "format", "json", + NULL) == 0); + + /* Start test */ + /* Start the engine */ + ret = flb_start(ctx); + TEST_CHECK_(ret == 0, "starting engine"); + + /* Expect 1 final record */ + n_expected = 1; + t_expected = 5000; + + /* Poll for up to 5 seconds or until we got a match */ + for (ret = 0; ret < t_expected && result.nMatched < n_expected; ret++) { + usleep(1000); + } + wait_with_timeout(5000, &result, n_expected); + + TEST_CHECK(result.nMatched == n_expected); + TEST_MSG("result.nMatched: %i\nnExpected: %i", result.nMatched, n_expected); + + ret = flb_stop(ctx); + TEST_CHECK_(ret == 0, "stopping engine"); + + if (ctx) { + flb_destroy(ctx); + } +} + +void flb_test_path_comma() +{ + struct flb_lib_out_cb cb_data; + struct test_tail_ctx *ctx; + char *file[] = {"a.log", "b.log", "c.log", "d.log"}; + char *path = "a.log, b.log, c.log, d.log"; + char *msg = "hello world"; + int ret; + int num; + int unused; + + clear_output_num(); + + cb_data.cb = cb_count_msgpack; + cb_data.data = &unused; + + ctx = test_tail_ctx_create(&cb_data, &file[0], sizeof(file)/sizeof(char*), FLB_TRUE); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("test_ctx_create failed"); + exit(EXIT_FAILURE); + } + + ret = flb_input_set(ctx->flb, ctx->o_ffd, + "path", path, + NULL); + TEST_CHECK(ret == 0); + + ret = flb_output_set(ctx->flb, ctx->o_ffd, + NULL); + TEST_CHECK(ret == 0); + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + ret = write_msg(ctx, msg, strlen(msg)); + if (!TEST_CHECK(ret > 0)) { + test_tail_ctx_destroy(ctx); + exit(EXIT_FAILURE); + } + + /* waiting to flush */ + flb_time_msleep(500); + + num = get_output_num(); + if (!TEST_CHECK(num == sizeof(file)/sizeof(char*))) { + TEST_MSG("output num error. expect=%lu got=%d", sizeof(file)/sizeof(char*), num); + } + + test_tail_ctx_destroy(ctx); +} + +void flb_test_path_key() +{ + struct flb_lib_out_cb cb_data; + struct test_tail_ctx *ctx; + char *file[] = {"path_key.log"}; + char *path_key = "path_key_is"; + char *msg = "hello world"; + int ret; + int num; + + char *expected_strs[] = {path_key, msg, file[0]}; + struct str_list expected = { + .size = sizeof(expected_strs)/sizeof(char*), + .lists = &expected_strs[0], + }; + + clear_output_num(); + + cb_data.cb = cb_check_json_str_list; + cb_data.data = &expected; + + ctx = test_tail_ctx_create(&cb_data, &file[0], sizeof(file)/sizeof(char*), FLB_TRUE); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("test_ctx_create failed"); + exit(EXIT_FAILURE); + } + + ret = flb_input_set(ctx->flb, ctx->o_ffd, + "path", file[0], + "path_key", path_key, + NULL); + TEST_CHECK(ret == 0); + + ret = flb_output_set(ctx->flb, ctx->o_ffd, + "format", "json", + NULL); + TEST_CHECK(ret == 0); + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + ret = write_msg(ctx, msg, strlen(msg)); + if (!TEST_CHECK(ret > 0)) { + test_tail_ctx_destroy(ctx); + exit(EXIT_FAILURE); + } + + /* waiting to flush */ + flb_time_msleep(500); + + num = get_output_num(); + if (!TEST_CHECK(num > 0)) { + TEST_MSG("no outputs"); + } + + test_tail_ctx_destroy(ctx); +} + +void flb_test_exclude_path() +{ + struct flb_lib_out_cb cb_data; + struct test_tail_ctx *ctx; + char *exclude_path = "ep_ignore*.txt"; + char *path = "ep_*.txt"; + char *file[] = {"ep_ignore_1.txt", "ep_ignore_2.txt", "ep_file1.txt", "ep_file2.txt", "ep_file3.txt"}; + char *msg = "hello world"; + int unused; + int ret; + int num; + + clear_output_num(); + + cb_data.cb = cb_count_msgpack; + cb_data.data = &unused; + + ctx = test_tail_ctx_create(&cb_data, &file[0], sizeof(file)/sizeof(char*), FLB_TRUE); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("test_ctx_create failed"); + exit(EXIT_FAILURE); + } + + ret = flb_input_set(ctx->flb, ctx->o_ffd, + "path", path, + "exclude_path", exclude_path, + NULL); + TEST_CHECK(ret == 0); + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + ret = write_msg(ctx, msg, strlen(msg)); + if (!TEST_CHECK(ret > 0)) { + test_tail_ctx_destroy(ctx); + exit(EXIT_FAILURE); + } + + /* waiting to flush */ + flb_time_msleep(500); + + num = get_output_num(); + if (!TEST_CHECK(num == 3 /* 3files. "ep_file1.txt", "ep_file2.txt", "ep_file3.txt" */)) { + TEST_MSG("output num error. expect=3 got=%d", num); + } + + test_tail_ctx_destroy(ctx); +} + +void flb_test_offset_key() +{ + struct flb_lib_out_cb cb_data; + struct test_tail_ctx *ctx; + char *file[] = {"offset_key.log"}; + char *offset_key = "OffsetKey"; + char *msg_before_tail = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; + char *msg_after_tail = "test test"; + char expected_msg[1024] = {0}; + int ret; + int num; + + char *expected_strs[] = {msg_after_tail, &expected_msg[0]}; + struct str_list expected = { + .size = sizeof(expected_strs)/sizeof(char*), + .lists = &expected_strs[0], + }; + + clear_output_num(); + + cb_data.cb = cb_check_json_str_list; + cb_data.data = &expected; + + ret = snprintf(&expected_msg[0], sizeof(expected_msg), "\"%s\":%ld", offset_key, strlen(msg_before_tail)+strlen(NEW_LINE)); + if(!TEST_CHECK(ret >= 0)) { + TEST_MSG("snprintf failed"); + exit(EXIT_FAILURE); + } + + + ctx = test_tail_ctx_create(&cb_data, &file[0], sizeof(file)/sizeof(char *), FLB_TRUE); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("test_ctx_create failed"); + exit(EXIT_FAILURE); + } + + ret = flb_input_set(ctx->flb, ctx->o_ffd, + "path", file[0], + "offset_key", offset_key, + NULL); + TEST_CHECK(ret == 0); + + ret = flb_output_set(ctx->flb, ctx->o_ffd, + "format", "json", + NULL); + TEST_CHECK(ret == 0); + + ret = write_msg(ctx, msg_before_tail, strlen(msg_before_tail)); + if (!TEST_CHECK(ret > 0)) { + test_tail_ctx_destroy(ctx); + exit(EXIT_FAILURE); + } + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + ret = write_msg(ctx, msg_after_tail, strlen(msg_after_tail)); + if (!TEST_CHECK(ret > 0)) { + test_tail_ctx_destroy(ctx); + exit(EXIT_FAILURE); + } + + /* waiting to flush */ + flb_time_msleep(500); + + num = get_output_num(); + if (!TEST_CHECK(num > 0)) { + TEST_MSG("no outputs"); + } + + test_tail_ctx_destroy(ctx); +} + +void flb_test_skip_empty_lines() +{ + struct flb_lib_out_cb cb_data; + struct test_tail_ctx *ctx; + char *file[] = {"skip_empty_lines.log"}; + char *empty_lines[] = {NEW_LINE, NEW_LINE}; + char *msg = "lalala"; + int ret; + int num; + int i; + + char *expected_strs[] = {msg}; + struct str_list expected = { + .size = sizeof(expected_strs)/sizeof(char*), + .lists = &expected_strs[0], + }; + + clear_output_num(); + + cb_data.cb = cb_check_json_str_list; + cb_data.data = &expected; + + ctx = test_tail_ctx_create(&cb_data, &file[0], sizeof(file)/sizeof(char *), FLB_TRUE); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("test_ctx_create failed"); + exit(EXIT_FAILURE); + } + + ret = flb_input_set(ctx->flb, ctx->o_ffd, + "path", file[0], + "skip_empty_lines", "true", + "Read_From_Head", "true", + NULL); + TEST_CHECK(ret == 0); + + ret = flb_output_set(ctx->flb, ctx->o_ffd, + "format", "json", + NULL); + TEST_CHECK(ret == 0); + + ret = write_msg(ctx, msg, strlen(msg)); + if (!TEST_CHECK(ret > 0)) { + test_tail_ctx_destroy(ctx); + exit(EXIT_FAILURE); + } + + for (i=0; i<sizeof(empty_lines)/sizeof(char*); i++) { + ret = write_msg(ctx, empty_lines[i], strlen(empty_lines[i])); + if (!TEST_CHECK(ret > 0)) { + test_tail_ctx_destroy(ctx); + exit(EXIT_FAILURE); + } + } + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + /* waiting to flush */ + flb_time_msleep(500); + + num = get_output_num(); + if (!TEST_CHECK(num == 1)) { + TEST_MSG("output error: expect=1 got=%d", num); + } + + test_tail_ctx_destroy(ctx); +} + +static int ignore_older(int expected, char *ignore_older) +{ + struct flb_lib_out_cb cb_data; + struct test_tail_ctx *ctx; + struct timespec times[2]; + struct flb_time tm; + char *file[] = {"time_now.log", "time_30m.log", "time_3h.log", "time_3d.log"}; + char *path = "time_*.log"; + char *msg = "hello world"; + int ret; + int num; + int unused; + + clear_output_num(); + + cb_data.cb = cb_count_msgpack; + cb_data.data = &unused; + + ctx = test_tail_ctx_create(&cb_data, &file[0], sizeof(file)/sizeof(char *), FLB_TRUE); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("test_ctx_create failed"); + return -1; + } + + times[0].tv_nsec = 0; + times[1].tv_nsec = 0; + + flb_time_get(&tm); + times[0].tv_sec = tm.tm.tv_sec - 3 * 24 * 60 * 60; + times[1].tv_sec = tm.tm.tv_sec - 3 * 24 * 60 * 60; + ret = utimensat(AT_FDCWD, file[3], times, 0); + if (!TEST_CHECK(ret == 0)) { + TEST_MSG("utimensat failed. errno=%d file=%s", errno, file[3]); + return -1; + } + + times[0].tv_sec = tm.tm.tv_sec - 3 * 60 * 60; + times[1].tv_sec = tm.tm.tv_sec - 3 * 60 * 60; + ret = utimensat(AT_FDCWD, file[2], times, 0); + if (!TEST_CHECK(ret == 0)) { + TEST_MSG("utimensat failed. errno=%d file=%s", errno, file[2]); + return -1; + } + + times[0].tv_sec = tm.tm.tv_sec - 30 * 60; + times[1].tv_sec = tm.tm.tv_sec - 30 * 60; + ret = utimensat(AT_FDCWD, file[1], times, 0); + if (!TEST_CHECK(ret == 0)) { + TEST_MSG("utimensat failed. errno=%d file=%s", errno, file[1]); + return -1; + } + + ret = flb_input_set(ctx->flb, ctx->o_ffd, + "path", path, + "ignore_older", ignore_older, + NULL); + TEST_CHECK(ret == 0); + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + ret = write_msg(ctx, msg, strlen(msg)); + if (!TEST_CHECK(ret > 0)) { + test_tail_ctx_destroy(ctx); + return -1; + } + + /* waiting to flush */ + flb_time_msleep(500); + + num = get_output_num(); + if (!TEST_CHECK(num == expected)) { + TEST_MSG("output num error. expect=%d got=%d", expected, num); + return -1; + } + + test_tail_ctx_destroy(ctx); + return 0; +} + +void flb_test_ignore_older() +{ + int ret; + char *ignore_olders[] = {"10m", "40m", "4h", "4d"}; + int expecteds[] = {1/*10m*/, 2/*10m, 40m*/, 3/*10m, 40m, 4h*/, 4 /*all*/}; + int i; + + TEST_CHECK(sizeof(ignore_olders)/sizeof(char*) == sizeof(expecteds)/sizeof(int)); + + for (i=0; i<sizeof(expecteds)/sizeof(int); i++) { + ret = ignore_older(expecteds[i], ignore_olders[i]); + if (!TEST_CHECK(ret == 0)) { + TEST_MSG("case %d failed. ignore_older=%s", i, ignore_olders[i]); + exit(EXIT_FAILURE); + } + } +} + +void flb_test_inotify_watcher_false() +{ + struct flb_lib_out_cb cb_data; + struct test_tail_ctx *ctx; + char *file[] = {"inotify_watcher_false.log"}; + char *msg = "hello world"; + int ret; + int num; + + char *expected_strs[] = {msg}; + struct str_list expected = { + .size = sizeof(expected_strs)/sizeof(char*), + .lists = &expected_strs[0], + }; + + clear_output_num(); + + cb_data.cb = cb_check_json_str_list; + cb_data.data = &expected; + + ctx = test_tail_ctx_create(&cb_data, &file[0], sizeof(file)/sizeof(char *), FLB_TRUE); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("test_ctx_create failed"); + exit(EXIT_FAILURE); + } + + ret = flb_input_set(ctx->flb, ctx->o_ffd, + "path", file[0], + "inotify_watcher", "false", + NULL); + TEST_CHECK(ret == 0); + + ret = flb_output_set(ctx->flb, ctx->o_ffd, + "format", "json", + NULL); + TEST_CHECK(ret == 0); + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + ret = write_msg(ctx, msg, strlen(msg)); + if (!TEST_CHECK(ret > 0)) { + test_tail_ctx_destroy(ctx); + exit(EXIT_FAILURE); + } + + /* waiting to flush */ + flb_time_msleep(1500); + + num = get_output_num(); + if (!TEST_CHECK(num > 0)) { + TEST_MSG("no output"); + } + + test_tail_ctx_destroy(ctx); +} + +#ifdef FLB_HAVE_REGEX +void flb_test_parser() +{ + struct flb_lib_out_cb cb_data; + struct test_tail_ctx *ctx; + char *file[] = {"parser.log"}; + /* https://httpd.apache.org/docs/2.4/en/logs.html */ + char *msg = "127.0.0.1 - frank [10/Oct/2000:13:55:36 -0700] \"GET /apache_pb.gif HTTP/1.0\" 200 2326"; + int ret; + int num; + + char *expected_strs[] = {"\"method\":\"GET\"", "\"host\":\"127.0.0.1\"","\"user\":\"frank\"", + "\"path\":\"/apache_pb.gif\"","\"code\":\"200\"","\"size\":\"2326\""}; + struct str_list expected = { + .size = sizeof(expected_strs)/sizeof(char*), + .lists = &expected_strs[0], + }; + + clear_output_num(); + + cb_data.cb = cb_check_json_str_list; + cb_data.data = &expected; + + ctx = test_tail_ctx_create(&cb_data, &file[0], sizeof(file)/sizeof(char *), FLB_TRUE); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("test_ctx_create failed"); + exit(EXIT_FAILURE); + } + + ret = flb_input_set(ctx->flb, ctx->o_ffd, + "path", file[0], + "parser", "apache2", + NULL); + TEST_CHECK(ret == 0); + + ret = flb_output_set(ctx->flb, ctx->o_ffd, + "format", "json", + NULL); + TEST_CHECK(ret == 0); + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + ret = write_msg(ctx, msg, strlen(msg)); + if (!TEST_CHECK(ret > 0)) { + test_tail_ctx_destroy(ctx); + exit(EXIT_FAILURE); + } + + /* waiting to flush */ + flb_time_msleep(500); + + num = get_output_num(); + if (!TEST_CHECK(num > 0)) { + TEST_MSG("no outputs"); + } + + test_tail_ctx_destroy(ctx); +} + +void flb_test_tag_regex() +{ + struct flb_lib_out_cb cb_data; + struct test_tail_ctx *ctx; + char *file[] = {"aa_bb_cc.log"}; + char *tag_regex = "(?<first>[a-z]+)_(?<second>[a-z]+)_(?<third>[a-z]+)\\.log"; + char *tag = "<first>.<second>.<third>"; /* tag will be "aa.bb.cc" */ + char *msg = "hello world"; + int ret; + int num; + + char *expected_strs[] = {msg}; + struct str_list expected = { + .size = sizeof(expected_strs)/sizeof(char*), + .lists = &expected_strs[0], + }; + + clear_output_num(); + + cb_data.cb = cb_check_json_str_list; + cb_data.data = &expected; + + ctx = test_tail_ctx_create(&cb_data, &file[0], sizeof(file)/sizeof(char *), FLB_TRUE); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("test_ctx_create failed"); + exit(EXIT_FAILURE); + } + + ret = flb_input_set(ctx->flb, ctx->o_ffd, + "path", file[0], + "tag", tag, + "tag_regex", tag_regex, + NULL); + TEST_CHECK(ret == 0); + + ret = flb_output_set(ctx->flb, ctx->o_ffd, + "match", "aa.bb.cc", + "format", "json", + NULL); + TEST_CHECK(ret == 0); + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + ret = write_msg(ctx, msg, strlen(msg)); + if (!TEST_CHECK(ret > 0)) { + test_tail_ctx_destroy(ctx); + exit(EXIT_FAILURE); + } + + /* waiting to flush */ + flb_time_msleep(500); + + num = get_output_num(); + if (!TEST_CHECK(num > 0)) { + TEST_MSG("no outputs"); + } + + test_tail_ctx_destroy(ctx); +} +#endif /* FLB_HAVE_REGEX */ + +#ifdef FLB_HAVE_SQLDB +void flb_test_db() +{ + struct flb_lib_out_cb cb_data; + struct test_tail_ctx *ctx; + char *file[] = {"test_db.log"}; + char *db = "test_db.db"; + char *msg_init = "hello world"; + char *msg = "hello db"; + char *msg_end = "hello db end"; + int i; + int ret; + int num; + int unused; + + unlink(db); + + clear_output_num(); + + cb_data.cb = cb_count_msgpack; + cb_data.data = &unused; + + ctx = test_tail_ctx_create(&cb_data, &file[0], sizeof(file)/sizeof(char *), FLB_FALSE); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("test_ctx_create failed"); + exit(EXIT_FAILURE); + } + + ret = flb_input_set(ctx->flb, ctx->o_ffd, + "path", file[0], + "db", db, + NULL); + TEST_CHECK(ret == 0); + + ret = flb_output_set(ctx->flb, ctx->o_ffd, + NULL); + TEST_CHECK(ret == 0); + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + ret = write_msg(ctx, msg_init, strlen(msg_init)); + if (!TEST_CHECK(ret > 0)) { + test_tail_ctx_destroy(ctx); + unlink(db); + exit(EXIT_FAILURE); + } + + /* waiting to flush */ + flb_time_msleep(500); + + num = get_output_num(); + if (!TEST_CHECK(num > 0)) { + TEST_MSG("no output"); + } + + if (ctx->fds != NULL) { + for (i=0; i<ctx->fd_num; i++) { + close(ctx->fds[i]); + } + flb_free(ctx->fds); + } + flb_stop(ctx->flb); + flb_destroy(ctx->flb); + flb_free(ctx); + + /* re-init to use db */ + clear_output_num(); + + cb_data.cb = cb_count_msgpack; + cb_data.data = &unused; + + ctx = test_tail_ctx_create(&cb_data, &file[0], sizeof(file)/sizeof(char *), FLB_FALSE); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("test_ctx_create failed"); + unlink(db); + exit(EXIT_FAILURE); + } + + ret = flb_input_set(ctx->flb, ctx->o_ffd, + "path", file[0], + "db", db, + "db.sync", "full", + NULL); + TEST_CHECK(ret == 0); + + ret = write_msg(ctx, msg, strlen(msg)); + if (!TEST_CHECK(ret > 0)) { + test_tail_ctx_destroy(ctx); + unlink(db); + exit(EXIT_FAILURE); + } + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + /* waiting to flush */ + flb_time_msleep(500); + + ret = write_msg(ctx, msg_end, strlen(msg_end)); + if (!TEST_CHECK(ret > 0)) { + test_tail_ctx_destroy(ctx); + unlink(db); + exit(EXIT_FAILURE); + } + + /* waiting to flush */ + flb_time_msleep(500); + + num = get_output_num(); + if (!TEST_CHECK(num == 2)) { + /* 2 = msg + msg_end */ + TEST_MSG("num error. expect=2 got=%d", num); + } + + test_tail_ctx_destroy(ctx); + unlink(db); +} +#endif /* FLB_HAVE_SQLDB */ + +/* Test list */ +TEST_LIST = { + {"issue_3943", flb_test_in_tail_issue_3943}, + /* Properties */ + {"skip_long_lines", flb_test_in_tail_skip_long_lines}, + {"path_comma", flb_test_path_comma}, + {"path_key", flb_test_path_key}, + {"exclude_path", flb_test_exclude_path}, + {"offset_key", flb_test_offset_key}, + {"skip_empty_lines", flb_test_skip_empty_lines}, + {"ignore_older", flb_test_ignore_older}, +#ifdef FLB_HAVE_INOTIFY + {"inotify_watcher_false", flb_test_inotify_watcher_false}, +#endif /* FLB_HAVE_INOTIFY */ + +#ifdef FLB_HAVE_REGEX + {"parser", flb_test_parser}, + {"tag_regex", flb_test_tag_regex}, +#endif /* FLB_HAVE_INOTIFY */ + +#ifdef FLB_HAVE_SQLDB + {"db", flb_test_db}, +#endif + +#ifdef in_tail + {"in_tail_dockermode", flb_test_in_tail_dockermode}, + {"in_tail_dockermode_splitted_line", flb_test_in_tail_dockermode_splitted_line}, + {"in_tail_dockermode_multiple_lines", flb_test_in_tail_dockermode_multiple_lines}, + {"in_tail_dockermode_splitted_multiple_lines", flb_test_in_tail_dockermode_splitted_multiple_lines}, + {"in_tail_dockermode_firstline_detection", flb_test_in_tail_dockermode_firstline_detection}, + {"in_tail_multiline_json_and_regex", flb_test_in_tail_multiline_json_and_regex}, +#endif + {NULL, NULL} +}; diff --git a/src/fluent-bit/tests/runtime/in_tcp.c b/src/fluent-bit/tests/runtime/in_tcp.c new file mode 100644 index 000000000..eee20bd64 --- /dev/null +++ b/src/fluent-bit/tests/runtime/in_tcp.c @@ -0,0 +1,561 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2019-2022 The Fluent Bit Authors + * Copyright (C) 2015-2018 Treasure Data Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <fluent-bit.h> +#include <fluent-bit/flb_compat.h> +#include <fluent-bit/flb_time.h> +#include <fluent-bit/flb_pack.h> +#include <fluent-bit/flb_socket.h> +#include <sys/types.h> +#include <sys/stat.h> +#include <fcntl.h> +#include "flb_tests_runtime.h" + +#define DEFAULT_IO_TIMEOUT 10 +#define DEFAULT_HOST "127.0.0.1" +#define DEFAULT_PORT 5170 + +#define TLS_CERTIFICATE_HOSTNAME "leo.vcap.me" +#define TLS_CERTIFICATE_FILENAME FLB_TESTS_DATA_PATH "/data/tls/certificate.pem" +#define TLS_PRIVATE_KEY_FILENAME FLB_TESTS_DATA_PATH "/data/tls/private_key.pem" + +struct test_ctx { + flb_ctx_t *flb; /* Fluent Bit library context */ + int i_ffd; /* Input fd */ + int f_ffd; /* Filter fd (unused) */ + int o_ffd; /* Output fd */ +}; + + +pthread_mutex_t result_mutex = PTHREAD_MUTEX_INITIALIZER; +int num_output = 0; +static int get_output_num() +{ + int ret; + pthread_mutex_lock(&result_mutex); + ret = num_output; + pthread_mutex_unlock(&result_mutex); + + return ret; +} + +static void set_output_num(int num) +{ + pthread_mutex_lock(&result_mutex); + num_output = num; + pthread_mutex_unlock(&result_mutex); +} + +static void clear_output_num() +{ + set_output_num(0); +} + +static int cb_count_msgpack(void *record, size_t size, void *data) +{ + msgpack_unpacked result; + size_t off = 0; + + if (!TEST_CHECK(data != NULL)) { + flb_error("data is NULL"); + } + + /* Iterate each item array and apply rules */ + msgpack_unpacked_init(&result); + while (msgpack_unpack_next(&result, record, size, &off) == MSGPACK_UNPACK_SUCCESS) { + pthread_mutex_lock(&result_mutex); + num_output++; + pthread_mutex_unlock(&result_mutex); + } + msgpack_unpacked_destroy(&result); + + flb_free(record); + return 0; +} + +/* Callback to check expected results */ +static int cb_check_result_json(void *record, size_t size, void *data) +{ + char *p; + char *expected; + char *result; + int num = get_output_num(); + + set_output_num(num+1); + + expected = (char *) data; + result = (char *) record; + + p = strstr(result, expected); + TEST_CHECK(p != NULL); + + if (p==NULL) { + flb_error("Expected to find: '%s' in result '%s'", + expected, result); + } + /* + * If you want to debug your test + * + * printf("Expect: '%s' in result '%s'", expected, result); + */ + flb_free(record); + return 0; +} + +static struct test_ctx *test_ctx_create(struct flb_lib_out_cb *data) +{ + int i_ffd; + int o_ffd; + struct test_ctx *ctx = NULL; + + ctx = flb_malloc(sizeof(struct test_ctx)); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("malloc failed"); + flb_errno(); + return NULL; + } + + /* Service config */ + ctx->flb = flb_create(); + flb_service_set(ctx->flb, + "Flush", "0.200000000", + "Grace", "1", + "Log_Level", "error", + NULL); + + /* Input */ + i_ffd = flb_input(ctx->flb, (char *) "tcp", NULL); + TEST_CHECK(i_ffd >= 0); + ctx->i_ffd = i_ffd; + + /* Output */ + o_ffd = flb_output(ctx->flb, (char *) "lib", (void *) data); + ctx->o_ffd = o_ffd; + + return ctx; +} + +static void test_ctx_destroy(struct test_ctx *ctx) +{ + TEST_CHECK(ctx != NULL); + + sleep(1); + flb_stop(ctx->flb); + flb_destroy(ctx->flb); + flb_free(ctx); +} + +static flb_sockfd_t connect_tcp(char *in_host, int in_port) +{ + int port = in_port; + char *host = in_host; + flb_sockfd_t fd; + int ret; + struct sockaddr_in addr; + + if (host == NULL) { + host = DEFAULT_HOST; + } + if (port < 0) { + port = DEFAULT_PORT; + } + + memset(&addr, 0, sizeof(addr)); + fd = socket(PF_INET, SOCK_STREAM, 0); + if (!TEST_CHECK(fd >= 0)) { + TEST_MSG("failed to socket. host=%s port=%d errno=%d", host, port, errno); + return -1; + } + + addr.sin_family = AF_INET; + addr.sin_addr.s_addr = inet_addr(host); + addr.sin_port = htons(port); + + ret = connect(fd, (const struct sockaddr *)&addr, sizeof(addr)); + if (!TEST_CHECK(ret >= 0)) { + TEST_MSG("failed to connect. host=%s port=%d errno=%d", host, port, errno); + flb_socket_close(fd); + return -1; + } + return fd; +} + +void flb_test_tcp() +{ + struct flb_lib_out_cb cb_data; + struct test_ctx *ctx; + flb_sockfd_t fd; + int ret; + int num; + ssize_t w_size; + + char *buf = "{\"test\":\"msg\"}"; + size_t size = strlen(buf); + + clear_output_num(); + + cb_data.cb = cb_check_result_json; + cb_data.data = "\"test\":\"msg\""; + + ctx = test_ctx_create(&cb_data); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("test_ctx_create failed"); + exit(EXIT_FAILURE); + } + + ret = flb_output_set(ctx->flb, ctx->o_ffd, + "match", "*", + "format", "json", + NULL); + TEST_CHECK(ret == 0); + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + /* use default host/port */ + fd = connect_tcp(NULL, -1); + if (!TEST_CHECK(fd >= 0)) { + exit(EXIT_FAILURE); + } + + w_size = send(fd, buf, size, 0); + if (!TEST_CHECK(w_size == size)) { + TEST_MSG("failed to send, errno=%d", errno); + flb_socket_close(fd); + exit(EXIT_FAILURE); + } + + /* waiting to flush */ + flb_time_msleep(1500); + + num = get_output_num(); + if (!TEST_CHECK(num > 0)) { + TEST_MSG("no outputs"); + } + + flb_socket_close(fd); + test_ctx_destroy(ctx); +} + +void flb_test_tcp_with_tls() +{ + struct flb_connection *client_connection; + struct flb_upstream *upstream; + struct flb_lib_out_cb cb_data; + size_t sent; + struct test_ctx *ctx; + int ret; + int num; + struct flb_tls *tls; + + char *buf = "{\"test\":\"msg\"}"; + size_t size = strlen(buf); + + clear_output_num(); + + cb_data.cb = cb_check_result_json; + cb_data.data = "\"test\":\"msg\""; + + ctx = test_ctx_create(&cb_data); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("test_ctx_create failed"); + exit(EXIT_FAILURE); + } + + ret = flb_input_set(ctx->flb, ctx->i_ffd, + "tls", "on", + "tls.verify", "no", + "tls.vhost", TLS_CERTIFICATE_HOSTNAME, + "tls.crt_file", TLS_CERTIFICATE_FILENAME, + "tls.key_file", TLS_PRIVATE_KEY_FILENAME, + NULL); + TEST_CHECK(ret == 0); + + ret = flb_output_set(ctx->flb, ctx->o_ffd, + "match", "*", + "format", "json", + NULL); + TEST_CHECK(ret == 0); + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + ret = flb_tls_init(); + TEST_CHECK(ret == 0); + + tls = flb_tls_create(FLB_TLS_CLIENT_MODE, + FLB_FALSE, + FLB_TRUE, + TLS_CERTIFICATE_HOSTNAME, + NULL, + NULL, + NULL, + NULL, + NULL); + + TEST_CHECK(tls != NULL); + + upstream = flb_upstream_create(ctx->flb->config, + DEFAULT_HOST, + DEFAULT_PORT, + FLB_IO_TCP | FLB_IO_TLS, + tls); + + TEST_CHECK(upstream != NULL); + + flb_stream_disable_async_mode(&upstream->base); + + upstream->base.net.io_timeout = DEFAULT_IO_TIMEOUT; + + client_connection = flb_upstream_conn_get(upstream); + + TEST_CHECK(client_connection != NULL); + + if (client_connection != NULL) { + ret = flb_io_net_write(client_connection, + (void *) buf, + size, + &sent); + + TEST_CHECK(ret > 0); + + /* waiting to flush */ + flb_time_msleep(1500); + } + + num = get_output_num(); + + if (!TEST_CHECK(num > 0)) { + TEST_MSG("no outputs"); + } + + sleep(1); + + flb_stop(ctx->flb); + flb_upstream_destroy(upstream); + flb_tls_destroy(tls); + flb_destroy(ctx->flb); + flb_free(ctx); +} + +void flb_test_format_none() +{ + struct flb_lib_out_cb cb_data; + struct test_ctx *ctx; + flb_sockfd_t fd; + int ret; + int num; + ssize_t w_size; + + char *buf = "message\n"; + size_t size = strlen(buf); + + clear_output_num(); + + cb_data.cb = cb_check_result_json; + cb_data.data = "\"log\":\"message\""; + + ctx = test_ctx_create(&cb_data); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("test_ctx_create failed"); + exit(EXIT_FAILURE); + } + + ret = flb_output_set(ctx->flb, ctx->o_ffd, + "match", "*", + "format", "json", + NULL); + TEST_CHECK(ret == 0); + + ret = flb_input_set(ctx->flb, ctx->i_ffd, + "format", "none", + NULL); + TEST_CHECK(ret == 0); + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + /* use default host/port */ + fd = connect_tcp(NULL, -1); + if (!TEST_CHECK(fd >= 0)) { + exit(EXIT_FAILURE); + } + + w_size = send(fd, buf, size, 0); + if (!TEST_CHECK(w_size == size)) { + TEST_MSG("failed to send, errno=%d", errno); + flb_socket_close(fd); + exit(EXIT_FAILURE); + } + + /* waiting to flush */ + flb_time_msleep(1500); + + num = get_output_num(); + if (!TEST_CHECK(num > 0)) { + TEST_MSG("no outputs"); + } + + flb_socket_close(fd); + test_ctx_destroy(ctx); +} + +void flb_test_format_none_separator() +{ + struct flb_lib_out_cb cb_data; + struct test_ctx *ctx; + flb_sockfd_t fd; + int ret; + int num; + ssize_t w_size; + + char *buf = "message:message:"; + size_t size = strlen(buf); + + clear_output_num(); + + cb_data.cb = cb_check_result_json; + cb_data.data = "\"log\":\"message\""; + + ctx = test_ctx_create(&cb_data); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("test_ctx_create failed"); + exit(EXIT_FAILURE); + } + + ret = flb_output_set(ctx->flb, ctx->o_ffd, + "match", "*", + "format", "json", + NULL); + TEST_CHECK(ret == 0); + + ret = flb_input_set(ctx->flb, ctx->i_ffd, + "format", "none", + "separator", ":", + NULL); + TEST_CHECK(ret == 0); + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + /* use default host/port */ + fd = connect_tcp(NULL, -1); + if (!TEST_CHECK(fd >= 0)) { + exit(EXIT_FAILURE); + } + + w_size = send(fd, buf, size, 0); + if (!TEST_CHECK(w_size == size)) { + TEST_MSG("failed to send, errno=%d", errno); + flb_socket_close(fd); + exit(EXIT_FAILURE); + } + + /* waiting to flush */ + flb_time_msleep(1500); + + num = get_output_num(); + if (!TEST_CHECK(num == 2)) { + TEST_MSG("no outputs"); + } + + flb_socket_close(fd); + test_ctx_destroy(ctx); +} + +/* + * Ingest 64k records. + * https://github.com/fluent/fluent-bit/issues/5336 + */ +void flb_test_issue_5336() +{ + struct flb_lib_out_cb cb_data; + struct test_ctx *ctx; + flb_sockfd_t fd; + int ret; + int num; + ssize_t w_size; + int not_used; + int i; + int count = 65535; + + char *buf = "{\"test\":\"msg\"}"; + size_t size = strlen(buf); + + clear_output_num(); + + cb_data.cb = cb_count_msgpack; + cb_data.data = ¬_used; + + ctx = test_ctx_create(&cb_data); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("test_ctx_create failed"); + exit(EXIT_FAILURE); + } + + ret = flb_output_set(ctx->flb, ctx->o_ffd, + "match", "*", + NULL); + TEST_CHECK(ret == 0); + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + /* use default host/port */ + fd = connect_tcp(NULL, -1); + if (!TEST_CHECK(fd >= 0)) { + exit(EXIT_FAILURE); + } + + for (i=0; i<count; i++) { + w_size = send(fd, buf, size, 0); + if (!TEST_CHECK(w_size == size)) { + TEST_MSG("failed to send, count=%d errno=%d",i, errno); + flb_socket_close(fd); + exit(EXIT_FAILURE); + } + } + + /* waiting to flush */ + flb_time_msleep(2500); + + num = get_output_num(); + if (!TEST_CHECK(num == count)) { + TEST_MSG("got %d, expected: %d", num, count); + } + + flb_socket_close(fd); + test_ctx_destroy(ctx); +} + +TEST_LIST = { + {"tcp", flb_test_tcp}, + {"tcp_with_tls", flb_test_tcp_with_tls}, + {"format_none", flb_test_format_none}, + {"format_none_separator", flb_test_format_none_separator}, + {"65535_records_issue_5336", flb_test_issue_5336}, + {NULL, NULL} +}; + diff --git a/src/fluent-bit/tests/runtime/in_udp.c b/src/fluent-bit/tests/runtime/in_udp.c new file mode 100644 index 000000000..df60c923f --- /dev/null +++ b/src/fluent-bit/tests/runtime/in_udp.c @@ -0,0 +1,440 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2019-2022 The Fluent Bit Authors + * Copyright (C) 2015-2018 Treasure Data Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <fluent-bit.h> +#include <fluent-bit/flb_compat.h> +#include <fluent-bit/flb_time.h> +#include <fluent-bit/flb_pack.h> +#include <fluent-bit/flb_socket.h> +#include <sys/types.h> +#include <sys/stat.h> +#include <fcntl.h> +#include "flb_tests_runtime.h" + +#define DEFAULT_IO_TIMEOUT 10 +#define DEFAULT_HOST "127.0.0.1" +#define DEFAULT_PORT 5180 + +#define TLS_CERTIFICATE_HOSTNAME "leo.vcap.me" +#define TLS_CERTIFICATE_FILENAME FLB_TESTS_DATA_PATH "/data/tls/certificate.pem" +#define TLS_PRIVATE_KEY_FILENAME FLB_TESTS_DATA_PATH "/data/tls/private_key.pem" + +struct test_ctx { + flb_ctx_t *flb; /* Fluent Bit library context */ + int i_ffd; /* Input fd */ + int f_ffd; /* Filter fd (unused) */ + int o_ffd; /* Output fd */ +}; + +pthread_mutex_t result_mutex = PTHREAD_MUTEX_INITIALIZER; +int num_output = 0; +static int get_output_num() +{ + int ret; + pthread_mutex_lock(&result_mutex); + ret = num_output; + pthread_mutex_unlock(&result_mutex); + + return ret; +} + +static void set_output_num(int num) +{ + pthread_mutex_lock(&result_mutex); + num_output = num; + pthread_mutex_unlock(&result_mutex); +} + +static void clear_output_num() +{ + set_output_num(0); +} + +/* Callback to check expected results */ +static int cb_check_result_json(void *record, size_t size, void *data) +{ + char *p; + char *expected; + char *result; + int num = get_output_num(); + + set_output_num(num+1); + + expected = (char *) data; + result = (char *) record; + + p = strstr(result, expected); + TEST_CHECK(p != NULL); + + if (p==NULL) { + flb_error("Expected to find: '%s' in result '%s'", + expected, result); + } + /* + * If you want to debug your test + * + * printf("Expect: '%s' in result '%s'", expected, result); + */ + flb_free(record); + return 0; +} + +static struct test_ctx *test_ctx_create(struct flb_lib_out_cb *data) +{ + int ret; + int i_ffd; + int o_ffd; + struct test_ctx *ctx = NULL; + char sport[8]; + + ctx = flb_malloc(sizeof(struct test_ctx)); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("malloc failed"); + flb_errno(); + return NULL; + } + + /* Service config */ + ctx->flb = flb_create(); + flb_service_set(ctx->flb, + "Flush", "0.200000000", + "Grace", "1", + "Log_Level", "error", + NULL); + + /* Input */ + i_ffd = flb_input(ctx->flb, (char *) "udp", NULL); + TEST_CHECK(i_ffd >= 0); + ctx->i_ffd = i_ffd; + + /* Avoid to port collisions for default port of in_tcp */ + snprintf(sport, 8, "%d", DEFAULT_PORT); + ret = flb_input_set(ctx->flb, ctx->i_ffd, + "port", sport, + NULL); + TEST_CHECK(ret == 0); + + /* Output */ + o_ffd = flb_output(ctx->flb, (char *) "lib", (void *) data); + ctx->o_ffd = o_ffd; + + return ctx; +} + +static void test_ctx_destroy(struct test_ctx *ctx) +{ + TEST_CHECK(ctx != NULL); + + sleep(1); + flb_stop(ctx->flb); + flb_destroy(ctx->flb); + flb_free(ctx); +} + +static int init_udp(char *in_host, int in_port, struct sockaddr_in *addr) +{ + int port = in_port; + char *host = in_host; + flb_sockfd_t fd; + + if (host == NULL) { + host = DEFAULT_HOST; + } + if (port < 0) { + port = DEFAULT_PORT; + } + + memset(addr, 0, sizeof(struct sockaddr_in)); + fd = socket(AF_INET, SOCK_DGRAM, 0); + if (!TEST_CHECK(fd >= 0)) { + TEST_MSG("failed to socket. host=%s port=%d errno=%d", host, port, errno); + return -1; + } + + addr->sin_family = AF_INET; + addr->sin_addr.s_addr = inet_addr(host); + addr->sin_port = htons(port); + + return fd; +} + +void flb_test_udp() +{ + struct flb_lib_out_cb cb_data; + struct test_ctx *ctx; + struct sockaddr_in addr; + flb_sockfd_t fd; + int ret; + int num; + ssize_t w_size; + + char *buf = "{\"test\":\"msg\"}"; + size_t size = strlen(buf); + + clear_output_num(); + + cb_data.cb = cb_check_result_json; + cb_data.data = "\"test\":\"msg\""; + + ctx = test_ctx_create(&cb_data); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("test_ctx_create failed"); + exit(EXIT_FAILURE); + } + + ret = flb_output_set(ctx->flb, ctx->o_ffd, + "match", "*", + "format", "json", + NULL); + TEST_CHECK(ret == 0); + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + /* use default host/port */ + fd = init_udp(NULL, -1, &addr); + if (!TEST_CHECK(fd >= 0)) { + exit(EXIT_FAILURE); + } + + w_size = sendto(fd, buf, size, 0, (const struct sockaddr *)&addr, sizeof(addr)); + if (!TEST_CHECK(w_size == size)) { + TEST_MSG("failed to send, errno=%d", errno); + flb_socket_close(fd); + test_ctx_destroy(ctx); + exit(EXIT_FAILURE); + } + + /* waiting to flush */ + flb_time_msleep(1500); + + num = get_output_num(); + if (!TEST_CHECK(num > 0)) { + TEST_MSG("no outputs"); + } + + flb_socket_close(fd); + test_ctx_destroy(ctx); +} + +void flb_test_udp_with_source_address() +{ + struct flb_lib_out_cb cb_data; + struct test_ctx *ctx; + struct sockaddr_in addr; + flb_sockfd_t fd; + int ret; + int num; + ssize_t w_size; + + char *buf = "{\"test\":\"msg\"}"; + size_t size = strlen(buf); + + clear_output_num(); + + cb_data.cb = cb_check_result_json; + cb_data.data = "\"test\":\"msg\",\"source_host\":\"udp://"; + + ctx = test_ctx_create(&cb_data); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("test_ctx_create failed"); + exit(EXIT_FAILURE); + } + + ret = flb_input_set(ctx->flb, ctx->i_ffd, + "source_address_key", "source_host", + NULL); + TEST_CHECK(ret == 0); + + ret = flb_output_set(ctx->flb, ctx->o_ffd, + "match", "*", + "format", "json", + NULL); + TEST_CHECK(ret == 0); + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + /* use default host/port */ + fd = init_udp(NULL, -1, &addr); + if (!TEST_CHECK(fd >= 0)) { + exit(EXIT_FAILURE); + } + + w_size = sendto(fd, buf, size, 0, (const struct sockaddr *)&addr, sizeof(addr)); + if (!TEST_CHECK(w_size == size)) { + TEST_MSG("failed to send, errno=%d", errno); + flb_socket_close(fd); + test_ctx_destroy(ctx); + exit(EXIT_FAILURE); + } + + /* waiting to flush */ + flb_time_msleep(1500); + + num = get_output_num(); + if (!TEST_CHECK(num > 0)) { + TEST_MSG("no outputs"); + } + + flb_socket_close(fd); + test_ctx_destroy(ctx); +} + +void flb_test_format_none() +{ + struct flb_lib_out_cb cb_data; + struct test_ctx *ctx; + struct sockaddr_in addr; + flb_sockfd_t fd; + int ret; + int num; + ssize_t w_size; + + char *buf = "message\n"; + size_t size = strlen(buf); + + clear_output_num(); + + cb_data.cb = cb_check_result_json; + cb_data.data = "\"log\":\"message\""; + + ctx = test_ctx_create(&cb_data); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("test_ctx_create failed"); + exit(EXIT_FAILURE); + } + + ret = flb_output_set(ctx->flb, ctx->o_ffd, + "match", "*", + "format", "json", + NULL); + TEST_CHECK(ret == 0); + + ret = flb_input_set(ctx->flb, ctx->i_ffd, + "format", "none", + NULL); + TEST_CHECK(ret == 0); + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + /* use default host/port */ + fd = init_udp(NULL, -1, &addr); + if (!TEST_CHECK(fd >= 0)) { + exit(EXIT_FAILURE); + } + + w_size = sendto(fd, buf, size, 0, (const struct sockaddr *)&addr, sizeof(addr)); + if (!TEST_CHECK(w_size == size)) { + TEST_MSG("failed to send, errno=%d", errno); + flb_socket_close(fd); + test_ctx_destroy(ctx); + exit(EXIT_FAILURE); + } + + /* waiting to flush */ + flb_time_msleep(1500); + + num = get_output_num(); + if (!TEST_CHECK(num > 0)) { + TEST_MSG("no outputs"); + } + + flb_socket_close(fd); + test_ctx_destroy(ctx); +} + +void flb_test_format_none_separator() +{ + struct flb_lib_out_cb cb_data; + struct test_ctx *ctx; + struct sockaddr_in addr; + flb_sockfd_t fd; + int ret; + int num; + ssize_t w_size; + + char *buf = "message:message:"; + size_t size = strlen(buf); + + clear_output_num(); + + cb_data.cb = cb_check_result_json; + cb_data.data = "\"log\":\"message\""; + + ctx = test_ctx_create(&cb_data); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("test_ctx_create failed"); + exit(EXIT_FAILURE); + } + + ret = flb_output_set(ctx->flb, ctx->o_ffd, + "match", "*", + "format", "json", + NULL); + TEST_CHECK(ret == 0); + + ret = flb_input_set(ctx->flb, ctx->i_ffd, + "format", "none", + "separator", ":", + NULL); + TEST_CHECK(ret == 0); + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + /* use default host/port */ + fd = init_udp(NULL, -1, &addr); + if (!TEST_CHECK(fd >= 0)) { + exit(EXIT_FAILURE); + } + + w_size = sendto(fd, buf, size, 0, (const struct sockaddr *)&addr, sizeof(addr)); + if (!TEST_CHECK(w_size == size)) { + TEST_MSG("failed to send, errno=%d", errno); + flb_socket_close(fd); + test_ctx_destroy(ctx); + exit(EXIT_FAILURE); + } + + /* waiting to flush */ + flb_time_msleep(1500); + + num = get_output_num(); + if (!TEST_CHECK(num == 2)) { + TEST_MSG("no outputs"); + } + + flb_socket_close(fd); + test_ctx_destroy(ctx); +} + +TEST_LIST = { + {"udp", flb_test_udp}, + {"udp_with_source_address", flb_test_udp_with_source_address}, + {"format_none", flb_test_format_none}, + {"format_none_separator", flb_test_format_none_separator}, + {NULL, NULL} +}; diff --git a/src/fluent-bit/tests/runtime/out_cloudwatch.c b/src/fluent-bit/tests/runtime/out_cloudwatch.c new file mode 100644 index 000000000..fc54e4c38 --- /dev/null +++ b/src/fluent-bit/tests/runtime/out_cloudwatch.c @@ -0,0 +1,362 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ +#include <fluent-bit.h> +#include "flb_tests_runtime.h" + +/* Test data */ +#include "data/td/json_td.h" /* JSON_TD */ + +#define ERROR_ALREADY_EXISTS "{\"__type\":\"ResourceAlreadyExistsException\"}" +/* not a real error code, but tests that the code can respond to any error */ +#define ERROR_UNKNOWN "{\"__type\":\"UNKNOWN\"}" + +/* It writes a big JSON message (copied from TD test) */ +void flb_test_cloudwatch_success(void) +{ + int ret; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + /* mocks calls- signals that we are in test mode */ + setenv("FLB_CLOUDWATCH_PLUGIN_UNDER_TEST", "true", 1); + + ctx = flb_create(); + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx,in_ffd, "tag", "test", NULL); + + out_ffd = flb_output(ctx, (char *) "cloudwatch_logs", NULL); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd,"match", "test", NULL); + flb_output_set(ctx, out_ffd,"region", "us-west-2", NULL); + flb_output_set(ctx, out_ffd,"log_group_name", "fluent", NULL); + flb_output_set(ctx, out_ffd,"log_stream_prefix", "from-fluent-", NULL); + flb_output_set(ctx, out_ffd,"auto_create_group", "On", NULL); + flb_output_set(ctx, out_ffd,"net.keepalive", "Off", NULL); + flb_output_set(ctx, out_ffd,"Retry_Limit", "1", NULL); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + flb_lib_push(ctx, in_ffd, (char *) JSON_TD , (int) sizeof(JSON_TD) - 1); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_cloudwatch_already_exists_create_group(void) +{ + int ret; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + /* mocks calls- signals that we are in test mode */ + setenv("FLB_CLOUDWATCH_PLUGIN_UNDER_TEST", "true", 1); + setenv("TEST_CREATE_LOG_GROUP_ERROR", ERROR_ALREADY_EXISTS, 1); + + ctx = flb_create(); + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx,in_ffd, "tag", "test", NULL); + + out_ffd = flb_output(ctx, (char *) "cloudwatch_logs", NULL); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd,"match", "test", NULL); + flb_output_set(ctx, out_ffd,"region", "us-west-2", NULL); + flb_output_set(ctx, out_ffd,"log_group_name", "fluent", NULL); + flb_output_set(ctx, out_ffd,"log_stream_prefix", "from-fluent-", NULL); + flb_output_set(ctx, out_ffd,"auto_create_group", "On", NULL); + flb_output_set(ctx, out_ffd,"net.keepalive", "Off", NULL); + flb_output_set(ctx, out_ffd,"Retry_Limit", "1", NULL); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + flb_lib_push(ctx, in_ffd, (char *) JSON_TD , (int) sizeof(JSON_TD) - 1); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_cloudwatch_already_exists_create_stream(void) +{ + int ret; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + /* mocks calls- signals that we are in test mode */ + setenv("FLB_CLOUDWATCH_PLUGIN_UNDER_TEST", "true", 1); + setenv("TEST_CREATE_LOG_STREAM_ERROR", ERROR_ALREADY_EXISTS, 1); + + ctx = flb_create(); + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx,in_ffd, "tag", "test", NULL); + + out_ffd = flb_output(ctx, (char *) "cloudwatch_logs", NULL); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd,"match", "test", NULL); + flb_output_set(ctx, out_ffd,"region", "us-west-2", NULL); + flb_output_set(ctx, out_ffd,"log_group_name", "fluent", NULL); + flb_output_set(ctx, out_ffd,"log_stream_prefix", "from-fluent-", NULL); + flb_output_set(ctx, out_ffd,"auto_create_group", "On", NULL); + flb_output_set(ctx, out_ffd,"net.keepalive", "Off", NULL); + flb_output_set(ctx, out_ffd,"Retry_Limit", "1", NULL); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + flb_lib_push(ctx, in_ffd, (char *) JSON_TD , (int) sizeof(JSON_TD) - 1); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_cloudwatch_error_create_group(void) +{ + int ret; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + /* mocks calls- signals that we are in test mode */ + setenv("FLB_CLOUDWATCH_PLUGIN_UNDER_TEST", "true", 1); + setenv("TEST_CREATE_LOG_GROUP_ERROR", ERROR_UNKNOWN, 1); + + ctx = flb_create(); + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx,in_ffd, "tag", "test", NULL); + + out_ffd = flb_output(ctx, (char *) "cloudwatch_logs", NULL); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd,"match", "test", NULL); + flb_output_set(ctx, out_ffd,"region", "us-west-2", NULL); + flb_output_set(ctx, out_ffd,"log_group_name", "fluent", NULL); + flb_output_set(ctx, out_ffd,"log_stream_prefix", "from-fluent-", NULL); + flb_output_set(ctx, out_ffd,"auto_create_group", "On", NULL); + flb_output_set(ctx, out_ffd,"net.keepalive", "Off", NULL); + flb_output_set(ctx, out_ffd,"Retry_Limit", "1", NULL); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + flb_lib_push(ctx, in_ffd, (char *) JSON_TD , (int) sizeof(JSON_TD) - 1); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_cloudwatch_error_create_stream(void) +{ + int ret; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + /* mocks calls- signals that we are in test mode */ + setenv("FLB_CLOUDWATCH_PLUGIN_UNDER_TEST", "true", 1); + setenv("TEST_CREATE_LOG_STREAM_ERROR", ERROR_UNKNOWN, 1); + + ctx = flb_create(); + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx,in_ffd, "tag", "test", NULL); + + out_ffd = flb_output(ctx, (char *) "cloudwatch_logs", NULL); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd,"match", "test", NULL); + flb_output_set(ctx, out_ffd,"region", "us-west-2", NULL); + flb_output_set(ctx, out_ffd,"log_group_name", "fluent", NULL); + flb_output_set(ctx, out_ffd,"log_stream_prefix", "from-fluent-", NULL); + flb_output_set(ctx, out_ffd,"auto_create_group", "On", NULL); + flb_output_set(ctx, out_ffd,"net.keepalive", "Off", NULL); + flb_output_set(ctx, out_ffd,"Retry_Limit", "1", NULL); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + flb_lib_push(ctx, in_ffd, (char *) JSON_TD , (int) sizeof(JSON_TD) - 1); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_cloudwatch_error_put_log_events(void) +{ + int ret; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + /* mocks calls- signals that we are in test mode */ + setenv("FLB_CLOUDWATCH_PLUGIN_UNDER_TEST", "true", 1); + setenv("TEST_PUT_LOG_EVENTS_ERROR", ERROR_UNKNOWN, 1); + + ctx = flb_create(); + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx,in_ffd, "tag", "test", NULL); + + out_ffd = flb_output(ctx, (char *) "cloudwatch_logs", NULL); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd,"match", "test", NULL); + flb_output_set(ctx, out_ffd,"region", "us-west-2", NULL); + flb_output_set(ctx, out_ffd,"log_group_name", "fluent", NULL); + flb_output_set(ctx, out_ffd,"log_stream_prefix", "from-fluent-", NULL); + flb_output_set(ctx, out_ffd,"auto_create_group", "On", NULL); + flb_output_set(ctx, out_ffd,"net.keepalive", "Off", NULL); + flb_output_set(ctx, out_ffd,"Retry_Limit", "1", NULL); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + flb_lib_push(ctx, in_ffd, (char *) JSON_TD , (int) sizeof(JSON_TD) - 1); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_cloudwatch_put_retention_policy_success(void) +{ + int ret; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + /* mocks calls- signals that we are in test mode */ + setenv("FLB_CLOUDWATCH_PLUGIN_UNDER_TEST", "true", 1); + + ctx = flb_create(); + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx,in_ffd, "tag", "test", NULL); + + out_ffd = flb_output(ctx, (char *) "cloudwatch_logs", NULL); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd,"match", "test", NULL); + flb_output_set(ctx, out_ffd,"region", "us-west-2", NULL); + flb_output_set(ctx, out_ffd,"log_group_name", "fluent", NULL); + flb_output_set(ctx, out_ffd,"log_stream_prefix", "from-fluent-", NULL); + flb_output_set(ctx, out_ffd,"auto_create_group", "On", NULL); + flb_output_set(ctx, out_ffd,"log_retention_days", "14", NULL); + flb_output_set(ctx, out_ffd,"net.keepalive", "Off", NULL); + flb_output_set(ctx, out_ffd,"Retry_Limit", "1", NULL); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + flb_lib_push(ctx, in_ffd, (char *) JSON_TD , (int) sizeof(JSON_TD) - 1); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_cloudwatch_already_exists_create_group_put_retention_policy(void) +{ + int ret; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + /* mocks calls- signals that we are in test mode */ + setenv("FLB_CLOUDWATCH_PLUGIN_UNDER_TEST", "true", 1); + setenv("TEST_CREATE_LOG_GROUP_ERROR", ERROR_ALREADY_EXISTS, 1); + + /* PutRetentionPolicy is not called if the group already exists */ + setenv("TEST_PUT_RETENTION_POLICY_ERROR", ERROR_UNKNOWN, 1); + + ctx = flb_create(); + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx,in_ffd, "tag", "test", NULL); + + out_ffd = flb_output(ctx, (char *) "cloudwatch_logs", NULL); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd,"match", "test", NULL); + flb_output_set(ctx, out_ffd,"region", "us-west-2", NULL); + flb_output_set(ctx, out_ffd,"log_group_name", "fluent", NULL); + flb_output_set(ctx, out_ffd,"log_stream_prefix", "from-fluent-", NULL); + flb_output_set(ctx, out_ffd,"auto_create_group", "On", NULL); + flb_output_set(ctx, out_ffd,"log_retention_days", "14", NULL); + flb_output_set(ctx, out_ffd,"net.keepalive", "Off", NULL); + flb_output_set(ctx, out_ffd,"Retry_Limit", "1", NULL); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + flb_lib_push(ctx, in_ffd, (char *) JSON_TD , (int) sizeof(JSON_TD) - 1); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_cloudwatch_error_put_retention_policy(void) +{ + int ret; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + /* mocks calls- signals that we are in test mode */ + setenv("FLB_CLOUDWATCH_PLUGIN_UNDER_TEST", "true", 1); + setenv("TEST_PUT_RETENTION_POLICY_ERROR", ERROR_UNKNOWN, 1); + + ctx = flb_create(); + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx,in_ffd, "tag", "test", NULL); + + out_ffd = flb_output(ctx, (char *) "cloudwatch_logs", NULL); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd,"match", "test", NULL); + flb_output_set(ctx, out_ffd,"region", "us-west-2", NULL); + flb_output_set(ctx, out_ffd,"log_group_name", "fluent", NULL); + flb_output_set(ctx, out_ffd,"log_stream_prefix", "from-fluent-", NULL); + flb_output_set(ctx, out_ffd,"auto_create_group", "On", NULL); + flb_output_set(ctx, out_ffd,"log_retention_days", "14", NULL); + flb_output_set(ctx, out_ffd,"net.keepalive", "Off", NULL); + flb_output_set(ctx, out_ffd,"Retry_Limit", "1", NULL); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + flb_lib_push(ctx, in_ffd, (char *) JSON_TD , (int) sizeof(JSON_TD) - 1); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + +/* Test list */ +TEST_LIST = { + {"success", flb_test_cloudwatch_success }, + {"group_already_exists", flb_test_cloudwatch_already_exists_create_group }, + {"stream_already_exists", flb_test_cloudwatch_already_exists_create_stream }, + {"create_group_error", flb_test_cloudwatch_error_create_group }, + {"create_stream_error", flb_test_cloudwatch_error_create_stream }, + {"put_log_events_error", flb_test_cloudwatch_error_put_log_events }, + {"put_retention_policy_success", flb_test_cloudwatch_put_retention_policy_success }, + {"already_exists_create_group_put_retention_policy", flb_test_cloudwatch_already_exists_create_group_put_retention_policy }, + {"error_put_retention_policy", flb_test_cloudwatch_error_put_retention_policy }, + {NULL, NULL} +}; diff --git a/src/fluent-bit/tests/runtime/out_counter.c b/src/fluent-bit/tests/runtime/out_counter.c new file mode 100644 index 000000000..525b78f47 --- /dev/null +++ b/src/fluent-bit/tests/runtime/out_counter.c @@ -0,0 +1,126 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +#include <fluent-bit.h> +#include "flb_tests_runtime.h" + +/* Test data */ +#include "data/common/json_invalid.h" /* JSON_INVALID */ +#include "data/common/json_long.h" /* JSON_LONG */ +#include "data/common/json_small.h" /* JSON_SMALL */ + +/* Test functions */ +void flb_test_counter_json_invalid(void); +void flb_test_counter_json_long(void); +void flb_test_counter_json_small(void); + +/* Test list */ +TEST_LIST = { + {"json_invalid", flb_test_counter_json_invalid }, + {"json_long", flb_test_counter_json_long }, + {"json_small", flb_test_counter_json_small }, + {NULL, NULL} +}; + +void flb_test_counter_json_invalid(void) +{ + int i; + int ret; + int bytes; + char *p = (char *) JSON_INVALID; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + ctx = flb_create(); + flb_service_set(ctx, "Flush", "1", "Grace", "1", "Log_Level", "error", NULL); + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + out_ffd = flb_output(ctx, (char *) "counter", NULL); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, "match", "test", NULL); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + for (i = 0; i < (int) sizeof(JSON_INVALID) - 1; i++) { + bytes = flb_lib_push(ctx, in_ffd, p + i, 1); + TEST_CHECK(bytes == 1); + } + + flb_stop(ctx); + flb_destroy(ctx); +} + +/* It writes a very long JSON map (> 100KB) byte by byte */ +void flb_test_counter_json_long(void) +{ + int i; + int ret; + int bytes; + char *p = (char *) JSON_LONG; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + ctx = flb_create(); + flb_service_set(ctx, "Flush", "1", "Grace", "1", "Log_Level", "error", NULL); + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + out_ffd = flb_output(ctx, (char *) "counter", NULL); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, "match", "test", NULL); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + for (i = 0; i < (int) sizeof(JSON_LONG) - 1; i++) { + bytes = flb_lib_push(ctx, in_ffd, p + i, 1); + TEST_CHECK(bytes == 1); + } + + sleep(1); /* waiting flush */ + + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_counter_json_small(void) +{ + int i; + int ret; + int bytes; + char *p = (char *) JSON_SMALL; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + ctx = flb_create(); + flb_service_set(ctx, "Flush", "1", "Grace", "1", "Log_Level", "error", NULL); + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + out_ffd = flb_output(ctx, (char *) "counter", NULL); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, "match", "test", NULL); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + for (i = 0; i < (int) sizeof(JSON_SMALL) - 1; i++) { + bytes = flb_lib_push(ctx, in_ffd, p + i, 1); + TEST_CHECK(bytes == 1); + } + + sleep(1); /* waiting flush */ + + flb_stop(ctx); + flb_destroy(ctx); +} diff --git a/src/fluent-bit/tests/runtime/out_datadog.c b/src/fluent-bit/tests/runtime/out_datadog.c new file mode 100644 index 000000000..26cd81fb4 --- /dev/null +++ b/src/fluent-bit/tests/runtime/out_datadog.c @@ -0,0 +1,173 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2019-2020 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <fluent-bit.h> +#include <fluent-bit/flb_pack.h> +#include <fluent-bit/flb_record_accessor.h> +#include <fluent-bit/flb_ra_key.h> + +#include "flb_tests_runtime.h" + +/* JSON payload example */ +#include "data/datadog/json.h" + +/* + * Fluent Bit Datadog plugin, always set as payload a JSON strings contained in a + * 'sds'. Since we want to validate specific keys and it values we expose here some + * helper functions to make testing easier. + * + * The approach is: + * + * - Convert formatter JSON to msgpack + * - use the helper function to check keys and values + * + * it returns FLB_TRUE if expected 'key/val' matches or FLB_FALSE if 'key' no exists + * or if there is a mismatch. + */ +static int mp_kv_cmp(char *json_data, size_t json_len, char *key_accessor, char *val) +{ + int ret; + int type; + char *mp_buf = NULL; + size_t mp_size; + size_t off = 0; + msgpack_object map; + msgpack_unpacked result; + struct flb_ra_value *rval = NULL; + struct flb_record_accessor *ra = NULL; + + /* Convert JSON to msgpack */ + ret = flb_pack_json((const char *) json_data, json_len, &mp_buf, &mp_size, + &type, NULL); + TEST_CHECK(ret != -1); + + /* Set return status */ + ret = FLB_FALSE; + + /* Unpack msgpack and reference the main 'map' */ + msgpack_unpacked_init(&result); + ret = msgpack_unpack_next(&result, mp_buf, mp_size, &off); + TEST_CHECK(ret == MSGPACK_UNPACK_SUCCESS); + map = result.data; + + /* Create a record_accessor context */ + ra = flb_ra_create(key_accessor, FLB_TRUE); + if (!ra) { + flb_error("invalid record accessor key, aborting test"); + goto out; + } + + rval = flb_ra_get_value_object(ra, map); + TEST_CHECK(rval != NULL); + msgpack_unpacked_destroy(&result); + if (!rval) { + goto out; + } + + /* We only validate strings, feel free to expand it as needed */ + TEST_CHECK(rval->type == FLB_RA_STRING); + if (strcmp(rval->val.string, val) == 0) { + ret = FLB_TRUE; + } + + out: + if (rval) { + flb_ra_key_value_destroy(rval); + } + if (ra) { + flb_ra_destroy(ra); + } + if (mp_buf) { + flb_free(mp_buf); + } + return ret; +} + +static void cb_check_dd_properties(void *ctx, int ffd, + int res_ret, void *res_data, size_t res_size, + void *data) +{ + int ret; + + /* source */ + ret = mp_kv_cmp((char *) res_data + 1, res_size - 2, "$ddsource", "my-app"); + TEST_CHECK(ret == FLB_TRUE); + + /* service */ + ret = mp_kv_cmp((char *) res_data + 1, res_size - 2, "$service", "my-app-service"); + TEST_CHECK(ret == FLB_TRUE); + + /* ddtags */ + ret = mp_kv_cmp((char *) res_data + 1, res_size - 2, "$ddtags", "project:dev"); + TEST_CHECK(ret == FLB_TRUE); + + flb_sds_destroy(res_data); +} + +void flb_test_dd_properties() +{ + int ret; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + + /* Lib input mode */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + /* Datadog output */ + out_ffd = flb_output(ctx, (char *) "datadog", NULL); + flb_output_set(ctx, out_ffd, + "match", "test", + "provider", "ecs", + "dd_service", "my-app-service", + "dd_source", "my-app", + "dd_tags", "project:dev", + "apikey", "abc", + NULL); + + /* Enable test mode */ + ret = flb_output_set_test(ctx, out_ffd, "formatter", + cb_check_dd_properties, + NULL, NULL); + + /* Start */ + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + sleep(1); + + /* Ingest data sample */ + flb_lib_push(ctx, in_ffd, (char *) JSON, sizeof(JSON) - 1); + + sleep(2); + + flb_stop(ctx); + flb_destroy(ctx); +} + +/* Test list */ +TEST_LIST = { + { "dd_properties", flb_test_dd_properties }, + { NULL, NULL} +}; diff --git a/src/fluent-bit/tests/runtime/out_elasticsearch.c b/src/fluent-bit/tests/runtime/out_elasticsearch.c new file mode 100644 index 000000000..9efe7610a --- /dev/null +++ b/src/fluent-bit/tests/runtime/out_elasticsearch.c @@ -0,0 +1,818 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +#include <fluent-bit.h> +#include "flb_tests_runtime.h" + +/* Test data */ +#include "data/es/json_es.h" /* JSON_ES */ + + +static void cb_check_write_op_index(void *ctx, int ffd, + int res_ret, void *res_data, + size_t res_size, void *data) +{ + char *p; + char *out_js = res_data; + char *index_line = "{\"index\":{"; + + p = strstr(out_js, index_line); + TEST_CHECK(p == out_js); + + flb_free(res_data); +} + +static void cb_check_write_op_create(void *ctx, int ffd, + int res_ret, void *res_data, + size_t res_size, void *data) +{ + char *p; + char *out_js = res_data; + char *index_line = "{\"create\":{"; + + p = strstr(out_js, index_line); + TEST_CHECK(p == out_js); + + flb_free(res_data); +} + +static void cb_check_write_op_update(void *ctx, int ffd, + int res_ret, void *res_data, + size_t res_size, void *data) +{ + char *p; + char *b; + char *out_js = res_data; + char *index_line = "{\"update\":{"; + char *body = "{\"doc\":"; + + p = strstr(out_js, index_line); + TEST_CHECK(p == out_js); + b = strstr(out_js, body); + TEST_CHECK(b != NULL); + + flb_free(res_data); +} + +static void cb_check_write_op_upsert(void *ctx, int ffd, + int res_ret, void *res_data, + size_t res_size, void *data) +{ + char *p; + char *b; + char *out_js = res_data; + char *index_line = "{\"update\":{"; + char *body = "{\"doc_as_upsert\":true,\"doc\":"; + + p = strstr(out_js, index_line); + TEST_CHECK(p == out_js); + b = strstr(out_js, body); + TEST_CHECK(b != NULL); + + flb_free(res_data); +} + +static void cb_check_index_type(void *ctx, int ffd, + int res_ret, void *res_data, size_t res_size, + void *data) +{ + char *p; + char *out_js = res_data; + char *index_line = "{\"create\":{\"_index\":\"index_test\",\"_type\":\"type_test\"}"; + + p = strstr(out_js, index_line); + TEST_CHECK(p != NULL); + + flb_free(res_data); +} + +static void cb_check_logstash_format(void *ctx, int ffd, + int res_ret, void *res_data, size_t res_size, + void *data) +{ + char *p; + char *out_js = res_data; + char *index_line = "{\"create\":{\"_index\":\"prefix-2015-11-24\",\"_type\":\"_doc\"}"; + + p = strstr(out_js, index_line); + if(!TEST_CHECK(p != NULL)) { + TEST_MSG("Got: %s", out_js); + } + flb_free(res_data); +} + +static void cb_check_logstash_prefix_separator(void *ctx, int ffd, + int res_ret, void *res_data, size_t res_size, + void *data) +{ + char *p; + char *out_js = res_data; + char *index_line = "{\"create\":{\"_index\":\"prefixSEP2015-11-24\",\"_type\":\"_doc\"}"; + + p = strstr(out_js, index_line); + if(!TEST_CHECK(p != NULL)) { + TEST_MSG("Got: %s", out_js); + } + flb_free(res_data); +} + +static void cb_check_logstash_format_nanos(void *ctx, int ffd, + int res_ret, void *res_data, size_t res_size, + void *data) +{ + char *p; + char *out_js = res_data; + char *index_line = "\"@timestamp\":\"2015-11-24T22:15:40.000000000Z\""; + + p = strstr(out_js, index_line); + TEST_CHECK(p != NULL); + flb_free(res_data); +} + +static void cb_check_tag_key(void *ctx, int ffd, + int res_ret, void *res_data, size_t res_size, + void *data) +{ + char *p; + char *out_js = res_data; + char *record = "\"mytag\":\"test\""; + + p = strstr(out_js, record); + TEST_CHECK(p != NULL); + flb_free(res_data); +} + +static void cb_check_replace_dots(void *ctx, int ffd, + int res_ret, void *res_data, size_t res_size, + void *data) +{ + char *p; + char *out_js = res_data; + char *record = "\"_o_k\":[{\"_b_ar\""; + + p = strstr(out_js, record); + TEST_CHECK(p != NULL); + flb_free(res_data); +} + +static void cb_check_id_key(void *ctx, int ffd, + int res_ret, void *res_data, size_t res_size, + void *data) +{ + char *p; + char *out_js = res_data; + char *record = "\"_id\":\"some string\""; // see data/es/json_es.h + + p = strstr(out_js, record); + TEST_CHECK(p != NULL); + flb_free(res_data); +} + +void flb_test_write_operation_index() +{ + int ret; + int size = sizeof(JSON_ES) - 1; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + + /* Lib input mode */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + /* Elasticsearch output */ + out_ffd = flb_output(ctx, (char *) "es", NULL); + flb_output_set(ctx, out_ffd, + "match", "test", + NULL); + + /* Override defaults of index and type */ + flb_output_set(ctx, out_ffd, + "write_operation", "index", + NULL); + + /* Enable test mode */ + ret = flb_output_set_test(ctx, out_ffd, "formatter", + cb_check_write_op_index, + NULL, NULL); + + /* Start */ + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + flb_lib_push(ctx, in_ffd, (char *) JSON_ES, size); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_write_operation_create() +{ + int ret; + int size = sizeof(JSON_ES) - 1; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + + /* Lib input mode */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + /* Elasticsearch output */ + out_ffd = flb_output(ctx, (char *) "es", NULL); + flb_output_set(ctx, out_ffd, + "match", "test", + NULL); + + /* Override defaults of index and type */ + flb_output_set(ctx, out_ffd, + "write_operation", "create", + NULL); + + /* Enable test mode */ + ret = flb_output_set_test(ctx, out_ffd, "formatter", + cb_check_write_op_create, + NULL, NULL); + + /* Start */ + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + flb_lib_push(ctx, in_ffd, (char *) JSON_ES, size); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + + +void flb_test_write_operation_update() +{ + int ret; + int size = sizeof(JSON_ES) - 1; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + + /* Lib input mode */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + /* Elasticsearch output */ + out_ffd = flb_output(ctx, (char *) "es", NULL); + flb_output_set(ctx, out_ffd, + "match", "test", + NULL); + + /* Override defaults of index and type */ + flb_output_set(ctx, out_ffd, + "Write_Operation", "Update", + "Generate_Id", "True", + NULL); + + /* Enable test mode */ + ret = flb_output_set_test(ctx, out_ffd, "formatter", + cb_check_write_op_update, + NULL, NULL); + + /* Start */ + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + flb_lib_push(ctx, in_ffd, (char *) JSON_ES, size); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + + +void flb_test_write_operation_upsert() +{ + int ret; + int size = sizeof(JSON_ES) - 1; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + + /* Lib input mode */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + /* Elasticsearch output */ + out_ffd = flb_output(ctx, (char *) "es", NULL); + flb_output_set(ctx, out_ffd, + "match", "test", + NULL); + + /* Override defaults of index and type */ + flb_output_set(ctx, out_ffd, + "Write_Operation", "Upsert", + "Generate_Id", "True", + NULL); + + /* Enable test mode */ + ret = flb_output_set_test(ctx, out_ffd, "formatter", + cb_check_write_op_upsert, + NULL, NULL); + + /* Start */ + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + flb_lib_push(ctx, in_ffd, (char *) JSON_ES, size); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_index_type() +{ + int ret; + int size = sizeof(JSON_ES) - 1; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + + /* Lib input mode */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + /* Elasticsearch output */ + out_ffd = flb_output(ctx, (char *) "es", NULL); + flb_output_set(ctx, out_ffd, + "match", "test", + NULL); + + /* Override defaults of index and type */ + flb_output_set(ctx, out_ffd, + "index", "index_test", + "type", "type_test", + NULL); + + /* Enable test mode */ + ret = flb_output_set_test(ctx, out_ffd, "formatter", + cb_check_index_type, + NULL, NULL); + + /* Start */ + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + flb_lib_push(ctx, in_ffd, (char *) JSON_ES, size); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_logstash_format() +{ + int ret; + int size = sizeof(JSON_ES) - 1; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + + /* Lib input mode */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + /* Elasticsearch output */ + out_ffd = flb_output(ctx, (char *) "es", NULL); + flb_output_set(ctx, out_ffd, + "match", "test", + NULL); + + /* Override defaults of index and type */ + flb_output_set(ctx, out_ffd, + "logstash_format", "on", + "logstash_prefix", "prefix", + "logstash_dateformat", "%Y-%m-%d", + NULL); + + /* Enable test mode */ + ret = flb_output_set_test(ctx, out_ffd, "formatter", + cb_check_logstash_format, + NULL, NULL); + + /* Start */ + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + flb_lib_push(ctx, in_ffd, (char *) JSON_ES, size); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_logstash_format_nanos() +{ + int ret; + int size = sizeof(JSON_ES) - 1; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + + /* Lib input mode */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + /* Elasticsearch output */ + out_ffd = flb_output(ctx, (char *) "es", NULL); + flb_output_set(ctx, out_ffd, + "match", "test", + NULL); + + /* Override defaults of index and type */ + flb_output_set(ctx, out_ffd, + "logstash_format", "on", + "logstash_prefix", "prefix", + "logstash_dateformat", "%Y-%m-%d", + "time_key_nanos", "on", + NULL); + + /* Enable test mode */ + ret = flb_output_set_test(ctx, out_ffd, "formatter", + cb_check_logstash_format_nanos, + NULL, NULL); + + /* Start */ + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + flb_lib_push(ctx, in_ffd, (char *) JSON_ES, size); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_tag_key() +{ + int ret; + int size = sizeof(JSON_ES) - 1; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + + /* Lib input mode */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + /* Elasticsearch output */ + out_ffd = flb_output(ctx, (char *) "es", NULL); + flb_output_set(ctx, out_ffd, + "match", "test", + NULL); + + /* Override defaults of index and type */ + flb_output_set(ctx, out_ffd, + "include_tag_key", "on", + "tag_key", "mytag", + NULL); + + /* Enable test mode */ + ret = flb_output_set_test(ctx, out_ffd, "formatter", + cb_check_tag_key, + NULL, NULL); + + /* Start */ + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + flb_lib_push(ctx, in_ffd, (char *) JSON_ES, size); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_replace_dots() +{ + int ret; + int size = sizeof(JSON_DOTS) - 1; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + + /* Lib input mode */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + /* Elasticsearch output */ + out_ffd = flb_output(ctx, (char *) "es", NULL); + flb_output_set(ctx, out_ffd, + "match", "test", + NULL); + + /* Override defaults of index and type */ + flb_output_set(ctx, out_ffd, + "replace_dots", "on", + NULL); + + /* Enable test mode */ + ret = flb_output_set_test(ctx, out_ffd, "formatter", + cb_check_replace_dots, + NULL, NULL); + + /* Start */ + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + flb_lib_push(ctx, in_ffd, (char *) JSON_DOTS, size); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_id_key() +{ + int ret; + int size = sizeof(JSON_ES) - 1; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + + /* Lib input mode */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + /* Elasticsearch output */ + out_ffd = flb_output(ctx, (char *) "es", NULL); + flb_output_set(ctx, out_ffd, + "match", "test", + NULL); + + /* Override defaults of index and type */ + flb_output_set(ctx, out_ffd, + "id_key", "key_2", + NULL); + + /* Enable test mode */ + ret = flb_output_set_test(ctx, out_ffd, "formatter", + cb_check_id_key, + NULL, NULL); + + /* Start */ + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + flb_lib_push(ctx, in_ffd, (char *) JSON_ES, size); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + +/* no check */ +static void cb_check_nothing(void *ctx, int ffd, + int res_ret, void *res_data, size_t res_size, + void *data) +{ + flb_free(res_data); +} + +/* https://github.com/fluent/fluent-bit/issues/3905 */ +void flb_test_div0() +{ + int ret; + char record[8000]; + char record_header[] = "[1448403340,{\"key\":\""; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + int i; + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + + /* Lib input mode */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + /* Elasticsearch output */ + out_ffd = flb_output(ctx, (char *) "es", NULL); + flb_output_set(ctx, out_ffd, + "match", "test", + NULL); + + /* Override defaults of index and type */ + flb_output_set(ctx, out_ffd, + NULL); + + /* Enable test mode */ + ret = flb_output_set_test(ctx, out_ffd, "formatter", + cb_check_nothing, + NULL, NULL); + + /* Start */ + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* create json */ + strncpy(&record[0], &record_header[0], strlen(record_header)); + for(i=strlen(record_header); i<sizeof(record)-4; i++) { + record[i] = 'a'; + } + record[sizeof(record)-4] = '"'; + record[sizeof(record)-3] = '}'; + record[sizeof(record)-2] = ']'; + record[sizeof(record)-1] = '\0'; + /* Ingest data sample */ + flb_lib_push(ctx, in_ffd, (char *) &record[0], strlen(record)); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + + +static void cb_check_long_index(void *ctx, int ffd, + int res_ret, void *res_data, size_t res_size, + void *data) +{ + char *p; + char *out_js = res_data; + char long_index[256] = {0}; + int i; + + for (i=0; i<sizeof(long_index)-1; i++) { + long_index[i] = '0' + (i%10); + } + + p = strstr(out_js, &long_index[0]); + TEST_CHECK(p != NULL); + flb_free(res_data); +} + +/* https://github.com/fluent/fluent-bit/issues/4311 */ +void flb_test_long_index() +{ + int ret; + int size = sizeof(JSON_ES) -1; + char long_index[256] = {0}; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + int i; + + for (i=0; i<sizeof(long_index)-1; i++) { + long_index[i] = '0' + (i%10); + } + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + + /* Lib input mode */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + /* Elasticsearch output */ + out_ffd = flb_output(ctx, (char *) "es", NULL); + flb_output_set(ctx, out_ffd, + "match", "test", + "generate_id", "true", + "index", &long_index[0], + NULL); + + /* Override defaults of index and type */ + flb_output_set(ctx, out_ffd, + NULL); + + /* Enable test mode */ + ret = flb_output_set_test(ctx, out_ffd, "formatter", + cb_check_long_index, + NULL, NULL); + + /* Start */ + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + flb_lib_push(ctx, in_ffd, (char *)JSON_ES, size); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_logstash_prefix_separator() +{ + int ret; + int size = sizeof(JSON_ES) - 1; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + + /* Lib input mode */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + /* Elasticsearch output */ + out_ffd = flb_output(ctx, (char *) "es", NULL); + flb_output_set(ctx, out_ffd, + "match", "test", + NULL); + + /* Override defaults of index and type */ + flb_output_set(ctx, out_ffd, + "logstash_format", "on", + "logstash_prefix", "prefix", + "logstash_prefix_separator", "SEP", + "logstash_dateformat", "%Y-%m-%d", + NULL); + + /* Enable test mode */ + ret = flb_output_set_test(ctx, out_ffd, "formatter", + cb_check_logstash_prefix_separator, + NULL, NULL); + + /* Start */ + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + flb_lib_push(ctx, in_ffd, (char *) JSON_ES, size); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + +/* Test list */ +TEST_LIST = { + {"long_index" , flb_test_long_index }, + {"div0_error" , flb_test_div0 }, + {"write_operation_index" , flb_test_write_operation_index }, + {"write_operation_create", flb_test_write_operation_create }, + {"write_operation_update", flb_test_write_operation_update }, + {"write_operation_upsert", flb_test_write_operation_upsert }, + {"index_type" , flb_test_index_type }, + {"logstash_format" , flb_test_logstash_format }, + {"logstash_format_nanos" , flb_test_logstash_format_nanos }, + {"tag_key" , flb_test_tag_key }, + {"replace_dots" , flb_test_replace_dots }, + {"id_key" , flb_test_id_key }, + {"logstash_prefix_separator" , flb_test_logstash_prefix_separator }, + {NULL, NULL} +}; diff --git a/src/fluent-bit/tests/runtime/out_exit.c b/src/fluent-bit/tests/runtime/out_exit.c new file mode 100644 index 000000000..7ddd70de7 --- /dev/null +++ b/src/fluent-bit/tests/runtime/out_exit.c @@ -0,0 +1,210 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +#include <fluent-bit.h> +#include "flb_tests_runtime.h" + +/* Test data */ +#include "data/common/json_invalid.h" /* JSON_INVALID */ +#include "data/common/json_long.h" /* JSON_LONG */ +#include "data/common/json_small.h" /* JSON_SMALL */ + +/* Test functions */ +void flb_test_exit_json_invalid(void); +void flb_test_exit_json_long(void); +void flb_test_exit_json_small(void); +void flb_test_exit_keep_alive(void); +void flb_test_exit_clean_shutdown(void); + +/* Test list */ +TEST_LIST = { + {"json_invalid", flb_test_exit_json_invalid }, + {"json_long", flb_test_exit_json_long }, + {"json_small", flb_test_exit_json_small }, + {"keep_alive", flb_test_exit_keep_alive }, + {"clean_shutdown", flb_test_exit_clean_shutdown}, + {NULL, NULL} +}; + + +#define WAIT_STOP (5+1) /* pause in flb_engine_stop and buffer period */ + +void flb_test_exit_json_invalid(void) +{ + int i; + int ret; + int bytes; + char *p = (char *) JSON_INVALID; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + ctx = flb_create(); + flb_service_set(ctx, "Flush", "1", "Grace", "1", "Log_Level", "error", NULL); + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + out_ffd = flb_output(ctx, (char *) "exit", NULL); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, "match", "test", NULL); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + for (i = 0; i < (int) sizeof(JSON_INVALID) - 1; i++) { + bytes = flb_lib_push(ctx, in_ffd, p + i, 1); + TEST_CHECK(bytes == 1); + } + + sleep(WAIT_STOP); /* waiting stop automatically */ + + /* On invalid case, flb_stop() is not called from out_exit plugin. + * To shutdown normally and cleanly, it needs to call flb_stop() here. */ + flb_stop(ctx); + flb_destroy(ctx); +} + +/* It writes a very long JSON map (> 100KB) byte by byte */ +void flb_test_exit_json_long(void) +{ + int i; + int ret; + int bytes; + char *p = (char *) JSON_LONG; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + ctx = flb_create(); + flb_service_set(ctx, "Flush", "1", "Grace", "1", "Log_Level", "error", NULL); + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + out_ffd = flb_output(ctx, (char *) "exit", NULL); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, "match", "test", NULL); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + for (i = 0; i < (int) sizeof(JSON_LONG) - 1; i++) { + bytes = flb_lib_push(ctx, in_ffd, p + i, 1); + TEST_CHECK(bytes == 1); + } + + sleep(WAIT_STOP); /* waiting stop automatically */ + + /* call flb_stop() from out_exit plugin */ + flb_destroy(ctx); +} + +void flb_test_exit_json_small(void) +{ + int i; + int ret; + int bytes; + char *p = (char *) JSON_SMALL; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + ctx = flb_create(); + flb_service_set(ctx, "Flush", "1", "Grace", "1", "Log_Level", "error", NULL); + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + out_ffd = flb_output(ctx, (char *) "exit", NULL); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, "match", "test", NULL); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + for (i = 0; i < (int) sizeof(JSON_SMALL) - 1; i++) { + bytes = flb_lib_push(ctx, in_ffd, p + i, 1); + TEST_CHECK(bytes == 1); + } + + sleep(WAIT_STOP); /* waiting stop automatically */ + + /* call flb_stop() from out_exit plugin */ + flb_destroy(ctx); +} + +void flb_test_exit_keep_alive(void) +{ + int i; + int ret; + int bytes; + char *p = (char *) JSON_SMALL; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + ctx = flb_create(); + flb_service_set(ctx, "Flush", "1", "Grace", "1", "Log_Level", "error", NULL); + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + out_ffd = flb_output(ctx, (char *) "exit", NULL); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, "match", "test", NULL); + flb_output_set(ctx, out_ffd, "flush_count", "10", NULL); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + for (i = 0; i < (int) sizeof(JSON_SMALL) - 1; i++) { + bytes = flb_lib_push(ctx, in_ffd, p + i, 1); + TEST_CHECK(bytes == 1); + } + + sleep(WAIT_STOP); /* waiting flush */ + + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_exit_clean_shutdown(void) +{ + int i; + int ret; + int bytes; + char *p = (char *) JSON_SMALL; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + ctx = flb_create(); + flb_service_set(ctx, "Flush", "1", "Grace", "-1", "Log_Level", "error", NULL); + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + out_ffd = flb_output(ctx, (char *) "exit", NULL); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, "match", "test", NULL); + flb_output_set(ctx, out_ffd, "flush_count", "10", NULL); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + for (i = 0; i < (int) sizeof(JSON_SMALL) - 1; i++) { + bytes = flb_lib_push(ctx, in_ffd, p + i, 1); + TEST_CHECK(bytes == 1); + } + + sleep(WAIT_STOP); /* waiting flush */ + + flb_stop(ctx); + flb_destroy(ctx); +} + diff --git a/src/fluent-bit/tests/runtime/out_file.c b/src/fluent-bit/tests/runtime/out_file.c new file mode 100644 index 000000000..33b1440d0 --- /dev/null +++ b/src/fluent-bit/tests/runtime/out_file.c @@ -0,0 +1,824 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +#include <fluent-bit.h> +#include <fluent-bit/flb_sds.h> +#include "flb_tests_runtime.h" +#include <sys/stat.h> +#include <sys/types.h> + +/* Test data */ +#include "data/common/json_invalid.h" /* JSON_INVALID */ +#include "data/common/json_long.h" /* JSON_LONG */ +#include "data/common/json_small.h" /* JSON_SMALL */ + +/* Test functions */ +void flb_test_file_json_invalid(void); +void flb_test_file_json_long(void); +void flb_test_file_json_small(void); +void flb_test_file_format_csv(void); +void flb_test_file_format_ltsv(void); +void flb_test_file_format_invalid(void); +void flb_test_file_format_out_file(void); +void flb_test_file_path_file(void); +void flb_test_file_path(void); +void flb_test_file_delim_csv(void); +void flb_test_file_delim_ltsv(void); +void flb_test_file_label_delim(void); +void flb_test_file_template(void); +void flb_test_file_mkdir(void); + +/* Test list */ +TEST_LIST = { + {"path", flb_test_file_path}, + {"path_file", flb_test_file_path_file}, + {"mkdir", flb_test_file_mkdir}, + {"template", flb_test_file_template}, + {"delimiter_ltsv", flb_test_file_delim_ltsv}, + {"delimiter_csv", flb_test_file_delim_csv}, + {"label_delimiter", flb_test_file_label_delim}, + {"json_invalid", flb_test_file_json_invalid }, + {"json_long", flb_test_file_json_long }, + {"json_small", flb_test_file_json_small }, + {"format_csv", flb_test_file_format_csv }, + {"format_ltsv", flb_test_file_format_ltsv }, + {"format_invalid", flb_test_file_format_invalid }, + {"format_out_file", flb_test_file_format_out_file}, + + {NULL, NULL} +}; + + +#define TEST_LOGFILE "flb_test_file_dummy.log" +#define TEST_LOGPATH "out_file" +#define TEST_TIMEOUT 5 + +void flb_test_file_json_invalid(void) +{ + int i; + int ret; + int bytes; + char *p = (char *) JSON_INVALID; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + FILE *fp; + + remove(TEST_LOGFILE); + + ctx = flb_create(); + flb_service_set(ctx, "Flush", "1", "Grace", "1", "Log_Level", "error", NULL); + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + out_ffd = flb_output(ctx, (char *) "file", NULL); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, "match", "test", NULL); + flb_output_set(ctx, out_ffd, "file", TEST_LOGFILE, NULL); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + for (i = 0; i < (int) sizeof(JSON_INVALID) - 1; i++) { + bytes = flb_lib_push(ctx, in_ffd, p + i, 1); + TEST_CHECK(bytes == 1); + } + + flb_stop(ctx); + flb_destroy(ctx); + + fp = fopen(TEST_LOGFILE, "r"); + TEST_CHECK(fp == NULL); + if (fp != NULL) { + fclose(fp); + remove(TEST_LOGFILE); + } +} + +/* It writes a very long JSON map (> 100KB) byte by byte */ +void flb_test_file_json_long(void) +{ + int i; + int ret; + int bytes; + char *p = (char *) JSON_LONG; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + FILE *fp; + + remove(TEST_LOGFILE); + + ctx = flb_create(); + flb_service_set(ctx, "Flush", "1", "Grace", "1", "Log_Level", "error", NULL); + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + out_ffd = flb_output(ctx, (char *) "file", NULL); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, "match", "test", NULL); + flb_output_set(ctx, out_ffd, "file", TEST_LOGFILE, NULL); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + for (i = 0; i < (int) sizeof(JSON_LONG) - 1; i++) { + bytes = flb_lib_push(ctx, in_ffd, p + i, 1); + TEST_CHECK(bytes == 1); + } + + ret = wait_for_file(TEST_LOGFILE, 1, TEST_TIMEOUT); + TEST_CHECK(ret == 0); + + flb_stop(ctx); + flb_destroy(ctx); + + fp = fopen(TEST_LOGFILE, "r"); + TEST_CHECK(fp != NULL); + if (fp != NULL) { + fclose(fp); + remove(TEST_LOGFILE); + } +} + +void flb_test_file_json_small(void) +{ + int i; + int ret; + int bytes; + char *p = (char *) JSON_SMALL; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + FILE *fp; + + remove(TEST_LOGFILE); + + ctx = flb_create(); + flb_service_set(ctx, "Flush", "1", "Grace", "1", "Log_Level", "error", NULL); + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + out_ffd = flb_output(ctx, (char *) "file", NULL); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, "match", "test", NULL); + flb_output_set(ctx, out_ffd, "file", TEST_LOGFILE, NULL); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + for (i = 0; i < (int) sizeof(JSON_SMALL) - 1; i++) { + bytes = flb_lib_push(ctx, in_ffd, p + i, 1); + TEST_CHECK(bytes == 1); + } + + ret = wait_for_file(TEST_LOGFILE, 1, TEST_TIMEOUT); + TEST_CHECK(ret == 0); + + flb_stop(ctx); + flb_destroy(ctx); + + fp = fopen(TEST_LOGFILE, "r"); + TEST_CHECK(fp != NULL); + if (fp != NULL) { + fclose(fp); + remove(TEST_LOGFILE); + } +} + +void flb_test_file_format_csv(void) +{ + int i; + int ret; + int bytes; + char *p = (char *) JSON_SMALL; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + FILE *fp; + + remove(TEST_LOGFILE); + + ctx = flb_create(); + flb_service_set(ctx, "Flush", "1", "Grace", "1", "Log_Level", "error", NULL); + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + out_ffd = flb_output(ctx, (char *) "file", NULL); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, "match", "test", NULL); + flb_output_set(ctx, out_ffd, "file", TEST_LOGFILE, NULL); + flb_output_set(ctx, out_ffd, "format", "csv", NULL); + flb_output_set(ctx, out_ffd, "delimiter", "comma", NULL); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + for (i = 0; i < (int) sizeof(JSON_SMALL) - 1; i++) { + bytes = flb_lib_push(ctx, in_ffd, p + i, 1); + TEST_CHECK(bytes == 1); + } + + ret = wait_for_file(TEST_LOGFILE, 1, TEST_TIMEOUT); + TEST_CHECK(ret == 0); + + flb_stop(ctx); + flb_destroy(ctx); + + fp = fopen(TEST_LOGFILE, "r"); + TEST_CHECK(fp != NULL); + if (fp != NULL) { + fclose(fp); + remove(TEST_LOGFILE); + } +} + +void flb_test_file_format_ltsv(void) +{ + int i; + int ret; + int bytes; + char *p = (char *) JSON_SMALL; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + FILE *fp; + + remove(TEST_LOGFILE); + + ctx = flb_create(); + flb_service_set(ctx, "Flush", "1", "Grace", "1", "Log_Level", "error", NULL); + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + out_ffd = flb_output(ctx, (char *) "file", NULL); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, "match", "test", NULL); + flb_output_set(ctx, out_ffd, "file", TEST_LOGFILE, NULL); + flb_output_set(ctx, out_ffd, "format", "ltsv", NULL); + flb_output_set(ctx, out_ffd, "delimiter", "tab", NULL); + flb_output_set(ctx, out_ffd, "label_delimiter", "comma", NULL); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + for (i = 0; i < (int) sizeof(JSON_SMALL) - 1; i++) { + bytes = flb_lib_push(ctx, in_ffd, p + i, 1); + TEST_CHECK(bytes == 1); + } + + ret = wait_for_file(TEST_LOGFILE, 1, TEST_TIMEOUT); + TEST_CHECK(ret == 0); + + flb_stop(ctx); + flb_destroy(ctx); + + fp = fopen(TEST_LOGFILE, "r"); + TEST_CHECK(fp != NULL); + if (fp != NULL) { + fclose(fp); + remove(TEST_LOGFILE); + } +} + +/* https://github.com/fluent/fluent-bit/issues/4152 */ +void flb_test_file_format_out_file(void) +{ + int i; + int ret; + int bytes; + char *p = (char *) JSON_SMALL; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + FILE *fp; + + remove(TEST_LOGFILE); + + ctx = flb_create(); + flb_service_set(ctx, "Flush", "1", "Grace", "1", "Log_Level", "error", NULL); + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + out_ffd = flb_output(ctx, (char *) "file", NULL); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, "match", "test", NULL); + flb_output_set(ctx, out_ffd, "file", TEST_LOGFILE, NULL); + flb_output_set(ctx, out_ffd, "format", "out_file", NULL); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + for (i = 0; i < (int) sizeof(JSON_SMALL) - 1; i++) { + bytes = flb_lib_push(ctx, in_ffd, p + i, 1); + TEST_CHECK(bytes == 1); + } + + ret = wait_for_file(TEST_LOGFILE, 1, TEST_TIMEOUT); + TEST_CHECK(ret == 0); + + flb_stop(ctx); + flb_destroy(ctx); + + fp = fopen(TEST_LOGFILE, "r"); + TEST_CHECK(fp != NULL); + if (fp != NULL) { + fclose(fp); + remove(TEST_LOGFILE); + } +} + +void flb_test_file_format_invalid(void) +{ + int ret; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + FILE *fp; + + remove(TEST_LOGFILE); + + ctx = flb_create(); + flb_service_set(ctx, "Flush", "1", "Grace", "1", "Log_Level", "off", NULL); + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + out_ffd = flb_output(ctx, (char *) "file", NULL); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, "match", "test", NULL); + flb_output_set(ctx, out_ffd, "file", TEST_LOGFILE, NULL); + flb_output_set(ctx, out_ffd, "format", "xxx", NULL); + flb_output_set(ctx, out_ffd, "delimiter", "yyy", NULL); + flb_output_set(ctx, out_ffd, "label_delimiter", "zzz", NULL); + + ret = flb_start(ctx); + if (!TEST_CHECK(ret != 0)) { + TEST_MSG("invalid format should be error"); + + flb_stop(ctx); + flb_destroy(ctx); + fp = fopen(TEST_LOGFILE, "r"); + TEST_CHECK(fp != NULL); + if (fp != NULL) { + fclose(fp); + remove(TEST_LOGFILE); + } + } + else { + flb_destroy(ctx); + } +} + +void flb_test_file_path(void) +{ + int i; + int ret; + int bytes; + char *p = (char *) JSON_LONG; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + FILE *fp; + flb_sds_t path; + flb_sds_t file; + + file = flb_sds_create("test"); + if (!TEST_CHECK(file != NULL)) { + TEST_MSG("flb_sds_create failed"); + return; + } + + path = flb_sds_create_size(256); + if (!TEST_CHECK(path != NULL)) { + TEST_MSG("flb_sds_create_size failed"); + flb_sds_destroy(file); + return; + } + flb_sds_printf(&path, "%s/%s", TEST_LOGPATH, file); + + remove(path); + remove(TEST_LOGPATH); + ret = mkdir(TEST_LOGPATH, S_IRUSR | S_IWUSR | S_IXUSR); + if (!TEST_CHECK(ret == 0)) { + TEST_MSG("mkdir failed:path=%s errno=%d",TEST_LOGPATH, errno); + flb_sds_destroy(path); + flb_sds_destroy(file); + return; + } + + ctx = flb_create(); + flb_service_set(ctx, "Flush", "1", "Grace", "1", "Log_Level", "error", NULL); + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx, in_ffd, "tag", file, NULL); + + out_ffd = flb_output(ctx, (char *) "file", NULL); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, "match", "test", NULL); + flb_output_set(ctx, out_ffd, "path", TEST_LOGPATH, NULL); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + for (i = 0; i < (int) sizeof(JSON_LONG) - 1; i++) { + bytes = flb_lib_push(ctx, in_ffd, p + i, 1); + TEST_CHECK(bytes == 1); + } + + ret = wait_for_file(path, 1, TEST_TIMEOUT); + TEST_CHECK(ret == 0); + + flb_stop(ctx); + flb_destroy(ctx); + + fp = fopen(path, "r"); + TEST_CHECK(fp != NULL); + if (fp != NULL) { + fclose(fp); + remove(path); + } + flb_sds_destroy(path); + flb_sds_destroy(file); + remove(TEST_LOGPATH); +} + +void flb_test_file_path_file(void) +{ + int i; + int ret; + int bytes; + char *p = (char *) JSON_LONG; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + FILE *fp; + flb_sds_t path; + + path = flb_sds_create_size(256); + if (!TEST_CHECK(path != NULL)) { + TEST_MSG("flb_sds_create_size failed"); + return; + } + flb_sds_printf(&path, "%s/%s", TEST_LOGPATH, TEST_LOGFILE); + + remove(path); + remove(TEST_LOGPATH); + ret = mkdir(TEST_LOGPATH, S_IRUSR | S_IWUSR | S_IXUSR); + if (!TEST_CHECK(ret == 0)) { + TEST_MSG("mkdir failed:path=%s errno=%d",TEST_LOGPATH, errno); + flb_sds_destroy(path); + return; + } + + ctx = flb_create(); + flb_service_set(ctx, "Flush", "1", "Grace", "1", "Log_Level", "error", NULL); + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + out_ffd = flb_output(ctx, (char *) "file", NULL); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, "match", "test", NULL); + flb_output_set(ctx, out_ffd, "file", TEST_LOGFILE, NULL); + flb_output_set(ctx, out_ffd, "path", TEST_LOGPATH, NULL); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + for (i = 0; i < (int) sizeof(JSON_LONG) - 1; i++) { + bytes = flb_lib_push(ctx, in_ffd, p + i, 1); + TEST_CHECK(bytes == 1); + } + + ret = wait_for_file(path, 1, TEST_TIMEOUT); + TEST_CHECK(ret == 0); + + flb_stop(ctx); + flb_destroy(ctx); + + fp = fopen(path, "r"); + TEST_CHECK(fp != NULL); + if (fp != NULL) { + fclose(fp); + remove(path); + } + flb_sds_destroy(path); + remove(TEST_LOGPATH); +} + +#define JSON_BASIC "[1448403340,{\"key1\":\"val1\", \"key2\":\"val2\"}]" +void flb_test_file_delim_csv(void) +{ + int ret; + int bytes; + char *p = JSON_BASIC; + char output[256] = {0}; + char *expect = "1448403340.000000000 \"val1\" \"val2\""; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + FILE *fp; + + remove(TEST_LOGFILE); + + ctx = flb_create(); + flb_service_set(ctx, "Flush", "1", "Grace", "1", "Log_Level", "error", NULL); + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + out_ffd = flb_output(ctx, (char *) "file", NULL); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, "match", "test", NULL); + flb_output_set(ctx, out_ffd, "file", TEST_LOGFILE, NULL); + flb_output_set(ctx, out_ffd, "format", "csv", NULL); + flb_output_set(ctx, out_ffd, "delimiter", "space", NULL); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + bytes = flb_lib_push(ctx, in_ffd, p, strlen(p)); + TEST_CHECK(bytes == strlen(p)); + + ret = wait_for_file(TEST_LOGFILE, 1, TEST_TIMEOUT); + TEST_CHECK(ret == 0); + + flb_stop(ctx); + flb_destroy(ctx); + + fp = fopen(TEST_LOGFILE, "r"); + TEST_CHECK(fp != NULL); + if (fp != NULL) { + bytes = fread(&output[0], sizeof(output), 1, fp); + if(!TEST_CHECK(bytes > 0 || feof(fp))) { + TEST_MSG("fread error bytes=%d", bytes); + } + if (!TEST_CHECK(strncmp(expect, &output[0], strlen(expect)) == 0)) { + TEST_MSG("format error\n"); + TEST_MSG("expect: %s\n", expect); + TEST_MSG("got : %s",output); + } + + fclose(fp); + remove(TEST_LOGFILE); + } +} + +void flb_test_file_delim_ltsv(void) +{ + int ret; + int bytes; + char *p = JSON_BASIC; + char output[256] = {0}; + char *expect = "\"time\":1448403340.000000 \"key1\":\"val1\" \"key2\":\"val2\""; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + FILE *fp; + + remove(TEST_LOGFILE); + + ctx = flb_create(); + flb_service_set(ctx, "Flush", "1", "Grace", "1", "Log_Level", "error", NULL); + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + out_ffd = flb_output(ctx, (char *) "file", NULL); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, "match", "test", NULL); + flb_output_set(ctx, out_ffd, "file", TEST_LOGFILE, NULL); + flb_output_set(ctx, out_ffd, "format", "ltsv", NULL); + flb_output_set(ctx, out_ffd, "delimiter", "space", NULL); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + bytes = flb_lib_push(ctx, in_ffd, p, strlen(p)); + TEST_CHECK(bytes == strlen(p)); + + ret = wait_for_file(TEST_LOGFILE, 1, TEST_TIMEOUT); + TEST_CHECK(ret == 0); + + flb_stop(ctx); + flb_destroy(ctx); + + fp = fopen(TEST_LOGFILE, "r"); + TEST_CHECK(fp != NULL); + if (fp != NULL) { + bytes = fread(&output[0], sizeof(output), 1, fp); + if(!TEST_CHECK(bytes > 0 || feof(fp))) { + TEST_MSG("fread error bytes=%d", bytes); + } + if (!TEST_CHECK(strncmp(expect, &output[0], strlen(expect)) == 0)) { + TEST_MSG("format error\n"); + TEST_MSG("expect: %s\n", expect); + TEST_MSG("got : %s",output); + } + + fclose(fp); + remove(TEST_LOGFILE); + } +} + +void flb_test_file_label_delim(void) +{ + int ret; + int bytes; + char *p = JSON_BASIC; + char output[256] = {0}; + char *expect = "\"time\" 1448403340.000000 \"key1\" \"val1\" \"key2\" \"val2\""; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + FILE *fp; + + remove(TEST_LOGFILE); + + ctx = flb_create(); + flb_service_set(ctx, "Flush", "1", "Grace", "1", "Log_Level", "error", NULL); + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + out_ffd = flb_output(ctx, (char *) "file", NULL); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, "match", "test", NULL); + flb_output_set(ctx, out_ffd, "file", TEST_LOGFILE, NULL); + flb_output_set(ctx, out_ffd, "format", "ltsv", NULL); + flb_output_set(ctx, out_ffd, "delimiter", "space", NULL); + flb_output_set(ctx, out_ffd, "label_delimiter", "space", NULL); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + bytes = flb_lib_push(ctx, in_ffd, p, strlen(p)); + TEST_CHECK(bytes == strlen(p)); + + ret = wait_for_file(TEST_LOGFILE, 1, TEST_TIMEOUT); + TEST_CHECK(ret == 0); + + flb_stop(ctx); + flb_destroy(ctx); + + fp = fopen(TEST_LOGFILE, "r"); + TEST_CHECK(fp != NULL); + if (fp != NULL) { + bytes = fread(&output[0], sizeof(output), 1, fp); + if(!TEST_CHECK(bytes > 0 || feof(fp))) { + TEST_MSG("fread error bytes=%d", bytes); + } + if (!TEST_CHECK(strncmp(expect, &output[0], strlen(expect)) == 0)) { + TEST_MSG("format error\n"); + TEST_MSG("expect: %s\n", expect); + TEST_MSG("got : %s",output); + } + + fclose(fp); + remove(TEST_LOGFILE); + } +} + +void flb_test_file_template(void) +{ + int ret; + int bytes; + char *p = JSON_BASIC; + char output[256] = {0}; + char *expect = "1448403340.000000 KEY1=val1 KEY2=val2"; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + FILE *fp; + + remove(TEST_LOGFILE); + + ctx = flb_create(); + flb_service_set(ctx, "Flush", "1", "Grace", "1", "Log_Level", "error", NULL); + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + out_ffd = flb_output(ctx, (char *) "file", NULL); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, "match", "test", NULL); + flb_output_set(ctx, out_ffd, "file", TEST_LOGFILE, NULL); + flb_output_set(ctx, out_ffd, "format", "template", NULL); + flb_output_set(ctx, out_ffd, "template", "{time} KEY1={key1} KEY2={key2}", NULL); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + bytes = flb_lib_push(ctx, in_ffd, p, strlen(p)); + TEST_CHECK(bytes == strlen(p)); + + ret = wait_for_file(TEST_LOGFILE, 1, TEST_TIMEOUT); + TEST_CHECK(ret == 0); + + flb_stop(ctx); + flb_destroy(ctx); + + fp = fopen(TEST_LOGFILE, "r"); + TEST_CHECK(fp != NULL); + if (fp != NULL) { + bytes = fread(&output[0], sizeof(output), 1, fp); + if(!TEST_CHECK(bytes > 0 || feof(fp))) { + TEST_MSG("fread error bytes=%d", bytes); + } + if (!TEST_CHECK(strncmp(expect, &output[0], strlen(expect)) == 0)) { + TEST_MSG("format error\n"); + TEST_MSG("expect: %s\n", expect); + TEST_MSG("got : %s",output); + } + + fclose(fp); + remove(TEST_LOGFILE); + } +} + +void flb_test_file_mkdir(void) +{ + int i; + int ret; + int bytes; + char *p = (char *) JSON_LONG; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + FILE *fp; + flb_sds_t path; + flb_sds_t file; + + file = flb_sds_create("test"); + if (!TEST_CHECK(file != NULL)) { + TEST_MSG("flb_sds_create failed"); + return; + } + + path = flb_sds_create_size(256); + if (!TEST_CHECK(path != NULL)) { + TEST_MSG("flb_sds_create_size failed"); + flb_sds_destroy(file); + return; + } + flb_sds_printf(&path, "%s/%s", TEST_LOGPATH, file); + + remove(path); + remove(TEST_LOGPATH); + + ctx = flb_create(); + flb_service_set(ctx, "Flush", "1", "Grace", "1", "Log_Level", "error", NULL); + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx, in_ffd, "tag", file, NULL); + + out_ffd = flb_output(ctx, (char *) "file", NULL); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, "match", "test", NULL); + flb_output_set(ctx, out_ffd, "path", TEST_LOGPATH, NULL); + flb_output_set(ctx, out_ffd, "mkdir", "true", NULL); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + for (i = 0; i < (int) sizeof(JSON_LONG) - 1; i++) { + bytes = flb_lib_push(ctx, in_ffd, p + i, 1); + TEST_CHECK(bytes == 1); + } + + ret = wait_for_file(path, 1, TEST_TIMEOUT); + TEST_CHECK(ret == 0); + + flb_stop(ctx); + flb_destroy(ctx); + + fp = fopen(path, "r"); + TEST_CHECK(fp != NULL); + if (fp != NULL) { + fclose(fp); + remove(path); + } + flb_sds_destroy(path); + flb_sds_destroy(file); + remove(TEST_LOGPATH); +} diff --git a/src/fluent-bit/tests/runtime/out_firehose.c b/src/fluent-bit/tests/runtime/out_firehose.c new file mode 100644 index 000000000..cead6b3ca --- /dev/null +++ b/src/fluent-bit/tests/runtime/out_firehose.c @@ -0,0 +1,200 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ +#include <fluent-bit.h> +#include "flb_tests_runtime.h" + +/* Test data */ +#include "data/td/json_td.h" /* JSON_TD */ + +#define ERROR_THROUGHPUT "{\"__type\":\"ServiceUnavailableException\"}" +/* not a real error code, but tests that the code can respond to any error */ +#define ERROR_UNKNOWN "{\"__type\":\"UNKNOWN\"}" + +/* It writes a big JSON message (copied from TD test) */ +void flb_test_firehose_success(void) +{ + int ret; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + /* mocks calls- signals that we are in test mode */ + setenv("FLB_FIREHOSE_PLUGIN_UNDER_TEST", "true", 1); + + ctx = flb_create(); + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx,in_ffd, "tag", "test", NULL); + + out_ffd = flb_output(ctx, (char *) "kinesis_firehose", NULL); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd,"match", "*", NULL); + flb_output_set(ctx, out_ffd,"region", "us-west-2", NULL); + flb_output_set(ctx, out_ffd,"delivery_stream", "fluent", NULL); + flb_output_set(ctx, out_ffd,"time_key", "time", NULL); + flb_output_set(ctx, out_ffd,"Retry_Limit", "1", NULL); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + flb_lib_push(ctx, in_ffd, (char *) JSON_TD , (int) sizeof(JSON_TD) - 1); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_firehose_partial_success(void) +{ + int ret; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + /* mocks calls- signals that we are in test mode */ + setenv("FLB_FIREHOSE_PLUGIN_UNDER_TEST", "true", 1); + setenv("PARTIAL_SUCCESS_CASE", "true", 1); + + ctx = flb_create(); + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx,in_ffd, "tag", "test", NULL); + + out_ffd = flb_output(ctx, (char *) "kinesis_firehose", NULL); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd,"match", "*", NULL); + flb_output_set(ctx, out_ffd,"region", "us-west-2", NULL); + flb_output_set(ctx, out_ffd,"delivery_stream", "fluent", NULL); + flb_output_set(ctx, out_ffd,"time_key", "time", NULL); + flb_output_set(ctx, out_ffd,"Retry_Limit", "1", NULL); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + flb_lib_push(ctx, in_ffd, (char *) JSON_TD , (int) sizeof(JSON_TD) - 1); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); + unsetenv("PARTIAL_SUCCESS_CASE"); +} + +void flb_test_firehose_throughput_error(void) +{ + int ret; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + /* mocks calls- signals that we are in test mode */ + setenv("FLB_FIREHOSE_PLUGIN_UNDER_TEST", "true", 1); + setenv("TEST_PUT_RECORD_BATCH_ERROR", ERROR_THROUGHPUT, 1); + + ctx = flb_create(); + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx,in_ffd, "tag", "test", NULL); + + out_ffd = flb_output(ctx, (char *) "kinesis_firehose", NULL); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd,"match", "*", NULL); + flb_output_set(ctx, out_ffd,"region", "us-west-2", NULL); + flb_output_set(ctx, out_ffd,"delivery_stream", "fluent", NULL); + flb_output_set(ctx, out_ffd,"time_key", "time", NULL); + flb_output_set(ctx, out_ffd,"Retry_Limit", "1", NULL); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + flb_lib_push(ctx, in_ffd, (char *) JSON_TD , (int) sizeof(JSON_TD) - 1); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); + unsetenv("TEST_PUT_RECORD_BATCH_ERROR"); +} + +void flb_test_firehose_error_unknown(void) +{ + int ret; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + /* mocks calls- signals that we are in test mode */ + setenv("FLB_FIREHOSE_PLUGIN_UNDER_TEST", "true", 1); + setenv("TEST_PUT_RECORD_BATCH_ERROR", ERROR_UNKNOWN, 1); + + ctx = flb_create(); + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx,in_ffd, "tag", "test", NULL); + + out_ffd = flb_output(ctx, (char *) "kinesis_firehose", NULL); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd,"match", "*", NULL); + flb_output_set(ctx, out_ffd,"region", "us-west-2", NULL); + flb_output_set(ctx, out_ffd,"delivery_stream", "fluent", NULL); + flb_output_set(ctx, out_ffd,"time_key", "time", NULL); + flb_output_set(ctx, out_ffd,"Retry_Limit", "1", NULL); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + flb_lib_push(ctx, in_ffd, (char *) JSON_TD , (int) sizeof(JSON_TD) - 1); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); + unsetenv("TEST_PUT_RECORD_BATCH_ERROR"); +} + +void flb_test_firehose_nonsense_error(void) +{ + int ret; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + /* mocks calls- signals that we are in test mode */ + setenv("FLB_FIREHOSE_PLUGIN_UNDER_TEST", "true", 1); + setenv("TEST_PUT_RECORD_BATCH_ERROR", "\tbadresponse\nnotparsable{}", 1); + + ctx = flb_create(); + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx,in_ffd, "tag", "test", NULL); + + out_ffd = flb_output(ctx, (char *) "kinesis_firehose", NULL); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd,"match", "*", NULL); + flb_output_set(ctx, out_ffd,"region", "us-west-2", NULL); + flb_output_set(ctx, out_ffd,"delivery_stream", "fluent", NULL); + flb_output_set(ctx, out_ffd,"time_key", "time", NULL); + flb_output_set(ctx, out_ffd,"Retry_Limit", "1", NULL); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + flb_lib_push(ctx, in_ffd, (char *) JSON_TD , (int) sizeof(JSON_TD) - 1); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); + unsetenv("TEST_PUT_RECORD_BATCH_ERROR"); +} + + +/* Test list */ +TEST_LIST = { + {"success", flb_test_firehose_success }, + {"partial_success", flb_test_firehose_partial_success }, + {"throughput_error", flb_test_firehose_throughput_error }, + {"unknown_error", flb_test_firehose_error_unknown }, + {"nonsense_error", flb_test_firehose_nonsense_error }, + {NULL, NULL} +}; diff --git a/src/fluent-bit/tests/runtime/out_flowcounter.c b/src/fluent-bit/tests/runtime/out_flowcounter.c new file mode 100644 index 000000000..814793699 --- /dev/null +++ b/src/fluent-bit/tests/runtime/out_flowcounter.c @@ -0,0 +1,317 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +#include <fluent-bit.h> +#include "flb_tests_runtime.h" + +/* Test data */ +#include "data/common/json_invalid.h" /* JSON_INVALID */ +#include "data/common/json_long.h" /* JSON_LONG */ +#include "data/common/json_small.h" /* JSON_SMALL */ + +/* Test functions */ +void flb_test_flowcounter_json_invalid(void); +void flb_test_flowcounter_json_long(void); +void flb_test_flowcounter_json_small(void); +void flb_test_flowcounter_unit_second(void); +void flb_test_flowcounter_unit_minute(void); +void flb_test_flowcounter_unit_hour(void); +void flb_test_flowcounter_unit_day(void); +void flb_test_flowcounter_unit_invalid(void); + +/* Test list */ +TEST_LIST = { + {"json_invalid", flb_test_flowcounter_json_invalid }, + {"json_long", flb_test_flowcounter_json_long }, + {"json_small", flb_test_flowcounter_json_small }, + {"unit_second", flb_test_flowcounter_unit_second }, + {"unit_minute", flb_test_flowcounter_unit_minute }, + {"unit_hour", flb_test_flowcounter_unit_hour }, + {"unit_day", flb_test_flowcounter_unit_day }, + {"unit_invalid", flb_test_flowcounter_unit_invalid }, + {NULL, NULL} +}; + + +void flb_test_flowcounter_json_invalid(void) +{ + int i; + int ret; + int bytes; + char *p = (char *) JSON_INVALID; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + ctx = flb_create(); + flb_service_set(ctx, "Flush", "1", "Grace", "1", "Log_Level", "error", NULL); + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + out_ffd = flb_output(ctx, (char *) "flowcounter", NULL); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, "match", "test", NULL); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + for (i = 0; i < (int) sizeof(JSON_INVALID) - 1; i++) { + bytes = flb_lib_push(ctx, in_ffd, p + i, 1); + TEST_CHECK(bytes == 1); + } + + flb_stop(ctx); + flb_destroy(ctx); +} + +/* It writes a very long JSON map (> 100KB) byte by byte */ +void flb_test_flowcounter_json_long(void) +{ + int i; + int ret; + int bytes; + char *p = (char *) JSON_LONG; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + ctx = flb_create(); + flb_service_set(ctx, "Flush", "1", "Grace", "1", "Log_Level", "error", NULL); + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + out_ffd = flb_output(ctx, (char *) "flowcounter", NULL); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, "match", "test", NULL); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + for (i = 0; i < (int) sizeof(JSON_LONG) - 1; i++) { + bytes = flb_lib_push(ctx, in_ffd, p + i, 1); + TEST_CHECK(bytes == 1); + } + + sleep(1); /* waiting flush */ + + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_flowcounter_json_small(void) +{ + int i; + int ret; + int bytes; + char *p = (char *) JSON_SMALL; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + ctx = flb_create(); + flb_service_set(ctx, "Flush", "1", "Grace", "1", "Log_Level", "error", NULL); + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + out_ffd = flb_output(ctx, (char *) "flowcounter", NULL); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, "match", "test", NULL); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + for (i = 0; i < (int) sizeof(JSON_SMALL) - 1; i++) { + bytes = flb_lib_push(ctx, in_ffd, p + i, 1); + TEST_CHECK(bytes == 1); + } + + sleep(1); /* waiting flush */ + + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_flowcounter_unit_second(void) +{ + int i; + int ret; + int bytes; + char *p = (char *) JSON_SMALL; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + ctx = flb_create(); + flb_service_set(ctx, "Flush", "1", "Grace", "1", "Log_Level", "error", NULL); + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + out_ffd = flb_output(ctx, (char *) "flowcounter", NULL); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, "match", "test", NULL); + flb_output_set(ctx, out_ffd, "Unit", "second", NULL); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + for (i = 0; i < (int) sizeof(JSON_SMALL) - 1; i++) { + bytes = flb_lib_push(ctx, in_ffd, p + i, 1); + TEST_CHECK(bytes == 1); + } + + sleep(1); /* waiting flush */ + + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_flowcounter_unit_minute(void) +{ + int i; + int ret; + int bytes; + char *p = (char *) JSON_SMALL; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + ctx = flb_create(); + flb_service_set(ctx, "Flush", "1", "Grace", "1", "Log_Level", "error", NULL); + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + out_ffd = flb_output(ctx, (char *) "flowcounter", NULL); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, "match", "test", NULL); + flb_output_set(ctx, out_ffd, "Unit", "minute", NULL); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + for (i = 0; i < (int) sizeof(JSON_SMALL) - 1; i++) { + bytes = flb_lib_push(ctx, in_ffd, p + i, 1); + TEST_CHECK(bytes == 1); + } + + sleep(1); /* waiting flush */ + + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_flowcounter_unit_hour(void) +{ + int i; + int ret; + int bytes; + char *p = (char *) JSON_SMALL; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + ctx = flb_create(); + flb_service_set(ctx, "Flush", "1", "Grace", "1", "Log_Level", "error", NULL); + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + out_ffd = flb_output(ctx, (char *) "flowcounter", NULL); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, "match", "test", NULL); + flb_output_set(ctx, out_ffd, "Unit", "hour", NULL); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + for (i = 0; i < (int) sizeof(JSON_SMALL) - 1; i++) { + bytes = flb_lib_push(ctx, in_ffd, p + i, 1); + TEST_CHECK(bytes == 1); + } + + sleep(1); /* waiting flush */ + + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_flowcounter_unit_day(void) +{ + int i; + int ret; + int bytes; + char *p = (char *) JSON_SMALL; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + ctx = flb_create(); + flb_service_set(ctx, "Flush", "1", "Grace", "1", "Log_Level", "error", NULL); + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + out_ffd = flb_output(ctx, (char *) "flowcounter", NULL); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, "match", "test", NULL); + flb_output_set(ctx, out_ffd, "Unit", "day", NULL); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + for (i = 0; i < (int) sizeof(JSON_SMALL) - 1; i++) { + bytes = flb_lib_push(ctx, in_ffd, p + i, 1); + TEST_CHECK(bytes == 1); + } + + sleep(1); /* waiting flush */ + + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_flowcounter_unit_invalid(void) +{ + int i; + int ret; + int bytes; + char *p = (char *) JSON_SMALL; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + ctx = flb_create(); + flb_service_set(ctx, "Flush", "1", "Grace", "1", "Log_Level", "error", NULL); + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + out_ffd = flb_output(ctx, (char *) "flowcounter", NULL); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, "match", "test", NULL); + flb_output_set(ctx, out_ffd, "Unit", "xxx", NULL); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + for (i = 0; i < (int) sizeof(JSON_SMALL) - 1; i++) { + bytes = flb_lib_push(ctx, in_ffd, p + i, 1); + TEST_CHECK(bytes == 1); + } + + sleep(1); /* waiting flush */ + + flb_stop(ctx); + flb_destroy(ctx); +} diff --git a/src/fluent-bit/tests/runtime/out_forward.c b/src/fluent-bit/tests/runtime/out_forward.c new file mode 100644 index 000000000..7e3b90aec --- /dev/null +++ b/src/fluent-bit/tests/runtime/out_forward.c @@ -0,0 +1,364 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <fluent-bit.h> +#include <fluent-bit/flb_pack.h> +#include <fluent-bit/flb_record_accessor.h> + +#include "flb_tests_runtime.h" + +/* Include plugin header to get the flush_ctx structure definition */ +#include "../../plugins/out_forward/forward.h" + +static void cb_check_message_mode(void *ctx, int ffd, + int res_ret, void *res_data, size_t res_size, + void *data) +{ + int ret; + size_t off = 0; + msgpack_object tag; + msgpack_object ts; + msgpack_object record; + msgpack_object root; + msgpack_unpacked result; + struct flb_time time = {0}; + + msgpack_unpacked_init(&result); + ret = msgpack_unpack_next(&result, res_data, res_size, &off); + root = result.data; + + TEST_CHECK(ret == MSGPACK_UNPACK_SUCCESS); + TEST_CHECK(root.type == MSGPACK_OBJECT_ARRAY); + TEST_CHECK(root.via.array.size == 4); + + /* Tag */ + tag = root.via.array.ptr[0]; + TEST_CHECK(tag.type == MSGPACK_OBJECT_STR); + ret = strncmp(tag.via.str.ptr, "new.tag.fluent", tag.via.str.size); + TEST_CHECK(ret == 0); + + /* Timestamp */ + ts = root.via.array.ptr[1]; + TEST_CHECK(ts.type == MSGPACK_OBJECT_EXT); + + ret = flb_time_msgpack_to_time(&time, &ts); + TEST_CHECK(ret == 0); + TEST_CHECK(time.tm.tv_nsec != 0); + + /* Record */ + record = root.via.array.ptr[2]; + TEST_CHECK(record.type == MSGPACK_OBJECT_MAP); + TEST_CHECK(record.via.map.size == 2); + + msgpack_unpacked_destroy(&result); + flb_free(res_data); +} + +static void cb_check_message_compat_mode(void *ctx, int ffd, + int res_ret, void *res_data, size_t res_size, + void *data) +{ + int ret; + size_t off = 0; + msgpack_object tag; + msgpack_object ts; + msgpack_object record; + msgpack_object root; + msgpack_unpacked result; + struct flb_time time = {0}; + + msgpack_unpacked_init(&result); + ret = msgpack_unpack_next(&result, res_data, res_size, &off); + root = result.data; + + TEST_CHECK(ret == MSGPACK_UNPACK_SUCCESS); + TEST_CHECK(root.type == MSGPACK_OBJECT_ARRAY); + TEST_CHECK(root.via.array.size == 4); + + /* Tag */ + tag = root.via.array.ptr[0]; + TEST_CHECK(tag.type == MSGPACK_OBJECT_STR); + ret = strncmp(tag.via.str.ptr, "new.tag.fluent", tag.via.str.size); + TEST_CHECK(ret == 0); + + /* Timestamp */ + ts = root.via.array.ptr[1]; + TEST_CHECK(ts.type == MSGPACK_OBJECT_POSITIVE_INTEGER); + + ret = flb_time_msgpack_to_time(&time, &ts); + TEST_CHECK(ret == 0); + TEST_CHECK(time.tm.tv_nsec == 0); + + /* Record */ + record = root.via.array.ptr[2]; + TEST_CHECK(record.type == MSGPACK_OBJECT_MAP); + TEST_CHECK(record.via.map.size == 2); + + msgpack_unpacked_destroy(&result); + flb_free(res_data); +} + +static void cb_check_forward_mode(void *ctx, int ffd, + int res_ret, void *res_data, size_t res_size, + void *data) +{ + int ret; + size_t off = 0; + msgpack_object key; + msgpack_object val; + msgpack_object root; + msgpack_unpacked result; + + /* + * the check for forward mode is a bit special, since no data is formatted, instead the formatter callback + * will return the "options" map that will be send after the records chunk. The options are set because the + * caller specified 'send_options true'. + */ + TEST_CHECK(res_ret == MODE_FORWARD); + + msgpack_unpacked_init(&result); + ret = msgpack_unpack_next(&result, res_data, res_size, &off); + root = result.data; + + TEST_CHECK(ret == MSGPACK_UNPACK_SUCCESS); + TEST_CHECK(root.type == MSGPACK_OBJECT_MAP); + + /* fluent_signal and size */ + TEST_CHECK(root.via.map.size == 2); + + /* Record */ + key = root.via.map.ptr[1].key; + val = root.via.map.ptr[1].val; + + ret = strncmp(key.via.str.ptr, "fluent_signal", 13); + TEST_CHECK(ret == 0); + TEST_CHECK(val.type == MSGPACK_OBJECT_POSITIVE_INTEGER); + TEST_CHECK(val.via.u64 == 0); + + msgpack_unpacked_destroy(&result); + flb_free(res_data); +} + +static void cb_check_forward_compat_mode(void *ctx, int ffd, + int res_ret, void *res_data, size_t res_size, + void *data) +{ + int ret; + size_t off = 0; + msgpack_object root; + msgpack_object records; + msgpack_object entry; + msgpack_unpacked result; + + TEST_CHECK(res_ret == MODE_FORWARD_COMPAT); + + msgpack_unpacked_init(&result); + ret = msgpack_unpack_next(&result, res_data, res_size, &off); + root = result.data; + records = root.via.array.ptr[1]; + + TEST_CHECK(ret == MSGPACK_UNPACK_SUCCESS); + TEST_CHECK(root.type == MSGPACK_OBJECT_ARRAY); + + /* Record */ + entry = records.via.array.ptr[0]; + TEST_CHECK(entry.type == MSGPACK_OBJECT_ARRAY); + TEST_CHECK(entry.via.array.ptr[0].type == MSGPACK_OBJECT_POSITIVE_INTEGER); + + msgpack_unpacked_destroy(&result); + flb_free(res_data); +} + +void flb_test_message_mode() +{ + int ret; + int in_ffd; + int out_ffd; + flb_ctx_t *ctx; + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", "2", "grace", "1", NULL); + + /* Lib input mode */ + in_ffd = flb_input(ctx, (char *) "dummy", NULL); + flb_input_set(ctx, in_ffd, + "tag", "test", + "samples", "1", + "dummy", "{\"key1\": 123, \"key2\": {\"s1\": \"fluent\"}}", + NULL); + + + /* Forward output */ + out_ffd = flb_output(ctx, (char *) "forward", NULL); + flb_output_set(ctx, out_ffd, + "match", "test", + "tag", "new.tag.$key2['s1']", + NULL); + + /* Enable test mode */ + ret = flb_output_set_test(ctx, out_ffd, "formatter", + cb_check_message_mode, + NULL, NULL); + + /* Start */ + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_message_compat_mode() +{ + int ret; + int in_ffd; + int out_ffd; + flb_ctx_t *ctx; + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", "2", "grace", "1", NULL); + + /* Lib input mode */ + in_ffd = flb_input(ctx, (char *) "dummy", NULL); + flb_input_set(ctx, in_ffd, + "tag", "test", + "samples", "1", + "dummy", "{\"key1\": 123, \"key2\": {\"s1\": \"fluent\"}}", + NULL); + + + /* Forward output with timestamp in integer mode (compat) */ + out_ffd = flb_output(ctx, (char *) "forward", NULL); + flb_output_set(ctx, out_ffd, + "match", "test", + "time_as_integer", "true", + "tag", "new.tag.$key2['s1']", + NULL); + + /* Enable test mode */ + ret = flb_output_set_test(ctx, out_ffd, "formatter", + cb_check_message_compat_mode, + NULL, NULL); + + /* Start */ + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_forward_mode() +{ + int ret; + int in_ffd; + int out_ffd; + flb_ctx_t *ctx; + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", "2", "grace", "1", NULL); + + /* Lib input mode */ + in_ffd = flb_input(ctx, (char *) "dummy", NULL); + flb_input_set(ctx, in_ffd, + "tag", "test", + "samples", "1", + "dummy", "{\"key1\": 123, \"key2\": {\"s1\": \"fluent\"}}", + NULL); + + + /* Forward output: without a tag key access, forward mode is used */ + out_ffd = flb_output(ctx, (char *) "forward", NULL); + flb_output_set(ctx, out_ffd, + "match", "test", + "tag", "new.tag", + "send_options", "true", + NULL); + + /* Enable test mode */ + ret = flb_output_set_test(ctx, out_ffd, "formatter", + cb_check_forward_mode, + NULL, NULL); + + /* Start */ + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_forward_compat_mode() +{ + int ret; + int in_ffd; + int out_ffd; + flb_ctx_t *ctx; + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", "3", "grace", "1", NULL); + + /* Lib input mode */ + in_ffd = flb_input(ctx, (char *) "dummy", NULL); + flb_input_set(ctx, in_ffd, + "tag", "test", + "samples", "2", + "dummy", "{\"key1\": 123, \"key2\": {\"s1\": \"fluent\"}}", + NULL); + + + /* Forward output: without a tag key access, forward mode is used */ + out_ffd = flb_output(ctx, (char *) "forward", NULL); + flb_output_set(ctx, out_ffd, + "match", "test", + "tag", "new.tag", + "time_as_integer", "true", + NULL); + + /* Enable test mode */ + ret = flb_output_set_test(ctx, out_ffd, "formatter", + cb_check_forward_compat_mode, + NULL, NULL); + + /* Start */ + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + +/* Test list */ +TEST_LIST = { +#ifdef FLB_HAVE_RECORD_ACCESSOR + {"message_mode" , flb_test_message_mode }, + {"message_compat_mode", flb_test_message_compat_mode }, +#endif + {"forward_mode" , flb_test_forward_mode }, + {"forward_compat_mode", flb_test_forward_compat_mode }, + {NULL, NULL} +}; diff --git a/src/fluent-bit/tests/runtime/out_http.c b/src/fluent-bit/tests/runtime/out_http.c new file mode 100644 index 000000000..c898693c1 --- /dev/null +++ b/src/fluent-bit/tests/runtime/out_http.c @@ -0,0 +1,1060 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2019-2022 The Fluent Bit Authors + * Copyright (C) 2015-2018 Treasure Data Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <fluent-bit.h> +#include <fluent-bit/flb_sds.h> +#include <fluent-bit/flb_time.h> +#include <float.h> +#include <math.h> +#include <msgpack.h> +#include "flb_tests_runtime.h" + +struct test_ctx { + flb_ctx_t *flb; /* Fluent Bit library context */ + int i_ffd; /* Input fd */ + int f_ffd; /* Filter fd (unused) */ + int o_ffd; /* Output fd */ +}; + + +pthread_mutex_t result_mutex = PTHREAD_MUTEX_INITIALIZER; +int num_output = 0; +static int get_output_num() +{ + int ret; + pthread_mutex_lock(&result_mutex); + ret = num_output; + pthread_mutex_unlock(&result_mutex); + + return ret; +} + +static void set_output_num(int num) +{ + pthread_mutex_lock(&result_mutex); + num_output = num; + pthread_mutex_unlock(&result_mutex); +} + +static void clear_output_num() +{ + set_output_num(0); +} + +struct str_list { + size_t size; + char **lists; +}; + +/* Callback to check expected results */ +static void cb_check_str_list(void *ctx, int ffd, int res_ret, + void *res_data, size_t res_size, void *data) +{ + char *p; + flb_sds_t out_line = res_data; + int num = get_output_num(); + size_t i; + struct str_list *l = (struct str_list *)data; + + if (!TEST_CHECK(res_data != NULL)) { + TEST_MSG("res_data is NULL"); + return; + } + + if (!TEST_CHECK(l != NULL)) { + TEST_MSG("l is NULL"); + flb_sds_destroy(out_line); + return; + } + + if(!TEST_CHECK(res_ret == 0)) { + TEST_MSG("callback ret=%d", res_ret); + } + if (!TEST_CHECK(res_data != NULL)) { + TEST_MSG("res_data is NULL"); + flb_sds_destroy(out_line); + return; + } + + for (i=0; i<l->size; i++) { + p = strstr(out_line, l->lists[i]); + if (!TEST_CHECK(p != NULL)) { + TEST_MSG(" Got :%s\n expect:%s", out_line, l->lists[i]); + } + } + set_output_num(num+1); + + flb_sds_destroy(out_line); +} + +static int msgpack_strncmp(char* str, size_t str_len, msgpack_object obj) +{ + int ret = -1; + + if (str == NULL) { + flb_error("str is NULL"); + return -1; + } + + switch (obj.type) { + case MSGPACK_OBJECT_STR: + if (obj.via.str.size != str_len) { + return -1; + } + ret = strncmp(str, obj.via.str.ptr, str_len); + break; + case MSGPACK_OBJECT_POSITIVE_INTEGER: + { + unsigned long val = strtoul(str, NULL, 10); + if (val == (unsigned long)obj.via.u64) { + ret = 0; + } + } + break; + case MSGPACK_OBJECT_NEGATIVE_INTEGER: + { + long long val = strtoll(str, NULL, 10); + if (val == (unsigned long)obj.via.i64) { + ret = 0; + } + } + break; + case MSGPACK_OBJECT_FLOAT32: + case MSGPACK_OBJECT_FLOAT64: + { + double val = strtod(str, NULL); + if (fabs(val - obj.via.f64) < DBL_EPSILON) { + ret = 0; + } + } + break; + case MSGPACK_OBJECT_BOOLEAN: + if (obj.via.boolean) { + if (str_len != 4 /*true*/) { + return -1; + } + ret = strncasecmp(str, "true", 4); + } + else { + if (str_len != 5 /*false*/) { + return -1; + } + ret = strncasecmp(str, "false", 5); + } + break; + default: + flb_error("not supported"); + } + + return ret; +} + +/* Callback to check expected results */ +static void cb_check_msgpack_kv(void *ctx, int ffd, int res_ret, + void *res_data, size_t res_size, void *data) +{ + msgpack_unpacked result; + msgpack_object obj; + size_t off = 0; + struct str_list *l = (struct str_list *)data; + int i_map; + int map_size; + int i_list; + int num = get_output_num(); + + if (!TEST_CHECK(res_data != NULL)) { + TEST_MSG("res_data is NULL"); + return; + } + + if (!TEST_CHECK(data != NULL)) { + flb_error("data is NULL"); + return; + } + + /* Iterate each item array and apply rules */ + msgpack_unpacked_init(&result); + while (msgpack_unpack_next(&result, res_data, res_size, &off) == MSGPACK_UNPACK_SUCCESS) { + obj = result.data; + /* + msgpack_object_print(stdout, obj); + */ + if (obj.type != MSGPACK_OBJECT_ARRAY || obj.via.array.size != 2) { + flb_error("array error. type = %d", obj.type); + continue; + } + obj = obj.via.array.ptr[1]; + if (obj.type != MSGPACK_OBJECT_MAP) { + flb_error("map error. type = %d", obj.type); + continue; + } + map_size = obj.via.map.size; + for (i_map=0; i_map<map_size; i_map++) { + if (obj.via.map.ptr[i_map].key.type != MSGPACK_OBJECT_STR) { + flb_error("key is not string. type =%d", obj.via.map.ptr[i_map].key.type); + continue; + } + for (i_list=0; i_list< l->size/2; i_list++) { + if (msgpack_strncmp(l->lists[i_list*2], strlen(l->lists[i_list*2]), + obj.via.map.ptr[i_map].key) == 0 && + msgpack_strncmp(l->lists[i_list*2+1], strlen(l->lists[i_list*2+1]), + obj.via.map.ptr[i_map].val) == 0) { + num++; + } + } + } + } + set_output_num(num); + + msgpack_unpacked_destroy(&result); + + return ; +} + +static struct test_ctx *test_ctx_create() +{ + int i_ffd; + int o_ffd; + struct test_ctx *ctx = NULL; + + ctx = flb_malloc(sizeof(struct test_ctx)); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("malloc failed"); + flb_errno(); + return NULL; + } + + /* Service config */ + ctx->flb = flb_create(); + flb_service_set(ctx->flb, + "Flush", "0.200000000", + "Grace", "1", + "Log_Level", "error", + NULL); + + /* Input */ + i_ffd = flb_input(ctx->flb, (char *) "lib", NULL); + TEST_CHECK(i_ffd >= 0); + ctx->i_ffd = i_ffd; + + /* Output */ + o_ffd = flb_output(ctx->flb, (char *) "http", NULL); + ctx->o_ffd = o_ffd; + + return ctx; +} + +static void test_ctx_destroy(struct test_ctx *ctx) +{ + TEST_CHECK(ctx != NULL); + + sleep(1); + flb_stop(ctx->flb); + flb_destroy(ctx->flb); + flb_free(ctx); +} + +void flb_test_format_msgpack() +{ + struct test_ctx *ctx; + int ret; + int num; + + char *buf1 = "[1, {\"msg\":\"hello world\", \"val\":1000, \"nval\":-10000, \"bool\":true, \"float\":1.234}]"; + size_t size1 = strlen(buf1); + + char *expected_strs[] = {"msg", "hello world", "val", "1000", "nval", "-10000", "bool", "true", "float", "1.234"}; + struct str_list expected = { + .size = sizeof(expected_strs)/sizeof(char*), + .lists = &expected_strs[0], + }; + + clear_output_num(); + + ctx = test_ctx_create(); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("test_ctx_create failed"); + exit(EXIT_FAILURE); + } + + ret = flb_output_set(ctx->flb, ctx->o_ffd, + "match", "*", + "format", "msgpack", + NULL); + TEST_CHECK(ret == 0); + + ret = flb_output_set_test(ctx->flb, ctx->o_ffd, + "formatter", cb_check_msgpack_kv, + &expected, NULL); + TEST_CHECK(ret == 0); + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + ret = flb_lib_push(ctx->flb, ctx->i_ffd, (char *) buf1, size1); + TEST_CHECK(ret >= 0); + + /* waiting to flush */ + flb_time_msleep(500); + + num = get_output_num(); + if (!TEST_CHECK(num == expected.size / 2)) { + TEST_MSG("got %d, expected %lu", num, expected.size/2); + } + + test_ctx_destroy(ctx); +} + +void flb_test_format_json() +{ + struct test_ctx *ctx; + int ret; + int num; + + char *buf1 = "[1, {\"msg\":\"hello world\"}]"; + size_t size1 = strlen(buf1); + char *buf2 = "[2, {\"msg\":\"hello world\"}]"; + size_t size2 = strlen(buf2); + + char *expected_strs[] = {"[{\"date\":1.0,\"msg\":\"hello world\"},{\"date\":2.0,\"msg\":\"hello world\"}]"}; + struct str_list expected = { + .size = sizeof(expected_strs)/sizeof(char*), + .lists = &expected_strs[0], + }; + + clear_output_num(); + + ctx = test_ctx_create(); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("test_ctx_create failed"); + exit(EXIT_FAILURE); + } + + ret = flb_output_set(ctx->flb, ctx->o_ffd, + "match", "*", + "format", "json", + NULL); + TEST_CHECK(ret == 0); + + ret = flb_output_set_test(ctx->flb, ctx->o_ffd, + "formatter", cb_check_str_list, + &expected, NULL); + TEST_CHECK(ret == 0); + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + ret = flb_lib_push(ctx->flb, ctx->i_ffd, (char *) buf1, size1); + TEST_CHECK(ret >= 0); + ret = flb_lib_push(ctx->flb, ctx->i_ffd, (char *) buf2, size2); + TEST_CHECK(ret >= 0); + + /* waiting to flush */ + flb_time_msleep(500); + + num = get_output_num(); + if (!TEST_CHECK(num > 0)) { + TEST_MSG("no outputs"); + } + + test_ctx_destroy(ctx); +} + +void flb_test_format_json_stream() +{ + struct test_ctx *ctx; + int ret; + int num; + + char *buf1 = "[1, {\"msg\":\"hello world\"}]"; + size_t size1 = strlen(buf1); + char *buf2 = "[2, {\"msg\":\"hello world\"}]"; + size_t size2 = strlen(buf2); + + char *expected_strs[] = {"{\"date\":1.0,\"msg\":\"hello world\"}{\"date\":2.0,\"msg\":\"hello world\"}"}; + struct str_list expected = { + .size = sizeof(expected_strs)/sizeof(char*), + .lists = &expected_strs[0], + }; + + clear_output_num(); + + ctx = test_ctx_create(); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("test_ctx_create failed"); + exit(EXIT_FAILURE); + } + + ret = flb_output_set(ctx->flb, ctx->o_ffd, + "match", "*", + "format", "json_stream", + NULL); + TEST_CHECK(ret == 0); + + ret = flb_output_set_test(ctx->flb, ctx->o_ffd, + "formatter", cb_check_str_list, + &expected, NULL); + TEST_CHECK(ret == 0); + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + ret = flb_lib_push(ctx->flb, ctx->i_ffd, (char *) buf1, size1); + TEST_CHECK(ret >= 0); + ret = flb_lib_push(ctx->flb, ctx->i_ffd, (char *) buf2, size2); + TEST_CHECK(ret >= 0); + + /* waiting to flush */ + flb_time_msleep(500); + + num = get_output_num(); + if (!TEST_CHECK(num > 0)) { + TEST_MSG("no outputs"); + } + + test_ctx_destroy(ctx); +} + +void flb_test_format_json_lines() +{ + struct test_ctx *ctx; + int ret; + int num; + + char *buf1 = "[1, {\"msg\":\"hello world\"}]"; + size_t size1 = strlen(buf1); + char *buf2 = "[2, {\"msg\":\"hello world\"}]"; + size_t size2 = strlen(buf2); + + char *expected_strs[] = {"{\"date\":1.0,\"msg\":\"hello world\"}\n{\"date\":2.0,\"msg\":\"hello world\"}"}; + struct str_list expected = { + .size = sizeof(expected_strs)/sizeof(char*), + .lists = &expected_strs[0], + }; + + clear_output_num(); + + ctx = test_ctx_create(); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("test_ctx_create failed"); + exit(EXIT_FAILURE); + } + + ret = flb_output_set(ctx->flb, ctx->o_ffd, + "match", "*", + "format", "json_lines", + NULL); + TEST_CHECK(ret == 0); + + ret = flb_output_set_test(ctx->flb, ctx->o_ffd, + "formatter", cb_check_str_list, + &expected, NULL); + TEST_CHECK(ret == 0); + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + ret = flb_lib_push(ctx->flb, ctx->i_ffd, (char *) buf1, size1); + TEST_CHECK(ret >= 0); + ret = flb_lib_push(ctx->flb, ctx->i_ffd, (char *) buf2, size2); + TEST_CHECK(ret >= 0); + + /* waiting to flush */ + flb_time_msleep(500); + + num = get_output_num(); + if (!TEST_CHECK(num > 0)) { + TEST_MSG("no outputs"); + } + + test_ctx_destroy(ctx); +} + +void flb_test_format_gelf() +{ + struct test_ctx *ctx; + int ret; + int num; + + char *buf1 = "[1, {\"msg\":\"hello world\"}]"; + size_t size1 = strlen(buf1); + + char *expected_strs[] = {"\"short_message\":\"hello world\"", "\"timestamp\":1.000"}; + struct str_list expected = { + .size = sizeof(expected_strs)/sizeof(char*), + .lists = &expected_strs[0], + }; + + clear_output_num(); + + ctx = test_ctx_create(); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("test_ctx_create failed"); + exit(EXIT_FAILURE); + } + + ret = flb_output_set(ctx->flb, ctx->o_ffd, + "match", "*", + "format", "gelf", + "gelf_short_message_key", "msg", + NULL); + TEST_CHECK(ret == 0); + + ret = flb_output_set_test(ctx->flb, ctx->o_ffd, + "formatter", cb_check_str_list, + &expected, NULL); + TEST_CHECK(ret == 0); + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + ret = flb_lib_push(ctx->flb, ctx->i_ffd, (char *) buf1, size1); + TEST_CHECK(ret >= 0); + + /* waiting to flush */ + flb_time_msleep(500); + + num = get_output_num(); + if (!TEST_CHECK(num > 0)) { + TEST_MSG("no outputs"); + } + + test_ctx_destroy(ctx); +} + + +void flb_test_format_gelf_host_key() +{ + struct test_ctx *ctx; + int ret; + int num; + + char *buf1 = "[1, {\"msg\":\"hello world\", \"h_key\":\"localhost\"}]"; + size_t size1 = strlen(buf1); + + char *expected_strs[] = {"\"short_message\":\"hello world\"", "\"timestamp\":1.000", "\"host\":\"localhost\""}; + struct str_list expected = { + .size = sizeof(expected_strs)/sizeof(char*), + .lists = &expected_strs[0], + }; + + clear_output_num(); + + ctx = test_ctx_create(); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("test_ctx_create failed"); + exit(EXIT_FAILURE); + } + + ret = flb_output_set(ctx->flb, ctx->o_ffd, + "match", "*", + "format", "gelf", + "gelf_short_message_key", "msg", + "gelf_host_key", "h_key", + NULL); + TEST_CHECK(ret == 0); + + ret = flb_output_set_test(ctx->flb, ctx->o_ffd, + "formatter", cb_check_str_list, + &expected, NULL); + TEST_CHECK(ret == 0); + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + ret = flb_lib_push(ctx->flb, ctx->i_ffd, (char *) buf1, size1); + TEST_CHECK(ret >= 0); + + /* waiting to flush */ + flb_time_msleep(500); + + num = get_output_num(); + if (!TEST_CHECK(num > 0)) { + TEST_MSG("no outputs"); + } + + test_ctx_destroy(ctx); +} + +void flb_test_format_gelf_timestamp_key() +{ + struct test_ctx *ctx; + int ret; + int num; + + char *buf1 = "[1, {\"msg\":\"hello world\", \"t_key\":\"2018-05-30T09:39:52.000681Z\"}]"; + size_t size1 = strlen(buf1); + + char *expected_strs[] = {"\"short_message\":\"hello world\"", "\"timestamp\":\"2018-05-30T09:39:52.000681Z\""}; + struct str_list expected = { + .size = sizeof(expected_strs)/sizeof(char*), + .lists = &expected_strs[0], + }; + + clear_output_num(); + + ctx = test_ctx_create(); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("test_ctx_create failed"); + exit(EXIT_FAILURE); + } + + ret = flb_output_set(ctx->flb, ctx->o_ffd, + "match", "*", + "format", "gelf", + "gelf_short_message_key", "msg", + "gelf_timestamp_key", "t_key", + NULL); + TEST_CHECK(ret == 0); + + ret = flb_output_set_test(ctx->flb, ctx->o_ffd, + "formatter", cb_check_str_list, + &expected, NULL); + TEST_CHECK(ret == 0); + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + ret = flb_lib_push(ctx->flb, ctx->i_ffd, (char *) buf1, size1); + TEST_CHECK(ret >= 0); + + /* waiting to flush */ + flb_time_msleep(500); + + num = get_output_num(); + if (!TEST_CHECK(num > 0)) { + TEST_MSG("no outputs"); + } + + test_ctx_destroy(ctx); +} + +void flb_test_format_gelf_full_message_key() +{ + struct test_ctx *ctx; + int ret; + int num; + + char *buf1 = "[1, {\"msg\":\"hello world\", \"f_msg\":\"full message\"}]"; + size_t size1 = strlen(buf1); + + char *expected_strs[] = {"\"short_message\":\"hello world\"", "\"timestamp\":1.000","\"full_message\":\"full message\""}; + struct str_list expected = { + .size = sizeof(expected_strs)/sizeof(char*), + .lists = &expected_strs[0], + }; + + clear_output_num(); + + ctx = test_ctx_create(); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("test_ctx_create failed"); + exit(EXIT_FAILURE); + } + + ret = flb_output_set(ctx->flb, ctx->o_ffd, + "match", "*", + "format", "gelf", + "gelf_short_message_key", "msg", + "gelf_full_message_key", "f_msg", + NULL); + TEST_CHECK(ret == 0); + + ret = flb_output_set_test(ctx->flb, ctx->o_ffd, + "formatter", cb_check_str_list, + &expected, NULL); + TEST_CHECK(ret == 0); + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + ret = flb_lib_push(ctx->flb, ctx->i_ffd, (char *) buf1, size1); + TEST_CHECK(ret >= 0); + + /* waiting to flush */ + flb_time_msleep(500); + + num = get_output_num(); + if (!TEST_CHECK(num > 0)) { + TEST_MSG("no outputs"); + } + + test_ctx_destroy(ctx); +} + +void flb_test_format_gelf_level_key() +{ + struct test_ctx *ctx; + int ret; + int num; + + char *buf1 = "[1, {\"msg\":\"hello world\", \"l_msg\":6\"}]"; + size_t size1 = strlen(buf1); + + char *expected_strs[] = {"\"short_message\":\"hello world\"", "\"timestamp\":1.000","\"level\":6"}; + struct str_list expected = { + .size = sizeof(expected_strs)/sizeof(char*), + .lists = &expected_strs[0], + }; + + clear_output_num(); + + ctx = test_ctx_create(); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("test_ctx_create failed"); + exit(EXIT_FAILURE); + } + + ret = flb_output_set(ctx->flb, ctx->o_ffd, + "match", "*", + "format", "gelf", + "gelf_short_message_key", "msg", + "gelf_level_key", "l_msg", + NULL); + TEST_CHECK(ret == 0); + + ret = flb_output_set_test(ctx->flb, ctx->o_ffd, + "formatter", cb_check_str_list, + &expected, NULL); + TEST_CHECK(ret == 0); + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + ret = flb_lib_push(ctx->flb, ctx->i_ffd, (char *) buf1, size1); + TEST_CHECK(ret >= 0); + + /* waiting to flush */ + flb_time_msleep(500); + + num = get_output_num(); + if (!TEST_CHECK(num > 0)) { + TEST_MSG("no outputs"); + } + + test_ctx_destroy(ctx); +} + +void flb_test_set_json_date_key() +{ + struct test_ctx *ctx; + int ret; + int num; + + char *buf1 = "[1, {\"msg\":\"hello world\"}]"; + size_t size1 = strlen(buf1); + + char *expected_strs[] = {"{\"timestamp\":1.0,\"msg\":\"hello world\"}"}; + struct str_list expected = { + .size = sizeof(expected_strs)/sizeof(char*), + .lists = &expected_strs[0], + }; + + clear_output_num(); + + ctx = test_ctx_create(); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("test_ctx_create failed"); + exit(EXIT_FAILURE); + } + + ret = flb_output_set(ctx->flb, ctx->o_ffd, + "match", "*", + "format", "json", + "json_date_key", "timestamp", + NULL); + TEST_CHECK(ret == 0); + + ret = flb_output_set_test(ctx->flb, ctx->o_ffd, + "formatter", cb_check_str_list, + &expected, NULL); + TEST_CHECK(ret == 0); + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + ret = flb_lib_push(ctx->flb, ctx->i_ffd, (char *) buf1, size1); + TEST_CHECK(ret >= 0); + + /* waiting to flush */ + flb_time_msleep(500); + + num = get_output_num(); + if (!TEST_CHECK(num > 0)) { + TEST_MSG("no outputs"); + } + + test_ctx_destroy(ctx); +} + +void flb_test_disable_json_date_key() +{ + struct test_ctx *ctx; + int ret; + int num; + + char *buf1 = "[1, {\"msg\":\"hello world\"}]"; + size_t size1 = strlen(buf1); + + char *expected_strs[] = {"{\"msg\":\"hello world\"}"}; + struct str_list expected = { + .size = sizeof(expected_strs)/sizeof(char*), + .lists = &expected_strs[0], + }; + + clear_output_num(); + + ctx = test_ctx_create(); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("test_ctx_create failed"); + exit(EXIT_FAILURE); + } + + ret = flb_output_set(ctx->flb, ctx->o_ffd, + "match", "*", + "format", "json", + "json_date_key", "false", + NULL); + TEST_CHECK(ret == 0); + + ret = flb_output_set_test(ctx->flb, ctx->o_ffd, + "formatter", cb_check_str_list, + &expected, NULL); + TEST_CHECK(ret == 0); + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + ret = flb_lib_push(ctx->flb, ctx->i_ffd, (char *) buf1, size1); + TEST_CHECK(ret >= 0); + + /* waiting to flush */ + flb_time_msleep(500); + + num = get_output_num(); + if (!TEST_CHECK(num > 0)) { + TEST_MSG("no outputs"); + } + + test_ctx_destroy(ctx); +} + +void flb_test_json_date_format_epoch() +{ + struct test_ctx *ctx; + int ret; + int num; + + char *buf1 = "[1, {\"msg\":\"hello world\"}]"; + size_t size1 = strlen(buf1); + + char *expected_strs[] = {"{\"date\":1,\"msg\":\"hello world\"}"}; + struct str_list expected = { + .size = sizeof(expected_strs)/sizeof(char*), + .lists = &expected_strs[0], + }; + + clear_output_num(); + + ctx = test_ctx_create(); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("test_ctx_create failed"); + exit(EXIT_FAILURE); + } + + ret = flb_output_set(ctx->flb, ctx->o_ffd, + "match", "*", + "format", "json", + "json_date_format", "epoch", + NULL); + TEST_CHECK(ret == 0); + + ret = flb_output_set_test(ctx->flb, ctx->o_ffd, + "formatter", cb_check_str_list, + &expected, NULL); + TEST_CHECK(ret == 0); + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + ret = flb_lib_push(ctx->flb, ctx->i_ffd, (char *) buf1, size1); + TEST_CHECK(ret >= 0); + + /* waiting to flush */ + flb_time_msleep(500); + + num = get_output_num(); + if (!TEST_CHECK(num > 0)) { + TEST_MSG("no outputs"); + } + + test_ctx_destroy(ctx); +} + +void flb_test_json_date_format_iso8601() +{ + struct test_ctx *ctx; + int ret; + int num; + + char *buf1 = "[1, {\"msg\":\"hello world\"}]"; + size_t size1 = strlen(buf1); + + char *expected_strs[] = {"\"msg\":\"hello world\"", "\"date\":\"1970-01-01T00:00:01.000000Z\""}; + struct str_list expected = { + .size = sizeof(expected_strs)/sizeof(char*), + .lists = &expected_strs[0], + }; + + clear_output_num(); + + ctx = test_ctx_create(); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("test_ctx_create failed"); + exit(EXIT_FAILURE); + } + + ret = flb_output_set(ctx->flb, ctx->o_ffd, + "match", "*", + "format", "json", + "json_date_format", "iso8601", + NULL); + TEST_CHECK(ret == 0); + + ret = flb_output_set_test(ctx->flb, ctx->o_ffd, + "formatter", cb_check_str_list, + &expected, NULL); + TEST_CHECK(ret == 0); + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + ret = flb_lib_push(ctx->flb, ctx->i_ffd, (char *) buf1, size1); + TEST_CHECK(ret >= 0); + + /* waiting to flush */ + flb_time_msleep(500); + + num = get_output_num(); + if (!TEST_CHECK(num > 0)) { + TEST_MSG("no outputs"); + } + + test_ctx_destroy(ctx); +} + +void flb_test_json_date_format_java_sql_timestamp() +{ + struct test_ctx *ctx; + int ret; + int num; + + char *buf1 = "[1, {\"msg\":\"hello world\"}]"; + size_t size1 = strlen(buf1); + + char *expected_strs[] = {"\"msg\":\"hello world\"", "\"date\":\"1970-01-01 00:00:01.000000\""}; + struct str_list expected = { + .size = sizeof(expected_strs)/sizeof(char*), + .lists = &expected_strs[0], + }; + + clear_output_num(); + + ctx = test_ctx_create(); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("test_ctx_create failed"); + exit(EXIT_FAILURE); + } + + ret = flb_output_set(ctx->flb, ctx->o_ffd, + "match", "*", + "format", "json", + "json_date_format", "java_sql_timestamp", + NULL); + TEST_CHECK(ret == 0); + + ret = flb_output_set_test(ctx->flb, ctx->o_ffd, + "formatter", cb_check_str_list, + &expected, NULL); + TEST_CHECK(ret == 0); + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + ret = flb_lib_push(ctx->flb, ctx->i_ffd, (char *) buf1, size1); + TEST_CHECK(ret >= 0); + + /* waiting to flush */ + flb_time_msleep(500); + + num = get_output_num(); + if (!TEST_CHECK(num > 0)) { + TEST_MSG("no outputs"); + } + + test_ctx_destroy(ctx); +} + +/* Test list */ +TEST_LIST = { + {"format_msgpack" , flb_test_format_msgpack}, + {"format_json" , flb_test_format_json}, + {"format_json_stream" , flb_test_format_json_stream}, + {"format_json_lines" , flb_test_format_json_lines}, + {"format_gelf" , flb_test_format_gelf}, + {"format_gelf_host_key" , flb_test_format_gelf_host_key}, + {"format_gelf_timestamp_key" , flb_test_format_gelf_timestamp_key}, + {"format_gelf_full_message_key" , flb_test_format_gelf_full_message_key}, + {"format_gelf_level_key" , flb_test_format_gelf_level_key}, + {"set_json_date_key" , flb_test_set_json_date_key}, + {"disable_json_date_key" , flb_test_disable_json_date_key}, + {"json_date_format_epoch" , flb_test_json_date_format_epoch}, + {"json_date_format_iso8601" , flb_test_json_date_format_iso8601}, + {"json_date_format_java_sql_timestamp" , flb_test_json_date_format_java_sql_timestamp}, + {NULL, NULL} +}; diff --git a/src/fluent-bit/tests/runtime/out_kinesis.c b/src/fluent-bit/tests/runtime/out_kinesis.c new file mode 100644 index 000000000..da3e925a0 --- /dev/null +++ b/src/fluent-bit/tests/runtime/out_kinesis.c @@ -0,0 +1,200 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ +#include <fluent-bit.h> +#include "flb_tests_runtime.h" + +/* Test data */ +#include "data/td/json_td.h" /* JSON_TD */ + +#define ERROR_THROUGHPUT "{\"__type\":\"ServiceUnavailableException\"}" +/* not a real error code, but tests that the code can respond to any error */ +#define ERROR_UNKNOWN "{\"__type\":\"UNKNOWN\"}" + +/* It writes a big JSON message (copied from TD test) */ +void flb_test_firehose_success(void) +{ + int ret; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + /* mocks calls- signals that we are in test mode */ + setenv("FLB_KINESIS_PLUGIN_UNDER_TEST", "true", 1); + + ctx = flb_create(); + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx,in_ffd, "tag", "test", NULL); + + out_ffd = flb_output(ctx, (char *) "kinesis_streams", NULL); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd,"match", "*", NULL); + flb_output_set(ctx, out_ffd,"region", "us-west-2", NULL); + flb_output_set(ctx, out_ffd,"stream", "fluent", NULL); + flb_output_set(ctx, out_ffd,"time_key", "time", NULL); + flb_output_set(ctx, out_ffd,"Retry_Limit", "1", NULL); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + flb_lib_push(ctx, in_ffd, (char *) JSON_TD , (int) sizeof(JSON_TD) - 1); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_firehose_partial_success(void) +{ + int ret; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + /* mocks calls- signals that we are in test mode */ + setenv("FLB_KINESIS_PLUGIN_UNDER_TEST", "true", 1); + setenv("PARTIAL_SUCCESS_CASE", "true", 1); + + ctx = flb_create(); + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx,in_ffd, "tag", "test", NULL); + + out_ffd = flb_output(ctx, (char *) "kinesis_streams", NULL); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd,"match", "*", NULL); + flb_output_set(ctx, out_ffd,"region", "us-west-2", NULL); + flb_output_set(ctx, out_ffd,"stream", "fluent", NULL); + flb_output_set(ctx, out_ffd,"time_key", "time", NULL); + flb_output_set(ctx, out_ffd,"Retry_Limit", "1", NULL); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + flb_lib_push(ctx, in_ffd, (char *) JSON_TD , (int) sizeof(JSON_TD) - 1); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); + unsetenv("PARTIAL_SUCCESS_CASE"); +} + +void flb_test_firehose_throughput_error(void) +{ + int ret; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + /* mocks calls- signals that we are in test mode */ + setenv("FLB_KINESIS_PLUGIN_UNDER_TEST", "true", 1); + setenv("TEST_PUT_RECORDS_ERROR", ERROR_THROUGHPUT, 1); + + ctx = flb_create(); + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx,in_ffd, "tag", "test", NULL); + + out_ffd = flb_output(ctx, (char *) "kinesis_streams", NULL); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd,"match", "*", NULL); + flb_output_set(ctx, out_ffd,"region", "us-west-2", NULL); + flb_output_set(ctx, out_ffd,"stream", "fluent", NULL); + flb_output_set(ctx, out_ffd,"time_key", "time", NULL); + flb_output_set(ctx, out_ffd,"Retry_Limit", "1", NULL); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + flb_lib_push(ctx, in_ffd, (char *) JSON_TD , (int) sizeof(JSON_TD) - 1); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); + unsetenv("TEST_PUT_RECORDS_ERROR"); +} + +void flb_test_firehose_error_unknown(void) +{ + int ret; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + /* mocks calls- signals that we are in test mode */ + setenv("FLB_KINESIS_PLUGIN_UNDER_TEST", "true", 1); + setenv("TEST_PUT_RECORDS_ERROR", ERROR_UNKNOWN, 1); + + ctx = flb_create(); + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx,in_ffd, "tag", "test", NULL); + + out_ffd = flb_output(ctx, (char *) "kinesis_streams", NULL); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd,"match", "*", NULL); + flb_output_set(ctx, out_ffd,"region", "us-west-2", NULL); + flb_output_set(ctx, out_ffd,"stream", "fluent", NULL); + flb_output_set(ctx, out_ffd,"time_key", "time", NULL); + flb_output_set(ctx, out_ffd,"Retry_Limit", "1", NULL); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + flb_lib_push(ctx, in_ffd, (char *) JSON_TD , (int) sizeof(JSON_TD) - 1); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); + unsetenv("TEST_PUT_RECORDS_ERROR"); +} + +void flb_test_firehose_nonsense_error(void) +{ + int ret; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + /* mocks calls- signals that we are in test mode */ + setenv("FLB_KINESIS_PLUGIN_UNDER_TEST", "true", 1); + setenv("TEST_PUT_RECORDS_ERROR", "\tbadresponse\nnotparsable{}", 1); + + ctx = flb_create(); + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx,in_ffd, "tag", "test", NULL); + + out_ffd = flb_output(ctx, (char *) "kinesis_streams", NULL); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd,"match", "*", NULL); + flb_output_set(ctx, out_ffd,"region", "us-west-2", NULL); + flb_output_set(ctx, out_ffd,"stream", "fluent", NULL); + flb_output_set(ctx, out_ffd,"time_key", "time", NULL); + flb_output_set(ctx, out_ffd,"Retry_Limit", "1", NULL); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + flb_lib_push(ctx, in_ffd, (char *) JSON_TD , (int) sizeof(JSON_TD) - 1); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); + unsetenv("TEST_PUT_RECORDS_ERROR"); +} + + +/* Test list */ +TEST_LIST = { + {"success", flb_test_firehose_success }, + {"partial_success", flb_test_firehose_partial_success }, + {"throughput_error", flb_test_firehose_throughput_error }, + {"unknown_error", flb_test_firehose_error_unknown }, + {"nonsense_error", flb_test_firehose_nonsense_error }, + {NULL, NULL} +}; diff --git a/src/fluent-bit/tests/runtime/out_lib.c b/src/fluent-bit/tests/runtime/out_lib.c new file mode 100644 index 000000000..580a480ff --- /dev/null +++ b/src/fluent-bit/tests/runtime/out_lib.c @@ -0,0 +1,587 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2019-2022 The Fluent Bit Authors + * Copyright (C) 2015-2018 Treasure Data Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <fluent-bit.h> +#include <fluent-bit/flb_compat.h> +#include <fluent-bit/flb_time.h> +#include <float.h> +#include <math.h> +#include "flb_tests_runtime.h" + +struct test_ctx { + flb_ctx_t *flb; /* Fluent Bit library context */ + int i_ffd; /* Input fd */ + int f_ffd; /* Filter fd (unused) */ + int o_ffd; /* Output fd */ +}; + + +pthread_mutex_t result_mutex = PTHREAD_MUTEX_INITIALIZER; +int num_output = 0; +static int get_output_num() +{ + int ret; + pthread_mutex_lock(&result_mutex); + ret = num_output; + pthread_mutex_unlock(&result_mutex); + + return ret; +} + +static void set_output_num(int num) +{ + pthread_mutex_lock(&result_mutex); + num_output = num; + pthread_mutex_unlock(&result_mutex); +} + +static void clear_output_num() +{ + set_output_num(0); +} + +void wait_with_timeout(uint32_t timeout_ms, int *output_num) +{ + struct flb_time start_time; + struct flb_time end_time; + struct flb_time diff_time; + uint64_t elapsed_time_flb = 0; + + flb_time_get(&start_time); + + while (true) { + *output_num = get_output_num(); + + if (*output_num > 0) { + break; + } + + flb_time_msleep(100); + flb_time_get(&end_time); + flb_time_diff(&end_time, &start_time, &diff_time); + elapsed_time_flb = flb_time_to_nanosec(&diff_time) / 1000000; + + if (elapsed_time_flb > timeout_ms) { + flb_warn("[timeout] elapsed_time: %ld", elapsed_time_flb); + // Reached timeout. + break; + } + } +} + +struct str_list { + size_t size; + char **lists; +}; + +/* Callback to check expected results */ +static int cb_check_json_str_list(void *record, size_t size, void *data) +{ + char *p; + char *result; + int num = get_output_num(); + size_t i; + struct str_list *l = (struct str_list*)data; + + if (!TEST_CHECK(l != NULL)) { + TEST_MSG("Data is NULL"); + flb_free(record); + return 0; + } + + set_output_num(num+1); + + result = (char *) record; + + for (i=0; i<l->size; i++) { + p = strstr(result, l->lists[i]); + if(!TEST_CHECK(p != NULL)) { + TEST_MSG("Expected to find: '%s' in result '%s'", + l->lists[i], result); + } + } + + /* + * If you want to debug your test + * + * printf("Expect: '%s' in result '%s'", expected, result); + */ + flb_free(record); + return 0; +} + +static int msgpack_strncmp(char* str, size_t str_len, msgpack_object obj) +{ + int ret = -1; + + if (str == NULL) { + flb_error("str is NULL"); + return -1; + } + + switch (obj.type) { + case MSGPACK_OBJECT_STR: + if (obj.via.str.size != str_len) { + return -1; + } + ret = strncmp(str, obj.via.str.ptr, str_len); + break; + case MSGPACK_OBJECT_POSITIVE_INTEGER: + { + unsigned long val = strtoul(str, NULL, 10); + if (val == (unsigned long)obj.via.u64) { + ret = 0; + } + } + break; + case MSGPACK_OBJECT_NEGATIVE_INTEGER: + { + long long val = strtoll(str, NULL, 10); + if (val == (unsigned long)obj.via.i64) { + ret = 0; + } + } + break; + case MSGPACK_OBJECT_FLOAT32: + case MSGPACK_OBJECT_FLOAT64: + { + double val = strtod(str, NULL); + if (fabs(val - obj.via.f64) < DBL_EPSILON) { + ret = 0; + } + } + break; + case MSGPACK_OBJECT_BOOLEAN: + if (obj.via.boolean) { + if (str_len != 4 /*true*/) { + return -1; + } + ret = strncasecmp(str, "true", 4); + } + else { + if (str_len != 5 /*false*/) { + return -1; + } + ret = strncasecmp(str, "false", 5); + } + break; + default: + flb_error("not supported"); + } + + return ret; +} + +/* Callback to check expected results */ +static int cb_check_msgpack_kv(void *record, size_t size, void *data) + + /*void *ctx, int ffd, int res_ret, + void *res_data, size_t res_size, void *data)*/ +{ + msgpack_unpacked result; + msgpack_object obj; + size_t off = 0; + struct str_list *l = (struct str_list *)data; + int i_map; + int map_size; + int i_list; + int num = get_output_num(); + + if (!TEST_CHECK(record != NULL)) { + TEST_MSG("record is NULL"); + return -1; + } + + if (!TEST_CHECK(data != NULL)) { + TEST_MSG("data is NULL"); + flb_free(record); + return -1; + } + + /* Iterate each item array and apply rules */ + msgpack_unpacked_init(&result); + while (msgpack_unpack_next(&result, record, size, &off) == MSGPACK_UNPACK_SUCCESS) { + obj = result.data; + /* + msgpack_object_print(stdout, obj); + */ + if (obj.type != MSGPACK_OBJECT_ARRAY || obj.via.array.size != 2) { + flb_error("array error. type = %d", obj.type); + continue; + } + obj = obj.via.array.ptr[1]; + if (obj.type != MSGPACK_OBJECT_MAP) { + flb_error("map error. type = %d", obj.type); + continue; + } + map_size = obj.via.map.size; + for (i_map=0; i_map<map_size; i_map++) { + if (obj.via.map.ptr[i_map].key.type != MSGPACK_OBJECT_STR) { + flb_error("key is not string. type =%d", obj.via.map.ptr[i_map].key.type); + continue; + } + for (i_list=0; i_list< l->size/2; i_list++) { + if (msgpack_strncmp(l->lists[i_list*2], strlen(l->lists[i_list*2]), + obj.via.map.ptr[i_map].key) == 0 && + msgpack_strncmp(l->lists[i_list*2+1], strlen(l->lists[i_list*2+1]), + obj.via.map.ptr[i_map].val) == 0) { + num++; + } + } + } + } + set_output_num(num); + + msgpack_unpacked_destroy(&result); + flb_free(record); + + return 0; +} + + +static int cb_count_msgpack(void *record, size_t size, void *data) +{ + msgpack_unpacked result; + size_t off = 0; + + if (!TEST_CHECK(data != NULL)) { + TEST_MSG("data is NULL"); + } + + /* Iterate each item array and apply rules */ + msgpack_unpacked_init(&result); + while (msgpack_unpack_next(&result, record, size, &off) == MSGPACK_UNPACK_SUCCESS) { + pthread_mutex_lock(&result_mutex); + num_output++; + /* + msgpack_object_print(stdout, result.data); + */ + pthread_mutex_unlock(&result_mutex); + } + msgpack_unpacked_destroy(&result); + + flb_free(record); + return 0; +} + +static int cb_count(void *record, size_t size, void *data) +{ + if (!TEST_CHECK(data != NULL)) { + TEST_MSG("data is NULL"); + } + pthread_mutex_lock(&result_mutex); + num_output++; + pthread_mutex_unlock(&result_mutex); + + flb_free(record); + return 0; +} + +static struct test_ctx *test_ctx_create(struct flb_lib_out_cb *data) +{ + int o_ffd; + struct test_ctx *ctx = NULL; + + ctx = flb_malloc(sizeof(struct test_ctx)); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("malloc failed"); + flb_errno(); + return NULL; + } + + /* Service config */ + ctx->flb = flb_create(); + flb_service_set(ctx->flb, + "Flush", "0.200000000", + "Grace", "1", + "Log_Level", "error", + NULL); + + /* Output */ + o_ffd = flb_output(ctx->flb, (char *) "lib", (void *) data); + ctx->o_ffd = o_ffd; + + return ctx; +} + +static void test_ctx_destroy(struct test_ctx *ctx) +{ + TEST_CHECK(ctx != NULL); + + sleep(1); + flb_stop(ctx->flb); + flb_destroy(ctx->flb); + flb_free(ctx); +} + +static void test_format_json(void) +{ + struct flb_lib_out_cb cb_data; + struct test_ctx *ctx; + char *input_json = "[1,{\"hoge\":\"moge\", \"bool\":true, \"int\":100, \"float\":-2.0}]"; + int size = strlen(input_json); + int ret; + int num; + + char *expected_strs[] = {"\"hoge\"", "\"moge\"", "\"bool\"", "true", "\"int\"", "100", "\"float\"", "-2."}; + struct str_list expected = { + .size = sizeof(expected_strs)/sizeof(char*), + .lists = &expected_strs[0], + }; + + clear_output_num(); + + cb_data.cb = cb_check_json_str_list; + cb_data.data = &expected; + + ctx = test_ctx_create(&cb_data); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("test_ctx_create failed"); + exit(EXIT_FAILURE); + } + /* Input */ + ctx->i_ffd = flb_input(ctx->flb, (char *) "lib", NULL); + TEST_CHECK(ctx->i_ffd >= 0); + + ret = flb_output_set(ctx->flb, ctx->o_ffd, + "format", "json", + NULL); + TEST_CHECK(ret == 0); + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + ret = flb_lib_push(ctx->flb, ctx->i_ffd, input_json,size); + TEST_CHECK(ret >= 0); + + /* waiting to flush */ + wait_with_timeout(2000, &num); + + if (!TEST_CHECK(num > 0)) { + TEST_MSG("no outputs"); + } + + test_ctx_destroy(ctx); +} + +static void test_format_msgpack(void) +{ + struct flb_lib_out_cb cb_data; + struct test_ctx *ctx; + int ret; + int num; + char *input_json = "[1,{\"hoge\":\"moge\", \"bool\":true, \"int\":100, \"float\":-2.0}]"; + int size = strlen(input_json); + + char *expected_strs[] = {"hoge", "moge", "bool", "true", "int", "100", "float", "-2.0"}; + struct str_list expected = { + .size = sizeof(expected_strs)/sizeof(char*), + .lists = &expected_strs[0], + }; + + clear_output_num(); + + cb_data.cb = cb_check_msgpack_kv; + cb_data.data = &expected; + + ctx = test_ctx_create(&cb_data); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("test_ctx_create failed"); + exit(EXIT_FAILURE); + } + /* Input */ + ctx->i_ffd = flb_input(ctx->flb, (char *) "lib", NULL); + TEST_CHECK(ctx->i_ffd >= 0); + + ret = flb_output_set(ctx->flb, ctx->o_ffd, + "format", "msgpack", + NULL); + TEST_CHECK(ret == 0); + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + ret = flb_lib_push(ctx->flb, ctx->i_ffd, input_json,size); + TEST_CHECK(ret >= 0); + + /* waiting to flush */ + wait_with_timeout(2000, &num); + + if (!TEST_CHECK(num > 0)) { + TEST_MSG("no outputs"); + } + + test_ctx_destroy(ctx); +} + +static void test_max_records(void) +{ + struct flb_lib_out_cb cb_data; + struct test_ctx *ctx; + int ret; + int num; + char *input_json = "[1,{\"hoge\":\"moge\", \"bool\":true, \"int\":100, \"float\":-2.0}]"; + int size = strlen(input_json); + int i; + int unused; + int expected = 5 /* max_records */; + + clear_output_num(); + + cb_data.cb = cb_count_msgpack; + cb_data.data = &unused; + + ctx = test_ctx_create(&cb_data); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("test_ctx_create failed"); + exit(EXIT_FAILURE); + } + /* Input */ + ctx->i_ffd = flb_input(ctx->flb, (char *) "lib", NULL); + TEST_CHECK(ctx->i_ffd >= 0); + + ret = flb_output_set(ctx->flb, ctx->o_ffd, + "format", "msgpack", + "max_records", "5", + NULL); + TEST_CHECK(ret == 0); + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + for (i=0; i<100; i++) { + ret = flb_lib_push(ctx->flb, ctx->i_ffd, input_json,size); + TEST_CHECK(ret >= 0); + } + + /* waiting to flush */ + wait_with_timeout(1000, &num); + + if (!TEST_CHECK(num == expected /* max_records */)) { + TEST_MSG("max_records error. got=%d, expected=%d", num, expected); + } + + test_ctx_destroy(ctx); +} +#ifdef FLB_HAVE_METRICS +static void test_metrics_msgpack(void) +{ + struct flb_lib_out_cb cb_data; + struct test_ctx *ctx; + int ret; + int num; + int unused; + + clear_output_num(); + + cb_data.cb = cb_count_msgpack; + cb_data.data = &unused; + + ctx = test_ctx_create(&cb_data); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("test_ctx_create failed"); + exit(EXIT_FAILURE); + } + /* Input */ + ctx->i_ffd = flb_input(ctx->flb, (char *) "fluentbit_metrics", NULL); + TEST_CHECK(ctx->i_ffd >= 0); + ret = flb_input_set(ctx->flb, ctx->i_ffd, + "scrape_interval", "1", + NULL); + TEST_CHECK(ret == 0); + + ret = flb_output_set(ctx->flb, ctx->o_ffd, + "format", "msgpack", + NULL); + TEST_CHECK(ret == 0); + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + /* waiting to flush */ + wait_with_timeout(5000, &num); + + if (!TEST_CHECK(num > 0)) { + TEST_MSG("no outputs"); + } + + test_ctx_destroy(ctx); +} + +static void test_metrics_json(void) +{ + struct flb_lib_out_cb cb_data; + struct test_ctx *ctx; + int ret; + int num; + int unused; + + clear_output_num(); + + cb_data.cb = cb_count; + cb_data.data = &unused; + + ctx = test_ctx_create(&cb_data); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("test_ctx_create failed"); + exit(EXIT_FAILURE); + } + /* Input */ + ctx->i_ffd = flb_input(ctx->flb, (char *) "fluentbit_metrics", NULL); + TEST_CHECK(ctx->i_ffd >= 0); + ret = flb_input_set(ctx->flb, ctx->i_ffd, + "scrape_interval", "1", + NULL); + TEST_CHECK(ret == 0); + + ret = flb_output_set(ctx->flb, ctx->o_ffd, + "format", "json", + NULL); + TEST_CHECK(ret == 0); + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + /* waiting to flush */ + wait_with_timeout(5000, &num); + + if (!TEST_CHECK(num > 0)) { + TEST_MSG("no outputs"); + } + + test_ctx_destroy(ctx); +} +#endif + +TEST_LIST = { +#ifdef FLB_HAVE_METRICS + {"metrics_msgpack", test_metrics_msgpack}, + {"metrics_json", test_metrics_json}, +#endif + {"format_json", test_format_json}, + {"format_msgpack", test_format_msgpack}, + {"max_records", test_max_records}, + {NULL, NULL} +}; diff --git a/src/fluent-bit/tests/runtime/out_loki.c b/src/fluent-bit/tests/runtime/out_loki.c new file mode 100644 index 000000000..48805d2c8 --- /dev/null +++ b/src/fluent-bit/tests/runtime/out_loki.c @@ -0,0 +1,623 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2019-2021 The Fluent Bit Authors + * Copyright (C) 2015-2018 Treasure Data Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <fluent-bit.h> +#include <fluent-bit/flb_sds.h> +#include "flb_tests_runtime.h" + +#define DPATH_LOKI FLB_TESTS_DATA_PATH "/data/loki" + +pthread_mutex_t result_mutex = PTHREAD_MUTEX_INITIALIZER; +int num_output = 0; +static int get_output_num() +{ + int ret; + pthread_mutex_lock(&result_mutex); + ret = num_output; + pthread_mutex_unlock(&result_mutex); + + return ret; +} + +static void set_output_num(int num) +{ + pthread_mutex_lock(&result_mutex); + num_output = num; + pthread_mutex_unlock(&result_mutex); +} + +static void clear_output_num() +{ + set_output_num(0); +} + +#define JSON_BASIC "[12345678, {\"key\":\"value\"}]" +static void cb_check_basic(void *ctx, int ffd, + int res_ret, void *res_data, size_t res_size, + void *data) +{ + char *p; + flb_sds_t out_js = res_data; + char *index_line = "{\\\"key\\\":\\\"value\\\"}"; + + p = strstr(out_js, index_line); + if (!TEST_CHECK(p != NULL)) { + TEST_MSG("Given:%s", out_js); + } + + flb_sds_destroy(out_js); +} + +void flb_test_basic() +{ + int ret; + int size = sizeof(JSON_BASIC) - 1; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", "1", "grace", "1", + "log_level", "error", + NULL); + + /* Lib input mode */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + /* Elasticsearch output */ + out_ffd = flb_output(ctx, (char *) "loki", NULL); + flb_output_set(ctx, out_ffd, + "match", "test", + NULL); + + /* Enable test mode */ + ret = flb_output_set_test(ctx, out_ffd, "formatter", + cb_check_basic, + NULL, NULL); + + /* Start */ + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + ret = flb_lib_push(ctx, in_ffd, (char *) JSON_BASIC, size); + TEST_CHECK(ret >= 0); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + +static void cb_check_labels(void *ctx, int ffd, + int res_ret, void *res_data, size_t res_size, + void *data) +{ + char *p; + flb_sds_t out_js = res_data; + char *index_line = "\"stream\":{\"a\":\"b\"}"; + + p = strstr(out_js, index_line); + if (!TEST_CHECK(p != NULL)) { + TEST_MSG("Given:%s", out_js); + } + + flb_sds_destroy(out_js); +} + +void flb_test_labels() +{ + int ret; + int size = sizeof(JSON_BASIC) - 1; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", "1", "grace", "1", + "log_level", "error", + NULL); + + /* Lib input mode */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + /* Elasticsearch output */ + out_ffd = flb_output(ctx, (char *) "loki", NULL); + flb_output_set(ctx, out_ffd, + "match", "test", + "labels", "a=b", /* "stream":{"a":"b"} */ + NULL); + + /* Enable test mode */ + ret = flb_output_set_test(ctx, out_ffd, "formatter", + cb_check_labels, + NULL, NULL); + + /* Start */ + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + ret = flb_lib_push(ctx, in_ffd, (char *) JSON_BASIC, size); + TEST_CHECK(ret >= 0); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + +static void cb_check_label_keys(void *ctx, int ffd, + int res_ret, void *res_data, size_t res_size, + void *data) +{ + char *p; + flb_sds_t out_js = res_data; + char *index_line = "{\"stream\":{\"data_l_key\":\"test\"}"; + + p = strstr(out_js, index_line); + if (!TEST_CHECK(p != NULL)) { + TEST_MSG("Given:%s", out_js); + } + + flb_sds_destroy(out_js); +} + +#define JSON_LABEL_KEYS "[12345678, {\"key\":\"value\",\"foo\":\"bar\", \"data\":{\"l_key\":\"test\"}}]" +void flb_test_label_keys() +{ + int ret; + int size = sizeof(JSON_LABEL_KEYS) - 1; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", "1", "grace", "1", + "log_level", "error", + NULL); + + /* Lib input mode */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + /* Elasticsearch output */ + out_ffd = flb_output(ctx, (char *) "loki", NULL); + flb_output_set(ctx, out_ffd, + "match", "test", + "label_keys", "$data['l_key']", /* {"stream":{"data_l_key":"test"} */ + NULL); + + /* Enable test mode */ + ret = flb_output_set_test(ctx, out_ffd, "formatter", + cb_check_label_keys, + NULL, NULL); + + /* Start */ + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + ret = flb_lib_push(ctx, in_ffd, (char *) JSON_LABEL_KEYS, size); + TEST_CHECK(ret >= 0); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + +static void cb_check_line_format(void *ctx, int ffd, + int res_ret, void *res_data, size_t res_size, + void *data) +{ + char *p; + flb_sds_t out_js = res_data; + char *index_line = "key=\\\"value\\\""; + + p = strstr(out_js, index_line); + if (!TEST_CHECK(p != NULL)) { + TEST_MSG("Given:%s", out_js); + } + + flb_sds_destroy(out_js); +} + +void flb_test_line_format() +{ + int ret; + int size = sizeof(JSON_BASIC) - 1; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", "1", "grace", "1", + "log_level", "error", + NULL); + + /* Lib input mode */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + /* Elasticsearch output */ + out_ffd = flb_output(ctx, (char *) "loki", NULL); + flb_output_set(ctx, out_ffd, + "match", "test", + "line_format", "key_value", + NULL); + + /* Enable test mode */ + ret = flb_output_set_test(ctx, out_ffd, "formatter", + cb_check_line_format, + NULL, NULL); + + /* Start */ + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + ret = flb_lib_push(ctx, in_ffd, (char *) JSON_BASIC, size); + TEST_CHECK(ret >= 0); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + + +static void cb_check_line_format_remove_keys(void *ctx, int ffd, + int res_ret, void *res_data, + size_t res_size, void *data) +{ + char *p; + flb_sds_t out_js = res_data; + char *index_line = "value_nested"; + + /* p == NULL is expected since it should be removed.*/ + p = strstr(out_js, index_line); + if (!TEST_CHECK(p == NULL)) { + TEST_MSG("Given:%s", out_js); + } + + flb_sds_destroy(out_js); +} +#define JSON_BASIC_NEST "[12345678, {\"key\": {\"nest\":\"value_nested\"}} ]" +/* https://github.com/fluent/fluent-bit/issues/3875 */ +void flb_test_remove_map() +{ + int ret; + int size = sizeof(JSON_BASIC_NEST) - 1; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", "1", "grace", "1", + "log_level", "error", + NULL); + + /* Lib input mode */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + /* Elasticsearch output */ + out_ffd = flb_output(ctx, (char *) "loki", NULL); + flb_output_set(ctx, out_ffd, + "match", "test", + "remove_keys", "key", + NULL); + + /* Enable test mode */ + ret = flb_output_set_test(ctx, out_ffd, "formatter", + cb_check_line_format_remove_keys, + NULL, NULL); + + /* Start */ + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + ret = flb_lib_push(ctx, in_ffd, (char *) JSON_BASIC_NEST, size); + TEST_CHECK(ret >= 0); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + +static void cb_check_labels_ra(void *ctx, int ffd, + int res_ret, void *res_data, size_t res_size, + void *data) +{ + char *p; + flb_sds_t out_js = res_data; + char *index_line = "\\\"data\\\":{\\\"l_key\\\":\\\"test\\\"}"; + + p = strstr(out_js, index_line); + if (!TEST_CHECK(p != NULL)) { + TEST_MSG("Given:%s", out_js); + } + + flb_sds_destroy(out_js); +} + +/* https://github.com/fluent/fluent-bit/issues/3867 */ +void flb_test_labels_ra() +{ + int ret; + int size = sizeof(JSON_LABEL_KEYS) - 1; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", "1", "grace", "1", + "log_level", "error", + NULL); + + /* Lib input mode */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + /* Elasticsearch output */ + out_ffd = flb_output(ctx, (char *) "loki", NULL); + flb_output_set(ctx, out_ffd, + "match", "test", + "labels", "$data['l_key']", + NULL); + + /* Enable test mode */ + ret = flb_output_set_test(ctx, out_ffd, "formatter", + cb_check_labels_ra, + NULL, NULL); + + /* Start */ + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + flb_lib_push(ctx, in_ffd, (char *) JSON_LABEL_KEYS, size); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + +static void cb_check_remove_keys(void *ctx, int ffd, + int res_ret, void *res_data, size_t res_size, + void *data) +{ + char *p; + flb_sds_t out_js = res_data; + + p = strstr(out_js, "foo"); + if (!TEST_CHECK(p == NULL)) { + TEST_MSG("Given:%s", out_js); + } + + p = strstr(out_js, "l_key"); + if (!TEST_CHECK(p == NULL)) { + TEST_MSG("Given:%s", out_js); + } + + flb_sds_destroy(out_js); +} + +void flb_test_remove_keys() +{ + int ret; + int size = sizeof(JSON_LABEL_KEYS) - 1; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", "1", "grace", "1", + "log_level", "error", + NULL); + + /* Lib input mode */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + /* Elasticsearch output */ + out_ffd = flb_output(ctx, (char *) "loki", NULL); + flb_output_set(ctx, out_ffd, + "match", "test", + "remove_keys", "foo, $data['l_key']", + NULL); + + /* Enable test mode */ + ret = flb_output_set_test(ctx, out_ffd, "formatter", + cb_check_remove_keys, + NULL, NULL); + + /* Start */ + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + flb_lib_push(ctx, in_ffd, (char *) JSON_LABEL_KEYS, size); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + +static void cb_check_label_map_path(void *ctx, int ffd, + int res_ret, void *res_data, size_t res_size, + void *data) +{ + char *p; + flb_sds_t out_log = res_data; + char *expected[] = { + "\"container\":\"promtail\"", + "\"pod\":\"promtail-xxx\"", + "\"namespace\":\"prod\"", + "\"team\":\"lalala\"", + NULL}; + int i = 0; + + set_output_num(1); + + while(expected[i] != NULL) { + p = strstr(out_log, expected[i]); + if (!TEST_CHECK(p != NULL)) { + TEST_MSG("Given:%s Expect:%s", out_log, expected[i]); + } + i++; + } + + flb_sds_destroy(out_log); +} + +void flb_test_label_map_path() +{ + int ret; + char *str = "[12345678, {\"kubernetes\":{\"container_name\":\"promtail\",\"pod_name\":\"promtail-xxx\",\"namespace_name\":\"prod\",\"labels\":{\"team\": \"lalala\"}},\"log\":\"log\"}]"; + int size = strlen(str); + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + clear_output_num(); + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", "1", "grace", "1", + "log_level", "error", + NULL); + + /* Lib input mode */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + /* Elasticsearch output */ + out_ffd = flb_output(ctx, (char *) "loki", NULL); + flb_output_set(ctx, out_ffd, + "match", "test", + "label_map_path", DPATH_LOKI "/labelmap.json", + NULL); + + /* Enable test mode */ + ret = flb_output_set_test(ctx, out_ffd, "formatter", + cb_check_label_map_path, + NULL, NULL); + + /* Start */ + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + ret = flb_lib_push(ctx, in_ffd, str, size); + TEST_CHECK(ret == size); + + sleep(2); + + ret = get_output_num(); + if (!TEST_CHECK(ret != 0)) { + TEST_MSG("no output"); + } + + flb_stop(ctx); + flb_destroy(ctx); +} + +static void cb_check_float_value(void *ctx, int ffd, + int res_ret, void *res_data, size_t res_size, + void *data) +{ + char *p; + flb_sds_t out_js = res_data; + char *index_line = "\"float=1.3\""; + + p = strstr(out_js, index_line); + if (!TEST_CHECK(p != NULL)) { + TEST_MSG("Given:%s", out_js); + } + + flb_sds_destroy(out_js); +} + +#define JSON_FLOAT "[12345678, {\"float\":1.3}]" +void flb_test_float_value() +{ + int ret; + int size = sizeof(JSON_FLOAT) - 1; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", "1", "grace", "1", + "log_level", "error", + NULL); + + /* Lib input mode */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + /* Elasticsearch output */ + out_ffd = flb_output(ctx, (char *) "loki", NULL); + flb_output_set(ctx, out_ffd, + "match", "test", + "line_format", "key_value", + NULL); + + /* Enable test mode */ + ret = flb_output_set_test(ctx, out_ffd, "formatter", + cb_check_float_value, + NULL, NULL); + + /* Start */ + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + ret = flb_lib_push(ctx, in_ffd, (char *) JSON_FLOAT, size); + TEST_CHECK(ret >= 0); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + + +/* Test list */ +TEST_LIST = { + {"remove_keys_remove_map" , flb_test_remove_map}, + {"labels_ra" , flb_test_labels_ra }, + {"remove_keys" , flb_test_remove_keys }, + {"basic" , flb_test_basic }, + {"labels" , flb_test_labels }, + {"label_keys" , flb_test_label_keys }, + {"line_format" , flb_test_line_format }, + {"label_map_path" , flb_test_label_map_path}, + {"float_value" , flb_test_float_value}, + {NULL, NULL} +}; diff --git a/src/fluent-bit/tests/runtime/out_null.c b/src/fluent-bit/tests/runtime/out_null.c new file mode 100644 index 000000000..c74e0977b --- /dev/null +++ b/src/fluent-bit/tests/runtime/out_null.c @@ -0,0 +1,127 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +#include <fluent-bit.h> +#include "flb_tests_runtime.h" + +/* Test data */ +#include "data/common/json_invalid.h" /* JSON_INVALID */ +#include "data/common/json_long.h" /* JSON_LONG */ +#include "data/common/json_small.h" /* JSON_SMALL */ + +/* Test functions */ +void flb_test_null_json_invalid(void); +void flb_test_null_json_long(void); +void flb_test_null_json_small(void); + +/* Test list */ +TEST_LIST = { + {"json_invalid", flb_test_null_json_invalid }, + {"json_long", flb_test_null_json_long }, + {"json_small", flb_test_null_json_small }, + {NULL, NULL} +}; + + +void flb_test_null_json_invalid(void) +{ + int i; + int ret; + int bytes; + char *p = (char *) JSON_INVALID; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + ctx = flb_create(); + flb_service_set(ctx, "Flush", "1", "Grace", "1", "Log_Level", "error", NULL); + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + out_ffd = flb_output(ctx, (char *) "null", NULL); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, "match", "test", NULL); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + for (i = 0; i < (int) sizeof(JSON_INVALID) - 1; i++) { + bytes = flb_lib_push(ctx, in_ffd, p + i, 1); + TEST_CHECK(bytes == 1); + } + + flb_stop(ctx); + flb_destroy(ctx); +} + +/* It writes a very long JSON map (> 100KB) byte by byte */ +void flb_test_null_json_long(void) +{ + int i; + int ret; + int bytes; + char *p = (char *) JSON_LONG; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + ctx = flb_create(); + flb_service_set(ctx, "Flush", "1", "Grace", "1", "Log_Level", "error", NULL); + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + out_ffd = flb_output(ctx, (char *) "null", NULL); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, "match", "test", NULL); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + for (i = 0; i < (int) sizeof(JSON_LONG) - 1; i++) { + bytes = flb_lib_push(ctx, in_ffd, p + i, 1); + TEST_CHECK(bytes == 1); + } + + sleep(1); /* waiting flush */ + + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_null_json_small(void) +{ + int i; + int ret; + int bytes; + char *p = (char *) JSON_SMALL; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + ctx = flb_create(); + flb_service_set(ctx, "Flush", "1", "Grace", "1", "Log_Level", "error", NULL); + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + out_ffd = flb_output(ctx, (char *) "null", NULL); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, "match", "test", NULL); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + for (i = 0; i < (int) sizeof(JSON_SMALL) - 1; i++) { + bytes = flb_lib_push(ctx, in_ffd, p + i, 1); + TEST_CHECK(bytes == 1); + } + + sleep(1); /* waiting flush */ + + flb_stop(ctx); + flb_destroy(ctx); +} diff --git a/src/fluent-bit/tests/runtime/out_opensearch.c b/src/fluent-bit/tests/runtime/out_opensearch.c new file mode 100644 index 000000000..26ced9e5a --- /dev/null +++ b/src/fluent-bit/tests/runtime/out_opensearch.c @@ -0,0 +1,1077 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +#include <fluent-bit.h> +#include "flb_tests_runtime.h" + +/* Test data */ +#include "data/es/json_es.h" /* JSON_ES */ + +static void cb_check_write_op_index(void *ctx, int ffd, + int res_ret, void *res_data, + size_t res_size, void *data) +{ + char *p; + char *out_js = res_data; + char *index_line = "{\"index\":{"; + + p = strstr(out_js, index_line); + TEST_CHECK(p == out_js); + + flb_sds_destroy(res_data); +} + +static void cb_check_write_op_create(void *ctx, int ffd, + int res_ret, void *res_data, + size_t res_size, void *data) +{ + char *p; + char *out_js = res_data; + char *index_line = "{\"create\":{"; + + p = strstr(out_js, index_line); + TEST_CHECK(p == out_js); + + flb_sds_destroy(res_data); +} + +static void cb_check_write_op_update(void *ctx, int ffd, + int res_ret, void *res_data, + size_t res_size, void *data) +{ + char *p; + char *b; + char *out_js = res_data; + char *index_line = "{\"update\":{"; + char *body = "{\"doc\":"; + + p = strstr(out_js, index_line); + TEST_CHECK(p == out_js); + b = strstr(out_js, body); + TEST_CHECK(b != NULL); + + flb_sds_destroy(res_data); +} + +static void cb_check_write_op_upsert(void *ctx, int ffd, + int res_ret, void *res_data, + size_t res_size, void *data) +{ + char *p; + char *b; + char *out_js = res_data; + char *index_line = "{\"update\":{"; + char *body = "{\"doc_as_upsert\":true,\"doc\":"; + + p = strstr(out_js, index_line); + TEST_CHECK(p == out_js); + b = strstr(out_js, body); + TEST_CHECK(b != NULL); + + flb_sds_destroy(res_data); +} + +static void cb_check_index_type(void *ctx, int ffd, + int res_ret, void *res_data, size_t res_size, + void *data) +{ + char *p; + char *out_js = res_data; + char *index_line = "{\"create\":{\"_index\":\"index_test\",\"_type\":\"type_test\"}"; + + p = strstr(out_js, index_line); + TEST_CHECK(p != NULL); + + flb_sds_destroy(res_data); +} + +static void cb_check_index_record_accessor(void *ctx, int ffd, + int res_ret, void *res_data, size_t res_size, + void *data) +{ + char *p; + char *out_js = res_data; + char *index_line = "{\"create\":{\"_index\":\"abc\",\"_type\":\"def\"}"; + + p = strstr(out_js, index_line); + TEST_CHECK(p != NULL); + + flb_sds_destroy(res_data); +} + +static void cb_check_index_record_accessor_suppress_type(void *ctx, int ffd, + int res_ret, void *res_data, size_t res_size, + void *data) +{ + char *p; + char *out_js = res_data; + char *index_line = "{\"create\":{\"_index\":\"abc\"}"; + + p = strstr(out_js, index_line); + TEST_CHECK(p != NULL); + + flb_sds_destroy(res_data); +} + +static void cb_check_index_record_accessor_id_key(void *ctx, int ffd, + int res_ret, void *res_data, size_t res_size, + void *data) +{ + char *p; + char *out_js = res_data; + char *index_line = "{\"create\":{\"_index\":\"abc\",\"_type\":\"def\",\"_id\":\"something\"}}"; + + p = strstr(out_js, index_line); + TEST_CHECK(p != NULL); + + flb_sds_destroy(res_data); +} + +static void cb_check_index_record_accessor_generate_id(void *ctx, int ffd, + int res_ret, void *res_data, size_t res_size, + void *data) +{ + char *p; + char *out_js = res_data; + char *index_line = "{\"create\":{\"_index\":\"code\",\"_type\":\"def\",\"_id\":\""; + char *id = NULL; + char c; + int i; + + // check that index is working + p = strstr(out_js, index_line); + TEST_CHECK(p != NULL); + + // check that we have a UUID + id = p + strlen(index_line); + TEST_CHECK(strlen(id) > 36); + for (i = 0; i < strlen(id) && i < 36; i++) { + c = (*(id+i)); + TEST_CHECK( + (c >= 'A' && c <= 'F') || + (*(id+i)) == '-' || + (c >= '0' && c <= '9') + ); + } + + flb_sds_destroy(res_data); +} + +static void cb_check_logstash_format(void *ctx, int ffd, + int res_ret, void *res_data, size_t res_size, + void *data) +{ + char *p; + char *out_js = res_data; + char *index_line = "{\"create\":{\"_index\":\"prefix-2015-11-24\",\"_type\":\"_doc\"}"; + + p = strstr(out_js, index_line); + TEST_CHECK(p != NULL); + + flb_sds_destroy(res_data); +} + +static void cb_check_logstash_prefix_separator(void *ctx, int ffd, + int res_ret, void *res_data, size_t res_size, + void *data) +{ + char *p; + char *out_js = res_data; + char *index_line = "{\"create\":{\"_index\":\"prefixSEP2015-11-24\",\"_type\":\"_doc\"}"; + + p = strstr(out_js, index_line); + if(!TEST_CHECK(p != NULL)) { + TEST_MSG("Got: %s", out_js); + } + flb_sds_destroy(res_data); +} + +static void cb_check_logstash_format_nanos(void *ctx, int ffd, + int res_ret, void *res_data, size_t res_size, + void *data) +{ + char *p; + char *out_js = res_data; + char *index_line = "\"@timestamp\":\"2015-11-24T22:15:40.000000000Z\""; + + p = strstr(out_js, index_line); + TEST_CHECK(p != NULL); + + flb_sds_destroy(res_data); +} + +static void cb_check_tag_key(void *ctx, int ffd, + int res_ret, void *res_data, size_t res_size, + void *data) +{ + char *p; + char *out_js = res_data; + char *record = "\"mytag\":\"test\""; + + p = strstr(out_js, record); + TEST_CHECK(p != NULL); + flb_sds_destroy(res_data); +} + +static void cb_check_replace_dots(void *ctx, int ffd, + int res_ret, void *res_data, size_t res_size, + void *data) +{ + char *p; + char *out_js = res_data; + char *record = "\"_o_k\":[{\"_b_ar\""; + + p = strstr(out_js, record); + TEST_CHECK(p != NULL); + + flb_sds_destroy(res_data); +} + +static void cb_check_id_key(void *ctx, int ffd, + int res_ret, void *res_data, size_t res_size, + void *data) +{ + char *p; + char *out_js = res_data; + char *record = "\"_id\":\"some string\""; // see data/es/json_es.h + + p = strstr(out_js, record); + TEST_CHECK(p != NULL); + + flb_sds_destroy(res_data); +} + +void flb_test_write_operation_index() +{ + int ret; + int size = sizeof(JSON_ES) - 1; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + + /* Lib input mode */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + /* Elasticsearch output */ + out_ffd = flb_output(ctx, (char *) "opensearch", NULL); + flb_output_set(ctx, out_ffd, + "match", "test", + NULL); + + /* Override defaults of index and type */ + flb_output_set(ctx, out_ffd, + "write_operation", "index", + NULL); + + /* Enable test mode */ + ret = flb_output_set_test(ctx, out_ffd, "formatter", + cb_check_write_op_index, + NULL, NULL); + + /* Start */ + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + flb_lib_push(ctx, in_ffd, (char *) JSON_ES, size); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_write_operation_create() +{ + int ret; + int size = sizeof(JSON_ES) - 1; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + + /* Lib input mode */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + /* Elasticsearch output */ + out_ffd = flb_output(ctx, (char *) "opensearch", NULL); + flb_output_set(ctx, out_ffd, + "match", "test", + NULL); + + /* Override defaults of index and type */ + flb_output_set(ctx, out_ffd, + "write_operation", "create", + NULL); + + /* Enable test mode */ + ret = flb_output_set_test(ctx, out_ffd, "formatter", + cb_check_write_op_create, + NULL, NULL); + + /* Start */ + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + flb_lib_push(ctx, in_ffd, (char *) JSON_ES, size); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + + +void flb_test_write_operation_update() +{ + int ret; + int size = sizeof(JSON_ES) - 1; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + + /* Lib input mode */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + /* Elasticsearch output */ + out_ffd = flb_output(ctx, (char *) "opensearch", NULL); + flb_output_set(ctx, out_ffd, + "match", "test", + NULL); + + /* Override defaults of index and type */ + flb_output_set(ctx, out_ffd, + "Write_Operation", "Update", + "Generate_Id", "True", + NULL); + + /* Enable test mode */ + ret = flb_output_set_test(ctx, out_ffd, "formatter", + cb_check_write_op_update, + NULL, NULL); + + /* Start */ + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + flb_lib_push(ctx, in_ffd, (char *) JSON_ES, size); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + + +void flb_test_write_operation_upsert() +{ + int ret; + int size = sizeof(JSON_ES) - 1; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + + /* Lib input mode */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + /* Elasticsearch output */ + out_ffd = flb_output(ctx, (char *) "opensearch", NULL); + flb_output_set(ctx, out_ffd, + "match", "test", + NULL); + + /* Override defaults of index and type */ + flb_output_set(ctx, out_ffd, + "Write_Operation", "Upsert", + "Generate_Id", "True", + NULL); + + /* Enable test mode */ + ret = flb_output_set_test(ctx, out_ffd, "formatter", + cb_check_write_op_upsert, + NULL, NULL); + + /* Start */ + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + flb_lib_push(ctx, in_ffd, (char *) JSON_ES, size); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_index_type() +{ + int ret; + int size = sizeof(JSON_ES) - 1; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + + /* Lib input mode */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + /* Elasticsearch output */ + out_ffd = flb_output(ctx, (char *) "opensearch", NULL); + flb_output_set(ctx, out_ffd, + "match", "test", + NULL); + + /* Override defaults of index and type */ + flb_output_set(ctx, out_ffd, + "index", "index_test", + "type", "type_test", + NULL); + + /* Enable test mode */ + ret = flb_output_set_test(ctx, out_ffd, "formatter", + cb_check_index_type, + NULL, NULL); + + /* Start */ + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + flb_lib_push(ctx, in_ffd, (char *) JSON_ES, size); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_index_record_accessor() +{ + int ret; + int len; + int in_ffd; + int out_ffd; + char *record = "[1448403340, {\"key\": \"something\", \"myindex\": \"abc\"}]"; + + flb_ctx_t *ctx; + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + + /* Lib input mode */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + /* Elasticsearch output */ + out_ffd = flb_output(ctx, (char *) "opensearch", NULL); + + flb_output_set(ctx, out_ffd, + "match", "test", + "index", "$myindex", + "type", "def", + NULL); + + /* Enable test mode */ + ret = flb_output_set_test(ctx, out_ffd, "formatter", + cb_check_index_record_accessor, + NULL, NULL); + + /* Start */ + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + len = strlen(record); + flb_lib_push(ctx, in_ffd, record, len); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_index_record_accessor_suppress_type() +{ + int ret; + int len; + int in_ffd; + int out_ffd; + char *record = "[1448403340, {\"key\": \"something\", \"myindex\": \"abc\"}]"; + + flb_ctx_t *ctx; + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + + /* Lib input mode */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + /* Elasticsearch output */ + out_ffd = flb_output(ctx, (char *) "opensearch", NULL); + + flb_output_set(ctx, out_ffd, + "match", "test", + "index", "$myindex", + "suppress_type_name", "true", + NULL); + + /* Enable test mode */ + ret = flb_output_set_test(ctx, out_ffd, "formatter", + cb_check_index_record_accessor_suppress_type, + NULL, NULL); + + /* Start */ + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + len = strlen(record); + flb_lib_push(ctx, in_ffd, record, len); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_index_record_accessor_with_id_key() +{ + int ret; + int len; + int in_ffd; + int out_ffd; + char *record = "[1448403340, {\"key\": \"something\", \"myindex\": \"abc\"}]"; + + flb_ctx_t *ctx; + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + + /* Lib input mode */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + /* Elasticsearch output */ + out_ffd = flb_output(ctx, (char *) "opensearch", NULL); + + flb_output_set(ctx, out_ffd, + "match", "test", + "index", "$myindex", + "type", "def", + "id_key", "key", + NULL); + + /* Enable test mode */ + ret = flb_output_set_test(ctx, out_ffd, "formatter", + cb_check_index_record_accessor_id_key, + NULL, NULL); + + /* Start */ + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + len = strlen(record); + flb_lib_push(ctx, in_ffd, record, len); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_index_record_accessor_with_generate_id() +{ + int ret; + int len; + int in_ffd; + int out_ffd; + char *record = "[1448403340, {\"key\": \"xul\", \"myindex\": \"code\"}]"; + + flb_ctx_t *ctx; + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + + /* Lib input mode */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + /* Elasticsearch output */ + out_ffd = flb_output(ctx, (char *) "opensearch", NULL); + + flb_output_set(ctx, out_ffd, + "match", "test", + "index", "$myindex", + "type", "def", + "generate_id", "true", + NULL); + + /* Enable test mode */ + ret = flb_output_set_test(ctx, out_ffd, "formatter", + cb_check_index_record_accessor_generate_id, + NULL, NULL); + + /* Start */ + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + len = strlen(record); + flb_lib_push(ctx, in_ffd, record, len); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_logstash_format() +{ + int ret; + int size = sizeof(JSON_ES) - 1; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + + /* Lib input mode */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + /* Elasticsearch output */ + out_ffd = flb_output(ctx, (char *) "opensearch", NULL); + flb_output_set(ctx, out_ffd, + "match", "test", + NULL); + + /* Override defaults of index and type */ + flb_output_set(ctx, out_ffd, + "logstash_format", "on", + "logstash_prefix", "prefix", + "logstash_dateformat", "%Y-%m-%d", + NULL); + + /* Enable test mode */ + ret = flb_output_set_test(ctx, out_ffd, "formatter", + cb_check_logstash_format, + NULL, NULL); + + /* Start */ + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + flb_lib_push(ctx, in_ffd, (char *) JSON_ES, size); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_logstash_format_nanos() +{ + int ret; + int size = sizeof(JSON_ES) - 1; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + + /* Lib input mode */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + /* Elasticsearch output */ + out_ffd = flb_output(ctx, (char *) "opensearch", NULL); + flb_output_set(ctx, out_ffd, + "match", "test", + NULL); + + /* Override defaults of index and type */ + flb_output_set(ctx, out_ffd, + "logstash_format", "on", + "logstash_prefix", "prefix", + "logstash_dateformat", "%Y-%m-%d", + "time_key_nanos", "on", + NULL); + + /* Enable test mode */ + ret = flb_output_set_test(ctx, out_ffd, "formatter", + cb_check_logstash_format_nanos, + NULL, NULL); + + /* Start */ + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + flb_lib_push(ctx, in_ffd, (char *) JSON_ES, size); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_tag_key() +{ + int ret; + int size = sizeof(JSON_ES) - 1; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + + /* Lib input mode */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + /* Elasticsearch output */ + out_ffd = flb_output(ctx, (char *) "opensearch", NULL); + flb_output_set(ctx, out_ffd, + "match", "test", + NULL); + + /* Override defaults of index and type */ + flb_output_set(ctx, out_ffd, + "include_tag_key", "on", + "tag_key", "mytag", + NULL); + + /* Enable test mode */ + ret = flb_output_set_test(ctx, out_ffd, "formatter", + cb_check_tag_key, + NULL, NULL); + + /* Start */ + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + flb_lib_push(ctx, in_ffd, (char *) JSON_ES, size); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_replace_dots() +{ + int ret; + int size = sizeof(JSON_DOTS) - 1; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + + /* Lib input mode */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + /* Elasticsearch output */ + out_ffd = flb_output(ctx, (char *) "opensearch", NULL); + flb_output_set(ctx, out_ffd, + "match", "test", + NULL); + + /* Override defaults of index and type */ + flb_output_set(ctx, out_ffd, + "replace_dots", "on", + NULL); + + /* Enable test mode */ + ret = flb_output_set_test(ctx, out_ffd, "formatter", + cb_check_replace_dots, + NULL, NULL); + + /* Start */ + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + flb_lib_push(ctx, in_ffd, (char *) JSON_DOTS, size); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_id_key() +{ + int ret; + int size = sizeof(JSON_ES) - 1; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + + /* Lib input mode */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + /* Elasticsearch output */ + out_ffd = flb_output(ctx, (char *) "opensearch", NULL); + flb_output_set(ctx, out_ffd, + "match", "test", + NULL); + + /* Override defaults of index and type */ + flb_output_set(ctx, out_ffd, + "id_key", "key_2", + NULL); + + /* Enable test mode */ + ret = flb_output_set_test(ctx, out_ffd, "formatter", + cb_check_id_key, + NULL, NULL); + + /* Start */ + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + flb_lib_push(ctx, in_ffd, (char *) JSON_ES, size); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + +/* no check */ +static void cb_check_nothing(void *ctx, int ffd, + int res_ret, void *res_data, size_t res_size, + void *data) +{ + flb_sds_destroy(res_data); +} + +/* https://github.com/fluent/fluent-bit/issues/3905 */ +void flb_test_div0() +{ + int ret; + char record[8000]; + char record_header[] = "[1448403340,{\"key\":\""; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + int i; + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + + /* Lib input mode */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + /* Elasticsearch output */ + out_ffd = flb_output(ctx, (char *) "opensearch", NULL); + flb_output_set(ctx, out_ffd, + "match", "test", + NULL); + + /* Override defaults of index and type */ + flb_output_set(ctx, out_ffd, + NULL); + + /* Enable test mode */ + ret = flb_output_set_test(ctx, out_ffd, "formatter", + cb_check_nothing, + NULL, NULL); + + /* Start */ + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* create json */ + strncpy(&record[0], &record_header[0], strlen(record_header)); + for(i=strlen(record_header); i<sizeof(record)-4; i++) { + record[i] = 'a'; + } + record[sizeof(record)-4] = '"'; + record[sizeof(record)-3] = '}'; + record[sizeof(record)-2] = ']'; + record[sizeof(record)-1] = '\0'; + /* Ingest data sample */ + flb_lib_push(ctx, in_ffd, (char *) &record[0], strlen(record)); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + + +static void cb_check_long_index(void *ctx, int ffd, + int res_ret, void *res_data, size_t res_size, + void *data) +{ + char *p; + char *out_js = res_data; + char long_index[256] = {0}; + int i; + + for (i=0; i<sizeof(long_index)-1; i++) { + long_index[i] = '0' + (i%10); + } + + p = strstr(out_js, &long_index[0]); + TEST_CHECK(p != NULL); + flb_sds_destroy(res_data); +} + +/* https://github.com/fluent/fluent-bit/issues/4311 */ +void flb_test_long_index() +{ + int ret; + int size = sizeof(JSON_ES) -1; + char long_index[256] = {0}; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + int i; + + for (i=0; i<sizeof(long_index)-1; i++) { + long_index[i] = '0' + (i%10); + } + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + + /* Lib input mode */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + /* Elasticsearch output */ + out_ffd = flb_output(ctx, (char *) "opensearch", NULL); + flb_output_set(ctx, out_ffd, + "match", "test", + "generate_id", "true", + "index", &long_index[0], + NULL); + + /* Override defaults of index and type */ + flb_output_set(ctx, out_ffd, + NULL); + + /* Enable test mode */ + ret = flb_output_set_test(ctx, out_ffd, "formatter", + cb_check_long_index, + NULL, NULL); + + /* Start */ + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + flb_lib_push(ctx, in_ffd, (char *)JSON_ES, size); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_logstash_prefix_separator() +{ + int ret; + int size = sizeof(JSON_ES) - 1; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + + /* Lib input mode */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + /* Elasticsearch output */ + out_ffd = flb_output(ctx, (char *) "opensearch", NULL); + flb_output_set(ctx, out_ffd, + "match", "test", + NULL); + + /* Override defaults of index and type */ + flb_output_set(ctx, out_ffd, + "logstash_format", "on", + "logstash_prefix", "prefix", + "logstash_prefix_separator", "SEP", + "logstash_dateformat", "%Y-%m-%d", + NULL); + + /* Enable test mode */ + ret = flb_output_set_test(ctx, out_ffd, "formatter", + cb_check_logstash_prefix_separator, + NULL, NULL); + + /* Start */ + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + flb_lib_push(ctx, in_ffd, (char *) JSON_ES, size); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + +/* Test list */ +TEST_LIST = { + {"long_index" , flb_test_long_index }, + {"div0_error" , flb_test_div0 }, + {"write_operation_index" , flb_test_write_operation_index }, + {"write_operation_create", flb_test_write_operation_create }, + {"write_operation_update", flb_test_write_operation_update }, + {"write_operation_upsert", flb_test_write_operation_upsert }, + {"index_type" , flb_test_index_type }, + {"index_record_accessor" , flb_test_index_record_accessor}, + {"index_record_accessor_suppress_type" , flb_test_index_record_accessor_suppress_type}, + {"index_record_accessor_id_key", flb_test_index_record_accessor_with_id_key}, + {"index_record_accessor_generate_id", flb_test_index_record_accessor_with_generate_id}, + {"logstash_format" , flb_test_logstash_format }, + {"logstash_format_nanos" , flb_test_logstash_format_nanos }, + {"tag_key" , flb_test_tag_key }, + {"replace_dots" , flb_test_replace_dots }, + {"id_key" , flb_test_id_key }, + {"logstash_prefix_separator" , flb_test_logstash_prefix_separator }, + {NULL, NULL} +}; diff --git a/src/fluent-bit/tests/runtime/out_plot.c b/src/fluent-bit/tests/runtime/out_plot.c new file mode 100644 index 000000000..67c841f2c --- /dev/null +++ b/src/fluent-bit/tests/runtime/out_plot.c @@ -0,0 +1,157 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +#include <fluent-bit.h> +#include "flb_tests_runtime.h" + +/* Test data */ +#include "data/common/json_invalid.h" /* JSON_INVALID */ + +/* Test functions */ +void flb_test_plot_json_invalid(void); +void flb_test_plot_json_multiple(void); +void flb_test_plot_key_mismatch(void); + +/* Test list */ +TEST_LIST = { + {"json_invalid", flb_test_plot_json_invalid }, + {"json_multiple", flb_test_plot_json_multiple }, + {"key_mismatch", flb_test_plot_key_mismatch }, + {NULL, NULL} +}; + +#define TEST_LOGFILE "flb_test_plot_dummy.log" +#define TEST_TIMEOUT 5 + +void flb_test_plot_json_invalid(void) +{ + int i; + int ret; + int bytes; + char *p = (char *) JSON_INVALID; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + ctx = flb_create(); + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + out_ffd = flb_output(ctx, (char *) "plot", NULL); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, "match", "test", NULL); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + for (i = 0; i < (int) sizeof(JSON_INVALID) - 1; i++) { + bytes = flb_lib_push(ctx, in_ffd, p + i, 1); + TEST_CHECK(bytes == 1); + } + + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_plot_json_multiple(void) +{ + int i; + int ret; + int bytes; + char p[100]; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + FILE *fp; + + remove(TEST_LOGFILE); + + ctx = flb_create(); + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + out_ffd = flb_output(ctx, (char *) "plot", NULL); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, "match", "test", NULL); + flb_output_set(ctx, out_ffd, "key", "val", NULL); + flb_output_set(ctx, out_ffd, "file", TEST_LOGFILE, NULL); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + for (i = 0; i < 256; i++) { + memset(p, '\0', sizeof(p)); + snprintf(p, sizeof(p), "[%d, {\"val\": %d,\"END_KEY\": \"JSON_END\"}]", i, (i * i)); + bytes = flb_lib_push(ctx, in_ffd, p, strlen(p)); + TEST_CHECK(bytes == strlen(p)); + } + + ret = wait_for_file(TEST_LOGFILE, 1, TEST_TIMEOUT); + TEST_CHECK(ret == 0); + + flb_stop(ctx); + flb_destroy(ctx); + + fp = fopen(TEST_LOGFILE, "r"); + TEST_CHECK(fp != NULL); + if (fp != NULL) { + fseek(fp, 0L ,SEEK_END); + TEST_CHECK(ftell(fp) > 0); /* file_size > 0 */ + fclose(fp); + remove(TEST_LOGFILE); + } +} + +void flb_test_plot_key_mismatch(void) +{ + int i; + int ret; + int bytes; + char p[100]; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + FILE *fp; + + remove(TEST_LOGFILE); + + ctx = flb_create(); + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + out_ffd = flb_output(ctx, (char *) "plot", NULL); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, "match", "test", NULL); + flb_output_set(ctx, out_ffd, "key", "xxx", NULL); + flb_output_set(ctx, out_ffd, "file", TEST_LOGFILE, NULL); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + for (i = 0; i < 256; i++) { + memset(p, '\0', sizeof(p)); + snprintf(p, sizeof(p), "[%d, {\"val\": %d,\"END_KEY\": \"JSON_END\"}]", i, (i * i)); + bytes = flb_lib_push(ctx, in_ffd, p, strlen(p)); + TEST_CHECK(bytes == strlen(p)); + } + + ret = wait_for_file(TEST_LOGFILE, 1, TEST_TIMEOUT); + TEST_CHECK(ret == 0); + + flb_stop(ctx); + flb_destroy(ctx); + + fp = fopen(TEST_LOGFILE, "r"); + TEST_CHECK(fp != NULL); + if (fp != NULL) { + fseek(fp, 0L ,SEEK_END); + TEST_CHECK(ftell(fp) == 0); /* file_size == 0 */ + fclose(fp); + remove(TEST_LOGFILE); + } +} diff --git a/src/fluent-bit/tests/runtime/out_retry.c b/src/fluent-bit/tests/runtime/out_retry.c new file mode 100644 index 000000000..16e4710dc --- /dev/null +++ b/src/fluent-bit/tests/runtime/out_retry.c @@ -0,0 +1,88 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +#include <fluent-bit.h> +#include "flb_tests_runtime.h" + +/* Test data */ +#include "data/common/json_invalid.h" /* JSON_INVALID */ + +/* Test functions */ +void flb_test_retry_json_invalid(void); +void flb_test_retry_normal(void); + +/* Test list */ +TEST_LIST = { + {"json_invalid", flb_test_retry_json_invalid }, + {"normal", flb_test_retry_normal }, + {NULL, NULL} +}; + + +void flb_test_retry_json_invalid(void) +{ + int i; + int ret; + int bytes; + char *p = (char *) JSON_INVALID; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + ctx = flb_create(); + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + out_ffd = flb_output(ctx, (char *) "retry", NULL); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, "match", "test", NULL); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + for (i = 0; i < (int) sizeof(JSON_INVALID) - 1; i++) { + bytes = flb_lib_push(ctx, in_ffd, p + i, 1); + TEST_CHECK(bytes == 1); + } + + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_retry_normal(void) +{ + int i; + int ret; + int bytes; + char p[100]; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + ctx = flb_create(); + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + out_ffd = flb_output(ctx, (char *) "retry", NULL); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, "match", "test", NULL); + flb_output_set(ctx, out_ffd, "retry", "10", NULL); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + for (i = 0; i < 256; i++) { + memset(p, '\0', sizeof(p)); + snprintf(p, sizeof(p), "[%d, {\"val\": %d,\"END_KEY\": \"JSON_END\"}]", i, (i * i)); + bytes = flb_lib_push(ctx, in_ffd, p, strlen(p)); + TEST_CHECK(bytes == strlen(p)); + } + + sleep(1); /* waiting flush */ + + flb_stop(ctx); + flb_destroy(ctx); +} diff --git a/src/fluent-bit/tests/runtime/out_s3.c b/src/fluent-bit/tests/runtime/out_s3.c new file mode 100644 index 000000000..5968ff12a --- /dev/null +++ b/src/fluent-bit/tests/runtime/out_s3.c @@ -0,0 +1,241 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ +#include <fluent-bit.h> +#include "flb_tests_runtime.h" + +/* Test data */ +#include "data/td/json_td.h" /* JSON_TD */ + +/* not a real error code, but tests that the code can respond to any error */ +#define ERROR_ACCESS_DENIED "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\ + <Error>\ + <Code>AccessDenied</Code>\ + <Message>Access Denied</Message>\ + <RequestId>656c76696e6727732072657175657374</RequestId>\ + <HostId>Uuag1LuByRx9e6j5Onimru9pO4ZVKnJ2Qz7/C1NPcfTWAtRPfTaOFg==</HostId>\ + </Error>" + +void flb_test_s3_multipart_success(void) +{ + int ret; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + /* mocks calls- signals that we are in test mode */ + setenv("FLB_S3_PLUGIN_UNDER_TEST", "true", 1); + + ctx = flb_create(); + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx,in_ffd, "tag", "test", NULL); + + out_ffd = flb_output(ctx, (char *) "s3", NULL); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd,"match", "*", NULL); + flb_output_set(ctx, out_ffd,"region", "us-west-2", NULL); + flb_output_set(ctx, out_ffd,"bucket", "fluent", NULL); + flb_output_set(ctx, out_ffd,"Retry_Limit", "1", NULL); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + flb_lib_push(ctx, in_ffd, (char *) JSON_TD , (int) sizeof(JSON_TD) - 1); + + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_s3_putobject_success(void) +{ + int ret; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + /* mocks calls- signals that we are in test mode */ + setenv("FLB_S3_PLUGIN_UNDER_TEST", "true", 1); + + ctx = flb_create(); + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx,in_ffd, "tag", "test", NULL); + + out_ffd = flb_output(ctx, (char *) "s3", NULL); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd,"match", "*", NULL); + flb_output_set(ctx, out_ffd,"region", "us-west-2", NULL); + flb_output_set(ctx, out_ffd,"bucket", "fluent", NULL); + flb_output_set(ctx, out_ffd,"use_put_object", "true", NULL); + flb_output_set(ctx, out_ffd,"total_file_size", "5M", NULL); + flb_output_set(ctx, out_ffd,"Retry_Limit", "1", NULL); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + flb_lib_push(ctx, in_ffd, (char *) JSON_TD , (int) sizeof(JSON_TD) - 1); + + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_s3_putobject_error(void) +{ + int ret; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + /* mocks calls- signals that we are in test mode */ + setenv("FLB_S3_PLUGIN_UNDER_TEST", "true", 1); + setenv("TEST_PUT_OBJECT_ERROR", ERROR_ACCESS_DENIED, 1); + + ctx = flb_create(); + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx,in_ffd, "tag", "test", NULL); + + out_ffd = flb_output(ctx, (char *) "s3", NULL); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd,"match", "*", NULL); + flb_output_set(ctx, out_ffd,"region", "us-west-2", NULL); + flb_output_set(ctx, out_ffd,"bucket", "fluent", NULL); + flb_output_set(ctx, out_ffd,"use_put_object", "true", NULL); + flb_output_set(ctx, out_ffd,"total_file_size", "5M", NULL); + flb_output_set(ctx, out_ffd,"Retry_Limit", "1", NULL); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + flb_lib_push(ctx, in_ffd, (char *) JSON_TD , (int) sizeof(JSON_TD) - 1); + + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); + unsetenv("TEST_PUT_OBJECT_ERROR"); + +} + +void flb_test_s3_create_upload_error(void) +{ + int ret; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + /* mocks calls- signals that we are in test mode */ + setenv("FLB_S3_PLUGIN_UNDER_TEST", "true", 1); + setenv("TEST_CREATE_MULTIPART_UPLOAD_ERROR", ERROR_ACCESS_DENIED, 1); + + ctx = flb_create(); + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx,in_ffd, "tag", "test", NULL); + + out_ffd = flb_output(ctx, (char *) "s3", NULL); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd,"match", "*", NULL); + flb_output_set(ctx, out_ffd,"region", "us-west-2", NULL); + flb_output_set(ctx, out_ffd,"bucket", "fluent", NULL); + flb_output_set(ctx, out_ffd,"Retry_Limit", "1", NULL); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + flb_lib_push(ctx, in_ffd, (char *) JSON_TD , (int) sizeof(JSON_TD) - 1); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); + unsetenv("TEST_CREATE_MULTIPART_UPLOAD_ERROR"); +} + +void flb_test_s3_upload_part_error(void) +{ + int ret; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + /* mocks calls- signals that we are in test mode */ + setenv("FLB_S3_PLUGIN_UNDER_TEST", "true", 1); + setenv("TEST_UPLOAD_PART_ERROR", ERROR_ACCESS_DENIED, 1); + + ctx = flb_create(); + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx,in_ffd, "tag", "test", NULL); + + out_ffd = flb_output(ctx, (char *) "s3", NULL); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd,"match", "*", NULL); + flb_output_set(ctx, out_ffd,"region", "us-west-2", NULL); + flb_output_set(ctx, out_ffd,"bucket", "fluent", NULL); + flb_output_set(ctx, out_ffd,"Retry_Limit", "1", NULL); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + flb_lib_push(ctx, in_ffd, (char *) JSON_TD , (int) sizeof(JSON_TD) - 1); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); + unsetenv("TEST_UPLOAD_PART_ERROR"); +} + +void flb_test_s3_complete_upload_error(void) +{ + int ret; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + /* mocks calls- signals that we are in test mode */ + setenv("FLB_S3_PLUGIN_UNDER_TEST", "true", 1); + setenv("TEST_COMPLETE_MULTIPART_UPLOAD_ERROR", ERROR_ACCESS_DENIED, 1); + + ctx = flb_create(); + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx,in_ffd, "tag", "test", NULL); + + out_ffd = flb_output(ctx, (char *) "s3", NULL); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd,"match", "*", NULL); + flb_output_set(ctx, out_ffd,"region", "us-west-2", NULL); + flb_output_set(ctx, out_ffd,"bucket", "fluent", NULL); + flb_output_set(ctx, out_ffd,"Retry_Limit", "1", NULL); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + flb_lib_push(ctx, in_ffd, (char *) JSON_TD , (int) sizeof(JSON_TD) - 1); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); + unsetenv("TEST_COMPLETE_MULTIPART_UPLOAD_ERROR"); +} + + +/* Test list */ +TEST_LIST = { + {"multipart_success", flb_test_s3_multipart_success }, + {"putobject_success", flb_test_s3_putobject_success }, + {"putobject_error", flb_test_s3_putobject_error }, + {"create_upload_error", flb_test_s3_create_upload_error }, + {"upload_part_error", flb_test_s3_upload_part_error }, + {"complete_upload_error", flb_test_s3_complete_upload_error }, + {NULL, NULL} +}; diff --git a/src/fluent-bit/tests/runtime/out_skywalking.c b/src/fluent-bit/tests/runtime/out_skywalking.c new file mode 100644 index 000000000..b59d7086e --- /dev/null +++ b/src/fluent-bit/tests/runtime/out_skywalking.c @@ -0,0 +1,55 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2019-2020 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <fluent-bit.h> + +#include "flb_tests_runtime.h" +#include "data/td/json_td.h" + +void flb_test_sw_success(void) { + int ret; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + setenv("FLB_SW_PLUGIN_UNDER_TEST", "true", 1); + + ctx = flb_create(); + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx,in_ffd, "tag", "test", NULL); + + out_ffd = flb_output(ctx, (char *) "skywalking", NULL); + TEST_CHECK(out_ffd >= 0); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + flb_lib_push(ctx, in_ffd, (char *) JSON_TD , (int) sizeof(JSON_TD) - 1); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + +TEST_LIST = { + {"sw_success", flb_test_sw_success}, + {NULL, NULL} +}; diff --git a/src/fluent-bit/tests/runtime/out_splunk.c b/src/fluent-bit/tests/runtime/out_splunk.c new file mode 100644 index 000000000..407c26b8b --- /dev/null +++ b/src/fluent-bit/tests/runtime/out_splunk.c @@ -0,0 +1,150 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2019-2021 The Fluent Bit Authors + * Copyright (C) 2015-2018 Treasure Data Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <fluent-bit.h> +#include <fluent-bit/flb_sds.h> +#include "flb_tests_runtime.h" + +#define JSON_BASIC "[12345678, {\"key\":\"value\"}]" +static void cb_check_basic(void *ctx, int ffd, + int res_ret, void *res_data, size_t res_size, + void *data) +{ + char *p; + flb_sds_t out_js = res_data; + char *index_line = "\"event\":{\"key\":\"value\"}"; + + p = strstr(out_js, index_line); + if (!TEST_CHECK(p != NULL)) { + TEST_MSG("Given:%s", out_js); + } + + flb_sds_destroy(out_js); +} + +static void cb_check_send_raw(void *ctx, int ffd, + int res_ret, void *res_data, size_t res_size, + void *data) +{ + char *p; + flb_sds_t out_js = res_data; + char *not_match_line = "\"event\":{\"key\":\"value\"}"; + char *match_line = "\"key\":\"value\""; + + p = strstr(out_js, not_match_line); + if (!TEST_CHECK(p == NULL)) { + TEST_MSG("Given:%s", out_js); + } + p = strstr(out_js, match_line); + if (!TEST_CHECK(p != NULL)) { + TEST_MSG("Given:%s", out_js); + } + + flb_sds_destroy(out_js); +} + +// Test "Splunk_Send_Raw" property. +void flb_test_send_raw() +{ + int ret; + int size = sizeof(JSON_BASIC) - 1; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + + /* Lib input mode */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + /* Elasticsearch output */ + out_ffd = flb_output(ctx, (char *) "splunk", NULL); + flb_output_set(ctx, out_ffd, + "match", "test", + "http_user", "alice", + "splunk_send_raw", "true", + NULL); + + /* Enable test mode */ + ret = flb_output_set_test(ctx, out_ffd, "formatter", + cb_check_send_raw, + NULL, NULL); + + /* Start */ + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + flb_lib_push(ctx, in_ffd, (char *) JSON_BASIC, size); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_basic() +{ + int ret; + int size = sizeof(JSON_BASIC) - 1; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + + /* Lib input mode */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + /* Elasticsearch output */ + out_ffd = flb_output(ctx, (char *) "splunk", NULL); + flb_output_set(ctx, out_ffd, + "match", "test", + "http_user", "alice", + NULL); + + /* Enable test mode */ + ret = flb_output_set_test(ctx, out_ffd, "formatter", + cb_check_basic, + NULL, NULL); + + /* Start */ + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + flb_lib_push(ctx, in_ffd, (char *) JSON_BASIC, size); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + +/* Test list */ +TEST_LIST = { + {"basic" , flb_test_basic }, + {"send_raw" , flb_test_send_raw}, + {NULL, NULL} +}; diff --git a/src/fluent-bit/tests/runtime/out_stackdriver.c b/src/fluent-bit/tests/runtime/out_stackdriver.c new file mode 100644 index 000000000..f379aa2e6 --- /dev/null +++ b/src/fluent-bit/tests/runtime/out_stackdriver.c @@ -0,0 +1,6252 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2019-2020 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <fluent-bit.h> +#include <fluent-bit/flb_pack.h> +#include <fluent-bit/flb_record_accessor.h> +#include <fluent-bit/flb_ra_key.h> + +#include "flb_tests_runtime.h" + +/* Local 'test' credentials file */ +#define SERVICE_CREDENTIALS \ + FLB_TESTS_DATA_PATH "/data/stackdriver/stackdriver-credentials.json" +#define STACKDRIVER_DATA_PATH "/data/stackdriver" + +/* JSON payload example */ +#include "data/stackdriver/json.h" +#include "data/stackdriver/stackdriver_test_operation.h" +#include "data/stackdriver/stackdriver_test_k8s_resource.h" +#include "data/stackdriver/stackdriver_test_labels.h" +#include "data/stackdriver/stackdriver_test_trace.h" +#include "data/stackdriver/stackdriver_test_span_id.h" +#include "data/stackdriver/stackdriver_test_trace_sampled.h" +#include "data/stackdriver/stackdriver_test_log_name.h" +#include "data/stackdriver/stackdriver_test_resource_labels.h" +#include "data/stackdriver/stackdriver_test_insert_id.h" +#include "data/stackdriver/stackdriver_test_source_location.h" +#include "data/stackdriver/stackdriver_test_http_request.h" +#include "data/stackdriver/stackdriver_test_timestamp.h" +#include "data/stackdriver/stackdriver_test_monitored_resource.h" + + +/* + * Fluent Bit Stackdriver plugin, always set as payload a JSON strings contained in a + * 'sds'. Since we want to validate specific keys and it values we expose here some + * helper functions to make testing easier. + * + * The approach is: + * + * - Convert formatter JSON to msgpack + * - use the helper function to check keys and values + * + * it returns FLB_TRUE if expected 'key/val' matches or FLB_FALSE if 'key' no exists + * or if there is a mismatch. + */ +static int mp_kv_cmp(char *json_data, size_t json_len, char *key_accessor, char *val) +{ + int ret; + int type; + char *mp_buf = NULL; + size_t mp_size; + size_t off = 0; + msgpack_object map; + msgpack_unpacked result; + struct flb_ra_value *rval = NULL; + struct flb_record_accessor *ra = NULL; + + /* Convert JSON to msgpack */ + ret = flb_pack_json((const char *) json_data, json_len, &mp_buf, &mp_size, + &type, NULL); + TEST_CHECK(ret != -1); + + /* Set return status */ + ret = FLB_FALSE; + + /* Unpack msgpack and reference the main 'map' */ + msgpack_unpacked_init(&result); + ret = msgpack_unpack_next(&result, mp_buf, mp_size, &off); + TEST_CHECK(ret == MSGPACK_UNPACK_SUCCESS); + map = result.data; + + /* Create a record_accessor context */ + ra = flb_ra_create(key_accessor, FLB_TRUE); + if (!ra) { + flb_error("invalid record accessor key, aborting test"); + goto out; + } + + rval = flb_ra_get_value_object(ra, map); + TEST_CHECK(rval != NULL); + msgpack_unpacked_destroy(&result); + if (!rval) { + goto out; + } + + /* We only validate strings, feel free to expand it as needed */ + TEST_CHECK(rval->type == FLB_RA_STRING); + if (strcmp(rval->val.string, val) == 0) { + ret = FLB_TRUE; + } + + out: + if (rval) { + flb_ra_key_value_destroy(rval); + } + if (ra) { + flb_ra_destroy(ra); + } + if (mp_buf) { + flb_free(mp_buf); + } + return ret; +} + +static int mp_kv_cmp_integer(char *json_data, size_t json_len, char *key_accessor, int64_t val) +{ + int ret; + int type; + char *mp_buf = NULL; + size_t mp_size; + size_t off = 0; + msgpack_object map; + msgpack_unpacked result; + struct flb_ra_value *rval = NULL; + struct flb_record_accessor *ra = NULL; + + /* Convert JSON to msgpack */ + ret = flb_pack_json((const char *) json_data, json_len, &mp_buf, &mp_size, + &type, NULL); + TEST_CHECK(ret != -1); + + /* Set return status */ + ret = FLB_FALSE; + + /* Unpack msgpack and reference the main 'map' */ + msgpack_unpacked_init(&result); + ret = msgpack_unpack_next(&result, mp_buf, mp_size, &off); + TEST_CHECK(ret == MSGPACK_UNPACK_SUCCESS); + map = result.data; + + /* Create a record_accessor context */ + ra = flb_ra_create(key_accessor, FLB_TRUE); + if (!ra) { + flb_error("invalid record accessor key, aborting test"); + goto out; + } + + rval = flb_ra_get_value_object(ra, map); + TEST_CHECK(rval != NULL); + msgpack_unpacked_destroy(&result); + if (!rval) { + goto out; + } + + TEST_CHECK(rval->type == FLB_RA_INT); + if (rval->val.i64 == val) { + ret = FLB_TRUE; + } + else { + ret = FLB_FALSE; + } + + out: + if (rval) { + flb_ra_key_value_destroy(rval); + } + if (ra) { + flb_ra_destroy(ra); + } + if (mp_buf) { + flb_free(mp_buf); + } + return ret; +} + +static int mp_kv_cmp_boolean(char *json_data, size_t json_len, char *key_accessor, bool val) +{ + int ret; + int type; + char *mp_buf = NULL; + size_t mp_size; + size_t off = 0; + msgpack_object map; + msgpack_unpacked result; + struct flb_ra_value *rval = NULL; + struct flb_record_accessor *ra = NULL; + + /* Convert JSON to msgpack */ + ret = flb_pack_json((const char *) json_data, json_len, &mp_buf, &mp_size, + &type, NULL); + TEST_CHECK(ret != -1); + + /* Set return status */ + ret = FLB_FALSE; + + /* Unpack msgpack and reference the main 'map' */ + msgpack_unpacked_init(&result); + ret = msgpack_unpack_next(&result, mp_buf, mp_size, &off); + TEST_CHECK(ret == MSGPACK_UNPACK_SUCCESS); + map = result.data; + + /* Create a record_accessor context */ + ra = flb_ra_create(key_accessor, FLB_TRUE); + if (!ra) { + flb_error("invalid record accessor key, aborting test"); + goto out; + } + + rval = flb_ra_get_value_object(ra, map); + TEST_CHECK(rval != NULL); + msgpack_unpacked_destroy(&result); + if (!rval) { + goto out; + } + + TEST_CHECK(rval->type == FLB_RA_BOOL); + if (rval->val.boolean == val) { + ret = FLB_TRUE; + } + else { + ret = FLB_FALSE; + } + + out: + if (rval) { + flb_ra_key_value_destroy(rval); + } + if (ra) { + flb_ra_destroy(ra); + } + if (mp_buf) { + flb_free(mp_buf); + } + return ret; +} + +static int mp_kv_exists(char *json_data, size_t json_len, char *key_accessor) +{ + int ret; + int type; + char *mp_buf = NULL; + size_t mp_size; + size_t off = 0; + msgpack_object map; + msgpack_unpacked result; + struct flb_ra_value *rval = NULL; + struct flb_record_accessor *ra = NULL; + + /* Convert JSON to msgpack */ + ret = flb_pack_json((const char *) json_data, json_len, &mp_buf, &mp_size, + &type, NULL); + TEST_CHECK(ret != -1); + + /* Set return status */ + ret = FLB_FALSE; + + /* Unpack msgpack and reference the main 'map' */ + msgpack_unpacked_init(&result); + ret = msgpack_unpack_next(&result, mp_buf, mp_size, &off); + TEST_CHECK(ret == MSGPACK_UNPACK_SUCCESS); + map = result.data; + + /* Create a record_accessor context */ + ra = flb_ra_create(key_accessor, FLB_TRUE); + if (!ra) { + flb_error("invalid record accessor key, aborting test"); + goto out; + } + + rval = flb_ra_get_value_object(ra, map); + msgpack_unpacked_destroy(&result); + if (rval) { + ret = FLB_TRUE; + } + else { + ret = FLB_FALSE; + } + + out: + if (rval) { + flb_ra_key_value_destroy(rval); + } + if (ra) { + flb_ra_destroy(ra); + } + if (mp_buf) { + flb_free(mp_buf); + } + return ret; +} + +static void cb_check_monitored_resource(void *ctx, int ffd, + int res_ret, void *res_data, size_t res_size, + void *data) +{ + int ret; + + ret = mp_kv_cmp(res_data, res_size, "$resource['type']", "monitored_resource"); + TEST_CHECK(ret == FLB_TRUE); + + ret = mp_kv_cmp(res_data, res_size, + "$resource['labels']['project_id']", "monitored_resource_project_id"); + TEST_CHECK(ret == FLB_TRUE); + + ret = mp_kv_cmp(res_data, res_size, + "$resource['labels']['location']", "monitored_resource_location"); + TEST_CHECK(ret == FLB_TRUE); + + ret = mp_kv_cmp(res_data, res_size, + "$resource['labels']['testA']", "valA"); + TEST_CHECK(ret == FLB_TRUE); + + ret = mp_kv_exists(res_data, res_size, + "$entries[0]['jsonPayload']['logging.googleapis.com/monitored_resource']"); + TEST_CHECK(ret == FLB_FALSE); + + flb_sds_destroy(res_data); +} + +static void cb_check_monitored_resource_priority_higher_than_local_resource_id(void *ctx, int ffd, + int res_ret, void *res_data, size_t res_size, + void *data) +{ + int ret; + + ret = mp_kv_cmp(res_data, res_size, "$resource['type']", "k8s_container"); + TEST_CHECK(ret == FLB_TRUE); + + ret = mp_kv_cmp(res_data, res_size, + "$resource['labels']['project_id']", "monitored_resource_project_id"); + TEST_CHECK(ret == FLB_TRUE); + + ret = mp_kv_cmp(res_data, res_size, + "$resource['labels']['location']", "monitored_resource_location"); + TEST_CHECK(ret == FLB_TRUE); + + ret = mp_kv_cmp(res_data, res_size, + "$resource['labels']['cluster_name']", "monitored_resource_cluster_name"); + TEST_CHECK(ret == FLB_TRUE); + + ret = mp_kv_cmp(res_data, res_size, + "$resource['labels']['namespace_name']", "monitored_resource_namespace_name"); + TEST_CHECK(ret == FLB_TRUE); + + ret = mp_kv_cmp(res_data, res_size, + "$resource['labels']['pod_name']", "monitored_resource_pod_name"); + TEST_CHECK(ret == FLB_TRUE); + + ret = mp_kv_cmp(res_data, res_size, + "$resource['labels']['container_name']", "monitored_resource_container_name"); + TEST_CHECK(ret == FLB_TRUE); + + ret = mp_kv_exists(res_data, res_size, + "$entries[0]['jsonPayload']['logging.googleapis.com/monitored_resource']"); + TEST_CHECK(ret == FLB_FALSE); + + flb_sds_destroy(res_data); +} + +static void cb_check_monitored_resource_priority_higher_than_gce_instance(void *ctx, int ffd, + int res_ret, void *res_data, size_t res_size, + void *data) +{ + int ret; + + ret = mp_kv_cmp(res_data, res_size, "$resource['type']", "gce_instance"); + TEST_CHECK(ret == FLB_TRUE); + + ret = mp_kv_cmp(res_data, res_size, + "$resource['labels']['project_id']", "monitored_resource_project_id"); + TEST_CHECK(ret == FLB_TRUE); + + ret = mp_kv_cmp(res_data, res_size, + "$resource['labels']['zone']", "monitored_resource_zone"); + TEST_CHECK(ret == FLB_TRUE); + + ret = mp_kv_cmp(res_data, res_size, + "$resource['labels']['instance_id']", "monitored_resource_instance_id"); + TEST_CHECK(ret == FLB_TRUE); + + ret = mp_kv_exists(res_data, res_size, + "$entries[0]['jsonPayload']['logging.googleapis.com/monitored_resource']"); + TEST_CHECK(ret == FLB_FALSE); + + flb_sds_destroy(res_data); +} + +static void cb_check_global_resource(void *ctx, int ffd, + int res_ret, void *res_data, size_t res_size, + void *data) +{ + int ret; + + ret = mp_kv_cmp(res_data, res_size, "$resource['type']", "global"); + TEST_CHECK(ret == FLB_TRUE); + + flb_sds_destroy(res_data); +} + +static void cb_check_generic_node_creds(void *ctx, int ffd, + int res_ret, void *res_data, size_t res_size, + void *data) +{ + int ret; + + /* resource type */ + ret = mp_kv_cmp(res_data, res_size, "$resource['type']", "generic_node"); + TEST_CHECK(ret == FLB_TRUE); + + /* project id */ + ret = mp_kv_cmp(res_data, res_size, + "$resource['labels']['project_id']", "fluent-bit"); + TEST_CHECK(ret == FLB_TRUE); + + /* location */ + ret = mp_kv_cmp(res_data, res_size, + "$resource['labels']['location']", "fluent"); + TEST_CHECK(ret == FLB_TRUE); + + /* namespace */ + ret = mp_kv_cmp(res_data, res_size, + "$resource['labels']['namespace']", "test"); + TEST_CHECK(ret == FLB_TRUE); + + /* node_id */ + ret = mp_kv_cmp(res_data, res_size, + "$resource['labels']['node_id']", "333222111"); + TEST_CHECK(ret == FLB_TRUE); + + flb_sds_destroy(res_data); +} + +static void cb_check_generic_node_metadata(void *ctx, int ffd, + int res_ret, void *res_data, size_t res_size, + void *data) +{ + int ret; + + /* resource type */ + ret = mp_kv_cmp(res_data, res_size, "$resource['type']", "generic_node"); + TEST_CHECK(ret == FLB_TRUE); + + /* project id */ + ret = mp_kv_cmp(res_data, res_size, + "$resource['labels']['project_id']", "fluent-bit-test"); + TEST_CHECK(ret == FLB_TRUE); + + /* location */ + ret = mp_kv_cmp(res_data, res_size, + "$resource['labels']['location']", "fluent"); + TEST_CHECK(ret == FLB_TRUE); + + /* namespace */ + ret = mp_kv_cmp(res_data, res_size, + "$resource['labels']['namespace']", "test"); + TEST_CHECK(ret == FLB_TRUE); + + /* node_id */ + ret = mp_kv_cmp(res_data, res_size, + "$resource['labels']['node_id']", "333222111"); + TEST_CHECK(ret == FLB_TRUE); + + flb_sds_destroy(res_data); +} + +static void cb_check_generic_task_creds(void *ctx, int ffd, + int res_ret, void *res_data, size_t res_size, + void *data) +{ + int ret; + + /* resource type */ + ret = mp_kv_cmp(res_data, res_size, "$resource['type']", "generic_task"); + TEST_CHECK(ret == FLB_TRUE); + + /* project id */ + ret = mp_kv_cmp(res_data, res_size, + "$resource['labels']['project_id']", "fluent-bit"); + TEST_CHECK(ret == FLB_TRUE); + + /* location */ + ret = mp_kv_cmp(res_data, res_size, + "$resource['labels']['location']", "fluent"); + TEST_CHECK(ret == FLB_TRUE); + + /* namespace */ + ret = mp_kv_cmp(res_data, res_size, + "$resource['labels']['namespace']", "test"); + TEST_CHECK(ret == FLB_TRUE); + + /* job */ + ret = mp_kv_cmp(res_data, res_size, + "$resource['labels']['job']", "test-job"); + TEST_CHECK(ret == FLB_TRUE); + + /* task_id */ + ret = mp_kv_cmp(res_data, res_size, + "$resource['labels']['task_id']", "333222111"); + TEST_CHECK(ret == FLB_TRUE); + + flb_sds_destroy(res_data); +} + +static void cb_check_generic_task_metadata(void *ctx, int ffd, + int res_ret, void *res_data, size_t res_size, + void *data) +{ + int ret; + + /* resource type */ + ret = mp_kv_cmp(res_data, res_size, "$resource['type']", "generic_task"); + TEST_CHECK(ret == FLB_TRUE); + + /* project id */ + ret = mp_kv_cmp(res_data, res_size, + "$resource['labels']['project_id']", "fluent-bit-test"); + TEST_CHECK(ret == FLB_TRUE); + + /* location */ + ret = mp_kv_cmp(res_data, res_size, + "$resource['labels']['location']", "fluent"); + TEST_CHECK(ret == FLB_TRUE); + + /* namespace */ + ret = mp_kv_cmp(res_data, res_size, + "$resource['labels']['namespace']", "test"); + TEST_CHECK(ret == FLB_TRUE); + + /* job */ + ret = mp_kv_cmp(res_data, res_size, + "$resource['labels']['job']", "test-job"); + TEST_CHECK(ret == FLB_TRUE); + + /* task_id */ + ret = mp_kv_cmp(res_data, res_size, + "$resource['labels']['task_id']", "333222111"); + TEST_CHECK(ret == FLB_TRUE); + + flb_sds_destroy(res_data); +} + +static void cb_check_gce_instance(void *ctx, int ffd, + int res_ret, void *res_data, size_t res_size, + void *data) +{ + int ret; + + /* resource type */ + ret = mp_kv_cmp(res_data, res_size, "$resource['type']", "gce_instance"); + TEST_CHECK(ret == FLB_TRUE); + + /* project id */ + ret = mp_kv_cmp(res_data, res_size, + "$resource['labels']['project_id']", "fluent-bit-test"); + TEST_CHECK(ret == FLB_TRUE); + + /* zone */ + ret = mp_kv_cmp(res_data, res_size, + "$resource['labels']['zone']", "fluent"); + TEST_CHECK(ret == FLB_TRUE); + + /* instance_id */ + ret = mp_kv_cmp(res_data, res_size, + "$resource['labels']['instance_id']", "333222111"); + TEST_CHECK(ret == FLB_TRUE); + + flb_sds_destroy(res_data); +} + +static void cb_check_k8s_container_resource(void *ctx, int ffd, + int res_ret, void *res_data, size_t res_size, + void *data) +{ + int ret; + + /* resource type */ + ret = mp_kv_cmp(res_data, res_size, "$resource['type']", "k8s_container"); + TEST_CHECK(ret == FLB_TRUE); + + /* project id */ + ret = mp_kv_cmp(res_data, res_size, + "$resource['labels']['project_id']", "fluent-bit"); + TEST_CHECK(ret == FLB_TRUE); + + /* location */ + ret = mp_kv_cmp(res_data, res_size, + "$resource['labels']['location']", "test_cluster_location"); + TEST_CHECK(ret == FLB_TRUE); + + /* cluster name */ + ret = mp_kv_cmp(res_data, res_size, + "$resource['labels']['cluster_name']", "test_cluster_name"); + TEST_CHECK(ret == FLB_TRUE); + + /* namespace name */ + ret = mp_kv_cmp(res_data, res_size, + "$resource['labels']['namespace_name']", "testnamespace"); + TEST_CHECK(ret == FLB_TRUE); + + /* pod name */ + ret = mp_kv_cmp(res_data, res_size, + "$resource['labels']['pod_name']", "testpod"); + TEST_CHECK(ret == FLB_TRUE); + + /* container name */ + ret = mp_kv_cmp(res_data, res_size, + "$resource['labels']['container_name']", "testctr"); + TEST_CHECK(ret == FLB_TRUE); + + /* check `local_resource_id` has been removed from jsonPayload */ + ret = mp_kv_exists(res_data, res_size, + "$entries[0]['jsonPayload']['logging.googleapis.com/local_resource_id']"); + TEST_CHECK(ret == FLB_FALSE); + + flb_sds_destroy(res_data); +} + +static void cb_check_k8s_container_resource_diff_tag(void *ctx, int ffd, + int res_ret, void *res_data, size_t res_size, + void *data) +{ + int ret; + + /* resource type */ + ret = mp_kv_cmp(res_data, res_size, "$resource['type']", "k8s_container"); + TEST_CHECK(ret == FLB_TRUE); + + /* project id */ + ret = mp_kv_cmp(res_data, res_size, + "$resource['labels']['project_id']", "fluent-bit"); + TEST_CHECK(ret == FLB_TRUE); + + /* location */ + ret = mp_kv_cmp(res_data, res_size, + "$resource['labels']['location']", "test_cluster_location"); + TEST_CHECK(ret == FLB_TRUE); + + /* cluster name */ + ret = mp_kv_cmp(res_data, res_size, + "$resource['labels']['cluster_name']", "test_cluster_name"); + TEST_CHECK(ret == FLB_TRUE); + + /* namespace name */ + ret = mp_kv_cmp(res_data, res_size, + "$resource['labels']['namespace_name']", "diffnamespace"); + TEST_CHECK(ret == FLB_TRUE); + + /* pod name */ + ret = mp_kv_cmp(res_data, res_size, + "$resource['labels']['pod_name']", "diffpod"); + TEST_CHECK(ret == FLB_TRUE); + + /* container name */ + ret = mp_kv_cmp(res_data, res_size, + "$resource['labels']['container_name']", "diffctr"); + TEST_CHECK(ret == FLB_TRUE); + + /* check `local_resource_id` has been removed from jsonPayload */ + ret = mp_kv_exists(res_data, res_size, + "$entries[0]['jsonPayload']['logging.googleapis.com/local_resource_id']"); + TEST_CHECK(ret == FLB_FALSE); + + flb_sds_destroy(res_data); +} + +static void cb_check_k8s_container_resource_default_regex(void *ctx, int ffd, + int res_ret, void *res_data, size_t res_size, + void *data) +{ + int ret; + + /* resource type */ + ret = mp_kv_cmp(res_data, res_size, "$resource['type']", "k8s_container"); + TEST_CHECK(ret == FLB_TRUE); + + /* project id */ + ret = mp_kv_cmp(res_data, res_size, + "$resource['labels']['project_id']", "fluent-bit"); + TEST_CHECK(ret == FLB_TRUE); + + /* location */ + ret = mp_kv_cmp(res_data, res_size, + "$resource['labels']['location']", "test_cluster_location"); + TEST_CHECK(ret == FLB_TRUE); + + /* cluster name */ + ret = mp_kv_cmp(res_data, res_size, + "$resource['labels']['cluster_name']", "test_cluster_name"); + TEST_CHECK(ret == FLB_TRUE); + + /* namespace name */ + ret = mp_kv_cmp(res_data, res_size, + "$resource['labels']['namespace_name']", "default"); + TEST_CHECK(ret == FLB_TRUE); + + /* pod name */ + ret = mp_kv_cmp(res_data, res_size, + "$resource['labels']['pod_name']", "apache-logs-annotated"); + TEST_CHECK(ret == FLB_TRUE); + + /* container name */ + ret = mp_kv_cmp(res_data, res_size, + "$resource['labels']['container_name']", "apache"); + TEST_CHECK(ret == FLB_TRUE); + + /* check `local_resource_id` has been removed from jsonPayload */ + ret = mp_kv_exists(res_data, res_size, + "$entries[0]['jsonPayload']['logging.googleapis.com/local_resource_id']"); + TEST_CHECK(ret == FLB_FALSE); + + flb_sds_destroy(res_data); +} + +static void cb_check_trace_no_autoformat(void *ctx, int ffd, + int res_ret, void *res_data, size_t res_size, + void *data) +{ + int ret; + + /* trace in the entries */ + ret = mp_kv_cmp(res_data, res_size, "$entries[0]['trace']", "test-trace-id-xyz"); + TEST_CHECK(ret == FLB_TRUE); + + /* trace has been removed from jsonPayload */ + ret = mp_kv_exists(res_data, res_size, "$entries[0]['jsonPayload']['trace']"); + TEST_CHECK(ret == FLB_FALSE); + + flb_sds_destroy(res_data); +} + +static void cb_check_trace_stackdriver_autoformat(void *ctx, int ffd, + int res_ret, void *res_data, size_t res_size, + void *data) +{ + int ret; + + /* trace in the entries */ + ret = mp_kv_cmp( + res_data, + res_size, + "$entries[0]['trace']", + "projects/fluent-bit-test/traces/test-trace-id-xyz"); + TEST_CHECK(ret == FLB_TRUE); + + /* trace has been removed from jsonPayload */ + ret = mp_kv_exists(res_data, res_size, "$entries[0]['jsonPayload']['trace']"); + TEST_CHECK(ret == FLB_FALSE); + + flb_sds_destroy(res_data); +} + +static void cb_check_span_id(void *ctx, int ffd, + int res_ret, void *res_data, size_t res_size, + void *data) +{ + int ret; + + /* span id in the entries */ + ret = mp_kv_cmp(res_data, res_size, "$entries[0]['spanId']", "000000000000004a"); + TEST_CHECK(ret == FLB_TRUE); + + /* span id has been removed from jsonPayload */ + ret = mp_kv_exists(res_data, res_size, + "$entries[0]['jsonPayload']['logging.googleapis.com/spanId']"); + TEST_CHECK(ret == FLB_FALSE); + + flb_sds_destroy(res_data); +} + +static void cb_check_trace_sampled_true(void *ctx, int ffd, + int res_ret, void *res_data, size_t res_size, + void *data) +{ + int ret; + + /* trace sampled in the entries */ + ret = mp_kv_cmp_boolean(res_data, res_size, "$entries[0]['traceSampled']", true); + TEST_CHECK(ret == FLB_TRUE); + + /* trace sampled has been removed from jsonPayload */ + ret = mp_kv_exists(res_data, res_size, + "$entries[0]['jsonPayload']['logging.googleapis.com/traceSampled']"); + TEST_CHECK(ret == FLB_FALSE); + + flb_sds_destroy(res_data); +} + +static void cb_check_trace_sampled_false(void *ctx, int ffd, + int res_ret, void *res_data, size_t res_size, + void *data) +{ + int ret; + + /* trace sampled in the entries */ + ret = mp_kv_cmp_boolean(res_data, res_size, "$entries[0]['traceSampled']", false); + TEST_CHECK(ret == FLB_TRUE); + + /* trace sampled has been removed from jsonPayload */ + ret = mp_kv_exists(res_data, res_size, + "$entries[0]['jsonPayload']['logging.googleapis.com/traceSampled']"); + TEST_CHECK(ret == FLB_FALSE); + + flb_sds_destroy(res_data); +} + +static void cb_check_log_name_override(void *ctx, int ffd, + int res_ret, void *res_data, size_t res_size, + void *data) +{ + int ret; + + /* logName in the entries is created using the value under log_name_key */ + ret = mp_kv_cmp( + res_data, res_size, "$entries[0]['logName']", "projects/fluent-bit-test/logs/custom_log_name"); + TEST_CHECK(ret == FLB_TRUE); + + /* log_name_key has been removed from jsonPayload */ + ret = mp_kv_exists(res_data, res_size, "$entries[0]['jsonPayload']['custom_log_name_key']"); + TEST_CHECK(ret == FLB_FALSE); + + flb_sds_destroy(res_data); +} + +static void cb_check_log_name_no_override(void *ctx, int ffd, + int res_ret, void *res_data, size_t res_size, + void *data) +{ + int ret; + + /* logName in the entries is created using the tag */ + ret = mp_kv_cmp( + res_data, res_size, "$entries[0]['logName']", "projects/fluent-bit-test/logs/test"); + TEST_CHECK(ret == FLB_TRUE); + + flb_sds_destroy(res_data); +} + +static void cb_check_k8s_node_resource(void *ctx, int ffd, + int res_ret, void *res_data, size_t res_size, + void *data) +{ + int ret; + + /* resource type */ + ret = mp_kv_cmp(res_data, res_size, "$resource['type']", "k8s_node"); + TEST_CHECK(ret == FLB_TRUE); + + /* project id */ + ret = mp_kv_cmp(res_data, res_size, + "$resource['labels']['project_id']", "fluent-bit"); + TEST_CHECK(ret == FLB_TRUE); + + /* location */ + ret = mp_kv_cmp(res_data, res_size, + "$resource['labels']['location']", "test_cluster_location"); + TEST_CHECK(ret == FLB_TRUE); + + /* cluster name */ + ret = mp_kv_cmp(res_data, res_size, + "$resource['labels']['cluster_name']", "test_cluster_name"); + TEST_CHECK(ret == FLB_TRUE); + + /* node name */ + ret = mp_kv_cmp(res_data, res_size, + "$resource['labels']['node_name']", "testnode"); + TEST_CHECK(ret == FLB_TRUE); + + /* check `local_resource_id` has been removed from jsonPayload */ + ret = mp_kv_exists(res_data, res_size, + "$entries[0]['jsonPayload']['logging.googleapis.com/local_resource_id']"); + TEST_CHECK(ret == FLB_FALSE); + + flb_sds_destroy(res_data); +} + +static void cb_check_k8s_node_custom_k8s_regex(void *ctx, int ffd, + int res_ret, void *res_data, size_t res_size, + void *data) +{ + int ret; + + /* resource type */ + ret = mp_kv_cmp(res_data, res_size, "$resource['type']", "k8s_node"); + TEST_CHECK(ret == FLB_TRUE); + + /* project id */ + ret = mp_kv_cmp(res_data, res_size, + "$resource['labels']['project_id']", "fluent-bit"); + TEST_CHECK(ret == FLB_TRUE); + + /* location */ + ret = mp_kv_cmp(res_data, res_size, + "$resource['labels']['location']", "test_cluster_location"); + TEST_CHECK(ret == FLB_TRUE); + + /* cluster name */ + ret = mp_kv_cmp(res_data, res_size, + "$resource['labels']['cluster_name']", "test_cluster_name"); + TEST_CHECK(ret == FLB_TRUE); + + /* node name */ + ret = mp_kv_cmp(res_data, res_size, + "$resource['labels']['node_name']", "testnode.withdot.dot"); + TEST_CHECK(ret == FLB_TRUE); + + /* check `local_resource_id` has been removed from jsonPayload */ + ret = mp_kv_exists(res_data, res_size, + "$entries[0]['jsonPayload']['logging.googleapis.com/local_resource_id']"); + TEST_CHECK(ret == FLB_FALSE); + + flb_sds_destroy(res_data); +} + +static void cb_check_k8s_pod_resource(void *ctx, int ffd, + int res_ret, void *res_data, size_t res_size, + void *data) +{ + int ret; + + /* resource type */ + ret = mp_kv_cmp(res_data, res_size, "$resource['type']", "k8s_pod"); + TEST_CHECK(ret == FLB_TRUE); + + /* project id */ + ret = mp_kv_cmp(res_data, res_size, + "$resource['labels']['project_id']", "fluent-bit"); + TEST_CHECK(ret == FLB_TRUE); + + /* location */ + ret = mp_kv_cmp(res_data, res_size, + "$resource['labels']['location']", "test_cluster_location"); + TEST_CHECK(ret == FLB_TRUE); + + /* cluster name */ + ret = mp_kv_cmp(res_data, res_size, + "$resource['labels']['cluster_name']", "test_cluster_name"); + TEST_CHECK(ret == FLB_TRUE); + + /* namespace name */ + ret = mp_kv_cmp(res_data, res_size, + "$resource['labels']['namespace_name']", "testnamespace"); + TEST_CHECK(ret == FLB_TRUE); + + /* pod name */ + ret = mp_kv_cmp(res_data, res_size, + "$resource['labels']['pod_name']", "testpod"); + TEST_CHECK(ret == FLB_TRUE); + + /* check `local_resource_id` has been removed from jsonPayload */ + ret = mp_kv_exists(res_data, res_size, + "$entries[0]['jsonPayload']['logging.googleapis.com/local_resource_id']"); + TEST_CHECK(ret == FLB_FALSE); + + flb_sds_destroy(res_data); +} + +static void cb_check_insert_id_common_case(void *ctx, int ffd, + int res_ret, void *res_data, size_t res_size, + void *data) +{ + int ret; + + /* insertId in the entries */ + ret = mp_kv_cmp(res_data, res_size, "$entries[0]['insertId']", "test_insertId"); + TEST_CHECK(ret == FLB_TRUE); + + /* insertId has been removed from jsonPayload */ + ret = mp_kv_exists(res_data, res_size, "$entries[0]['jsonPayload']['logging.googleapis.com/insertId']"); + TEST_CHECK(ret == FLB_FALSE); + + flb_sds_destroy(res_data); +} + +static void cb_check_empty_insert_id(void *ctx, int ffd, + int res_ret, void *res_data, size_t res_size, + void *data) +{ + TEST_CHECK(res_size == 0); + + flb_sds_destroy(res_data); +} + +static void cb_check_insert_id_incorrect_type(void *ctx, int ffd, + int res_ret, void *res_data, size_t res_size, + void *data) +{ + TEST_CHECK(res_size == 0); + + flb_sds_destroy(res_data); +} + +static void cb_check_operation_common_case(void *ctx, int ffd, + int res_ret, void *res_data, size_t res_size, + void *data) +{ + int ret; + + /* operation_id */ + ret = mp_kv_cmp(res_data, res_size, "$entries[0]['operation']['id']", "test_id"); + TEST_CHECK(ret == FLB_TRUE); + + /* operation_producer */ + ret = mp_kv_cmp(res_data, res_size, "$entries[0]['operation']['producer']", "test_producer"); + TEST_CHECK(ret == FLB_TRUE); + + /* operation_first */ + ret = mp_kv_cmp_boolean(res_data, res_size, "$entries[0]['operation']['first']", true); + TEST_CHECK(ret == FLB_TRUE); + + /* operation_last */ + ret = mp_kv_cmp_boolean(res_data, res_size, "$entries[0]['operation']['last']", true); + TEST_CHECK(ret == FLB_TRUE); + + /* check `operation` has been removed from jsonPayload */ + ret = mp_kv_exists(res_data, res_size, "$entries[0]['jsonPayload']['logging.googleapis.com/operation']"); + TEST_CHECK(ret == FLB_FALSE); + + flb_sds_destroy(res_data); +} + +static void cb_check_empty_operation(void *ctx, int ffd, + int res_ret, void *res_data, size_t res_size, + void *data) +{ + int ret; + + /* operation_id */ + ret = mp_kv_cmp(res_data, res_size, "$entries[0]['operation']['id']", ""); + TEST_CHECK(ret == FLB_TRUE); + + /* operation_producer */ + ret = mp_kv_cmp(res_data, res_size, "$entries[0]['operation']['producer']", ""); + TEST_CHECK(ret == FLB_TRUE); + + /* operation_first */ + ret = mp_kv_cmp_boolean(res_data, res_size, "$entries[0]['operation']['first']", false); + TEST_CHECK(ret == FLB_TRUE); + + /* operation_last */ + ret = mp_kv_cmp_boolean(res_data, res_size, "$entries[0]['operation']['last']", false); + TEST_CHECK(ret == FLB_TRUE); + + /* check `operation` has been removed from jsonPayload */ + ret = mp_kv_exists(res_data, res_size, "$entries[0]['jsonPayload']['logging.googleapis.com/operation']"); + TEST_CHECK(ret == FLB_FALSE); + + flb_sds_destroy(res_data); +} + +static void cb_check_operation_in_string(void *ctx, int ffd, + int res_ret, void *res_data, size_t res_size, + void *data) +{ + int ret; + + /* 'operation' is not a map, won't be extracted from jsonPayload */ + ret = mp_kv_cmp(res_data, res_size, "$entries[0]['jsonPayload']['logging.googleapis.com/operation']", "some string"); + TEST_CHECK(ret == FLB_TRUE); + + ret = mp_kv_exists(res_data, res_size, "$entries[0]['operation']"); + TEST_CHECK(ret == FLB_FALSE); + + flb_sds_destroy(res_data); +} + +static void cb_check_operation_partial_subfields(void *ctx, int ffd, + int res_ret, void *res_data, size_t res_size, + void *data) +{ + int ret; + + /* operation_id */ + ret = mp_kv_cmp(res_data, res_size, "$entries[0]['operation']['id']", ""); + TEST_CHECK(ret == FLB_TRUE); + + /* operation_producer */ + ret = mp_kv_cmp(res_data, res_size, "$entries[0]['operation']['producer']", ""); + TEST_CHECK(ret == FLB_TRUE); + + /* operation_first */ + ret = mp_kv_cmp_boolean(res_data, res_size, "$entries[0]['operation']['first']", false); + TEST_CHECK(ret == FLB_TRUE); + + /* operation_last */ + ret = mp_kv_cmp_boolean(res_data, res_size, "$entries[0]['operation']['last']", false); + TEST_CHECK(ret == FLB_TRUE); + + /* check `operation` has been removed from jsonPayload */ + ret = mp_kv_exists(res_data, res_size, "$entries[0]['jsonPayload']['logging.googleapis.com/operation']"); + TEST_CHECK(ret == FLB_FALSE); + + flb_sds_destroy(res_data); +} + +static void cb_check_operation_incorrect_type_subfields(void *ctx, int ffd, + int res_ret, void *res_data, size_t res_size, + void *data) +{ + int ret; + + /* operation_id */ + ret = mp_kv_cmp(res_data, res_size, "$entries[0]['operation']['id']", ""); + TEST_CHECK(ret == FLB_TRUE); + + /* operation_producer */ + ret = mp_kv_cmp(res_data, res_size, "$entries[0]['operation']['producer']", ""); + TEST_CHECK(ret == FLB_TRUE); + + /* operation_first */ + ret = mp_kv_cmp_boolean(res_data, res_size, "$entries[0]['operation']['first']", false); + TEST_CHECK(ret == FLB_TRUE); + + /* operation_last */ + ret = mp_kv_cmp_boolean(res_data, res_size, "$entries[0]['operation']['last']", false); + TEST_CHECK(ret == FLB_TRUE); + + /* check `operation` has been removed from jsonPayload */ + ret = mp_kv_exists(res_data, res_size, "$entries[0]['jsonPayload']['logging.googleapis.com/operation']"); + TEST_CHECK(ret == FLB_FALSE); + + flb_sds_destroy(res_data); +} + +static void cb_check_operation_extra_subfields(void *ctx, int ffd, + int res_ret, void *res_data, size_t res_size, + void *data) +{ + int ret; + + /* operation_id */ + ret = mp_kv_cmp(res_data, res_size, "$entries[0]['operation']['id']", "test_id"); + TEST_CHECK(ret == FLB_TRUE); + + /* operation_producer */ + ret = mp_kv_cmp(res_data, res_size, "$entries[0]['operation']['producer']", "test_producer"); + TEST_CHECK(ret == FLB_TRUE); + + /* operation_first */ + ret = mp_kv_cmp_boolean(res_data, res_size, "$entries[0]['operation']['first']", true); + TEST_CHECK(ret == FLB_TRUE); + + /* operation_last */ + ret = mp_kv_cmp_boolean(res_data, res_size, "$entries[0]['operation']['last']", true); + TEST_CHECK(ret == FLB_TRUE); + + /* Preserve extra subfields inside jsonPayload */ + ret = mp_kv_cmp(res_data, res_size, "$entries[0]['jsonPayload']['logging.googleapis.com/operation']['extra_key1']", "extra_val1"); + TEST_CHECK(ret == FLB_TRUE); + + ret = mp_kv_cmp_integer(res_data, res_size, "$entries[0]['jsonPayload']['logging.googleapis.com/operation']['extra_key2']", 123); + TEST_CHECK(ret == FLB_TRUE); + + ret = mp_kv_cmp_boolean(res_data, res_size, "$entries[0]['jsonPayload']['logging.googleapis.com/operation']['extra_key3']", true); + TEST_CHECK(ret == FLB_TRUE); + + flb_sds_destroy(res_data); +} + +static void cb_check_default_labels(void *ctx, int ffd, + int res_ret, void *res_data, size_t res_size, + void *data) +{ + int ret; + + /* check 'labels' field has been added to root-level of log entry */ + ret = mp_kv_exists(res_data, res_size, "$entries[0]['labels']"); + TEST_CHECK(ret == FLB_TRUE); + + /* check fields inside 'labels' field */ + ret = mp_kv_exists(res_data, res_size, "$entries[0]['labels']['testA']"); + TEST_CHECK(ret == FLB_TRUE); + + /* check field inside 'labels' field */ + ret = mp_kv_exists(res_data, res_size, "$entries[0]['labels']['testB']"); + TEST_CHECK(ret == FLB_TRUE); + + /* check `labels_key` has been removed from jsonPayload */ + ret = mp_kv_exists(res_data, res_size, "$entries[0]['jsonPayload']['logging.googleapis.com/labels']"); + TEST_CHECK(ret == FLB_FALSE); + + flb_sds_destroy(res_data); +} + +static void cb_check_custom_labels(void *ctx, int ffd, + int res_ret, void *res_data, size_t res_size, + void *data) +{ + int ret; + + /* check 'labels' field has been added to root-level of log entry */ + ret = mp_kv_exists(res_data, res_size, "$entries[0]['labels']"); + TEST_CHECK(ret == FLB_TRUE); + + /* check fields inside 'labels' field */ + ret = mp_kv_exists(res_data, res_size, "$entries[0]['labels']['testA']"); + TEST_CHECK(ret == FLB_TRUE); + + /* check field inside 'labels' field */ + ret = mp_kv_exists(res_data, res_size, "$entries[0]['labels']['testB']"); + TEST_CHECK(ret == FLB_TRUE); + + /* check `labels_key` has been removed from jsonPayload */ + ret = mp_kv_exists(res_data, res_size, "$entries[0]['jsonPayload']['logging.googleapis.com/customlabels']"); + TEST_CHECK(ret == FLB_FALSE); + + flb_sds_destroy(res_data); +} + +static void cb_check_config_labels_no_conflict(void *ctx, int ffd, + int res_ret, void *res_data, size_t res_size, + void *data) +{ + int ret; + + /* check 'labels' field has been added to root-level of log entry */ + ret = mp_kv_exists(res_data, res_size, "$entries[0]['labels']"); + TEST_CHECK(ret == FLB_TRUE); + + /* check fields inside 'labels' field */ + ret = mp_kv_exists(res_data, res_size, "$entries[0]['labels']['testA']"); + TEST_CHECK(ret == FLB_TRUE); + + /* check field inside 'labels' field */ + ret = mp_kv_exists(res_data, res_size, "$entries[0]['labels']['testB']"); + TEST_CHECK(ret == FLB_TRUE); + + /* check field inside 'labels' field */ + ret = mp_kv_exists(res_data, res_size, "$entries[0]['labels']['testC']"); + TEST_CHECK(ret == FLB_TRUE); + + /* check `labels_key` has been removed from jsonPayload */ + ret = mp_kv_exists(res_data, res_size, "$entries[0]['jsonPayload']['logging.googleapis.com/labels']"); + TEST_CHECK(ret == FLB_FALSE); + + flb_sds_destroy(res_data); +} + +static void cb_check_config_labels_conflict(void *ctx, int ffd, + int res_ret, void *res_data, size_t res_size, + void *data) +{ + int ret; + + /* check 'labels' field has been added to root-level of log entry */ + ret = mp_kv_exists(res_data, res_size, "$entries[0]['labels']"); + TEST_CHECK(ret == FLB_TRUE); + + /* check fields inside 'labels' field */ + ret = mp_kv_exists(res_data, res_size, "$entries[0]['labels']['testA']"); + TEST_CHECK(ret == FLB_TRUE); + + /* check field inside 'labels' field */ + ret = mp_kv_exists(res_data, res_size, "$entries[0]['labels']['testB']"); + TEST_CHECK(ret == FLB_TRUE); + + /* check static 'labels' override value */ + ret = mp_kv_cmp(res_data, res_size, "$entries[0]['labels']['testB']", "valC"); + TEST_CHECK(ret == FLB_TRUE); + + + /* check `labels_key` has been removed from jsonPayload */ + ret = mp_kv_exists(res_data, res_size, "$entries[0]['jsonPayload']['logging.googleapis.com/labels']"); + TEST_CHECK(ret == FLB_FALSE); + + flb_sds_destroy(res_data); +} + +static void cb_check_default_labels_k8s_resource_type(void *ctx, int ffd, + int res_ret, void *res_data, size_t res_size, + void *data) +{ + int ret; + + /* resource type */ + ret = mp_kv_cmp(res_data, res_size, "$resource['type']", "k8s_container"); + TEST_CHECK(ret == FLB_TRUE); + + /* project id */ + ret = mp_kv_cmp(res_data, res_size, + "$resource['labels']['project_id']", "fluent-bit"); + TEST_CHECK(ret == FLB_TRUE); + + /* location */ + ret = mp_kv_cmp(res_data, res_size, + "$resource['labels']['location']", "test_cluster_location"); + TEST_CHECK(ret == FLB_TRUE); + + /* cluster name */ + ret = mp_kv_cmp(res_data, res_size, + "$resource['labels']['cluster_name']", "test_cluster_name"); + TEST_CHECK(ret == FLB_TRUE); + + /* namespace name */ + ret = mp_kv_cmp(res_data, res_size, + "$resource['labels']['namespace_name']", "testnamespace"); + TEST_CHECK(ret == FLB_TRUE); + + /* pod name */ + ret = mp_kv_cmp(res_data, res_size, + "$resource['labels']['pod_name']", "testpod"); + TEST_CHECK(ret == FLB_TRUE); + + /* container name */ + ret = mp_kv_cmp(res_data, res_size, + "$resource['labels']['container_name']", "testctr"); + TEST_CHECK(ret == FLB_TRUE); + + /* check `local_resource_id` has been removed from jsonPayload */ + ret = mp_kv_exists(res_data, res_size, + "$entries[0]['jsonPayload']['logging.googleapis.com/local_resource_id']"); + TEST_CHECK(ret == FLB_FALSE); + + /* check 'labels' field has been added to root-level of log entry */ + ret = mp_kv_exists(res_data, res_size, "$entries[0]['labels']"); + TEST_CHECK(ret == FLB_TRUE); + + /* check fields inside 'labels' field */ + ret = mp_kv_exists(res_data, res_size, "$entries[0]['labels']['testA']"); + TEST_CHECK(ret == FLB_TRUE); + + /* check field inside 'labels' field */ + ret = mp_kv_exists(res_data, res_size, "$entries[0]['labels']['testB']"); + TEST_CHECK(ret == FLB_TRUE); + + /* check `labels_key` has been removed from jsonPayload */ + ret = mp_kv_exists(res_data, res_size, "$entries[0]['jsonPayload']['logging.googleapis.com/labels']"); + TEST_CHECK(ret == FLB_FALSE); + + flb_sds_destroy(res_data); +} + +static void cb_check_resource_labels_one_field(void *ctx, int ffd, + int res_ret, void *res_data, size_t res_size, void *data) +{ + int ret; + + ret = mp_kv_cmp(res_data, res_size, "$entries[0]['jsonPayload']['keyA']", "valA"); + TEST_CHECK(ret == FLB_TRUE); + + ret = mp_kv_cmp(res_data, res_size, "$resource['labels']['keyB']", "valA"); + TEST_CHECK(ret == FLB_TRUE); + + ret = mp_kv_cmp(res_data, res_size, "$resource['labels']['project_id']", "fluent-bit"); + TEST_CHECK(ret == FLB_TRUE); + + flb_sds_destroy(res_data); +} + +static void cb_check_resource_labels_plaintext(void *ctx, int ffd, + int res_ret, void *res_data, size_t res_size, void *data) +{ + int ret; + + ret = mp_kv_cmp(res_data, res_size, "$entries[0]['jsonPayload']['keyA']", "valA"); + TEST_CHECK(ret == FLB_TRUE); + + ret = mp_kv_cmp(res_data, res_size, "$resource['labels']['keyB']", "plaintext"); + TEST_CHECK(ret == FLB_TRUE); + + ret = mp_kv_cmp(res_data, res_size, "$resource['labels']['project_id']", "fluent-bit"); + TEST_CHECK(ret == FLB_TRUE); + + flb_sds_destroy(res_data); +} + +static void cb_check_resource_labels_k8s(void *ctx, int ffd, + int res_ret, void *res_data, size_t res_size, void *data) +{ + int ret; + + ret = mp_kv_cmp(res_data, res_size, "$resource['labels']['project_id']", "fluent-bit"); + TEST_CHECK(ret == FLB_TRUE); + + ret = mp_kv_cmp(res_data, res_size, + "$resource['labels']['cluster_name']", "name_from_labels"); + TEST_CHECK(ret == FLB_TRUE); + + ret = mp_kv_cmp(res_data, res_size, + "$resource['labels']['location']", "loc_from_labels"); + TEST_CHECK(ret == FLB_TRUE); + + flb_sds_destroy(res_data); +} + +static void cb_check_nested_fields_mapped(void *ctx, int ffd, + int res_ret, void *res_data, size_t res_size, void *data) +{ + int ret; + + ret = mp_kv_cmp(res_data, res_size, + "$entries[0]['jsonPayload']['toplevel']['keyB']", "valB"); + TEST_CHECK(ret == FLB_TRUE); + + ret = mp_kv_cmp(res_data, res_size, "$resource['labels']['keyD']", "valB"); + TEST_CHECK(ret == FLB_TRUE); + + ret = mp_kv_cmp(res_data, res_size, + "$entries[0]['jsonPayload']['toplevel']['keyA']", "valA"); + TEST_CHECK(ret == FLB_TRUE); + + ret = mp_kv_cmp(res_data, res_size, "$resource['labels']['keyC']", "valA"); + TEST_CHECK(ret == FLB_TRUE); + + flb_sds_destroy(res_data); +} + +static void cb_check_layered_nested_fields_mapped(void *ctx, int ffd, + int res_ret, void *res_data, size_t res_size, void *data) +{ + int ret; + + ret = mp_kv_cmp(res_data, res_size, + "$entries[0]['jsonPayload']['toplevel']['keyB']", "valB"); + TEST_CHECK(ret == FLB_TRUE); + + ret = mp_kv_cmp(res_data, res_size, "$resource['labels']['keyD']", "valB"); + TEST_CHECK(ret == FLB_TRUE); + + ret = mp_kv_cmp(res_data, res_size, + "$entries[0]['jsonPayload']['toplevel']['midlevel']['keyA']", "valA"); + TEST_CHECK(ret == FLB_TRUE); + + ret = mp_kv_cmp(res_data, res_size, "$resource['labels']['keyC']", "valA"); + TEST_CHECK(ret == FLB_TRUE); + + flb_sds_destroy(res_data); +} + +static void cb_check_resource_labels_original_does_not_exist(void *ctx, int ffd, + int res_ret, void *res_data, size_t res_size, void *data) +{ + int ret; + + ret = mp_kv_cmp(res_data, res_size, "$entries[0]['jsonPayload']['keyA']", "valA"); + TEST_CHECK(ret == FLB_TRUE); + + ret = mp_kv_exists(res_data, res_size, "$resource['labels']['keyC']"); + TEST_CHECK(ret == FLB_FALSE); + + flb_sds_destroy(res_data); +} + +static void cb_check_resource_labels_nested_original_partially_exists(void *ctx, int ffd, + int res_ret, void *res_data, size_t res_size, void *data) +{ + int ret; + + ret = mp_kv_exists(res_data, res_size, "$entries[0]['jsonPayload']['toplevel']"); + TEST_CHECK(ret == FLB_TRUE); + + ret = mp_kv_cmp(res_data, res_size, + "$entries[0]['jsonPayload']['toplevel']['keyA']", "valA"); + TEST_CHECK(ret == FLB_TRUE); + + ret = mp_kv_cmp(res_data, res_size, + "$entries[0]['jsonPayload']['toplevel']['keyB']", "valB"); + TEST_CHECK(ret == FLB_TRUE); + + ret = mp_kv_exists(res_data, res_size, "$resource['labels']['keyC']"); + TEST_CHECK(ret == FLB_FALSE); + + flb_sds_destroy(res_data); +} + +static void cb_check_multiple_fields_mapped(void *ctx, int ffd, int res_ret, + void *res_data, size_t res_size, void *data) +{ + int ret; + + ret = mp_kv_cmp(res_data, res_size, "$entries[0]['jsonPayload']['keyA']", "valA"); + TEST_CHECK(ret == FLB_TRUE); + + ret = mp_kv_cmp(res_data, res_size, "$entries[0]['jsonPayload']['keyB']", "valB"); + TEST_CHECK(ret == FLB_TRUE); + + ret = mp_kv_cmp(res_data, res_size, "$resource['labels']['keyC']", "valA"); + TEST_CHECK(ret == FLB_TRUE); + + ret = mp_kv_cmp(res_data, res_size, "$resource['labels']['keyD']", "valB"); + TEST_CHECK(ret == FLB_TRUE); + + flb_sds_destroy(res_data); +} + +static void cb_check_resource_labels_duplicate_assignment(void *ctx, int ffd, int res_ret, + void *res_data, size_t res_size, void *data) +{ + int ret; + + ret = mp_kv_cmp(res_data, res_size, "$entries[0]['jsonPayload']['keyA']", "valA"); + TEST_CHECK(ret == FLB_TRUE); + + ret = mp_kv_cmp(res_data, res_size, "$entries[0]['jsonPayload']['keyB']", "valB"); + TEST_CHECK(ret == FLB_TRUE); + + ret = mp_kv_cmp(res_data, res_size, "$resource['labels']['keyC']", "valB"); + TEST_CHECK(ret == FLB_TRUE); + + flb_sds_destroy(res_data); +} + + +static void cb_check_resource_labels_project_id_not_overridden(void *ctx, int ffd, + int res_ret, void *res_data, size_t res_size, void *data) +{ + int ret; + + ret = mp_kv_cmp(res_data, res_size, "$resource['labels']['project_id']", "fluent-bit"); + TEST_CHECK(ret == FLB_TRUE); + + flb_sds_destroy(res_data); +} + +static void cb_check_custom_labels_k8s_resource_type(void *ctx, int ffd, + int res_ret, void *res_data, size_t res_size, + void *data) +{ + int ret; + + /* resource type */ + ret = mp_kv_cmp(res_data, res_size, "$resource['type']", "k8s_container"); + TEST_CHECK(ret == FLB_TRUE); + + /* project id */ + ret = mp_kv_cmp(res_data, res_size, + "$resource['labels']['project_id']", "fluent-bit"); + TEST_CHECK(ret == FLB_TRUE); + + /* location */ + ret = mp_kv_cmp(res_data, res_size, + "$resource['labels']['location']", "test_cluster_location"); + TEST_CHECK(ret == FLB_TRUE); + + /* cluster name */ + ret = mp_kv_cmp(res_data, res_size, + "$resource['labels']['cluster_name']", "test_cluster_name"); + TEST_CHECK(ret == FLB_TRUE); + + /* namespace name */ + ret = mp_kv_cmp(res_data, res_size, + "$resource['labels']['namespace_name']", "testnamespace"); + TEST_CHECK(ret == FLB_TRUE); + + /* pod name */ + ret = mp_kv_cmp(res_data, res_size, + "$resource['labels']['pod_name']", "testpod"); + TEST_CHECK(ret == FLB_TRUE); + + /* container name */ + ret = mp_kv_cmp(res_data, res_size, + "$resource['labels']['container_name']", "testctr"); + TEST_CHECK(ret == FLB_TRUE); + + /* check `local_resource_id` has been removed from jsonPayload */ + ret = mp_kv_exists(res_data, res_size, + "$entries[0]['jsonPayload']['logging.googleapis.com/local_resource_id']"); + TEST_CHECK(ret == FLB_FALSE); + + /* check 'labels' field has been added to root-level of log entry */ + ret = mp_kv_exists(res_data, res_size, "$entries[0]['labels']"); + TEST_CHECK(ret == FLB_TRUE); + + /* check fields inside 'labels' field */ + ret = mp_kv_exists(res_data, res_size, "$entries[0]['labels']['testA']"); + TEST_CHECK(ret == FLB_TRUE); + + /* check field inside 'labels' field */ + ret = mp_kv_exists(res_data, res_size, "$entries[0]['labels']['testB']"); + TEST_CHECK(ret == FLB_TRUE); + + /* check `labels_key` has been removed from jsonPayload */ + ret = mp_kv_exists(res_data, res_size, "$entries[0]['jsonPayload']['logging.googleapis.com/customlabels']"); + TEST_CHECK(ret == FLB_FALSE); + + flb_sds_destroy(res_data); +} + +static void cb_check_multi_entries_severity(void *ctx, int ffd, + int res_ret, void *res_data, size_t res_size, + void *data) +{ + int ret; + + ret = mp_kv_cmp(res_data, res_size, "$entries[0]['severity']", "INFO"); + TEST_CHECK(ret == FLB_TRUE); + + // verifies that severity is removed from jsonPayload + ret = mp_kv_exists(res_data, res_size, "$entries[0]['jsonPayload']['severity']"); + TEST_CHECK(ret == FLB_FALSE); + + ret = mp_kv_exists(res_data, res_size, "$entries[1]['severity']"); + TEST_CHECK(ret == FLB_FALSE); + + ret = mp_kv_cmp(res_data, res_size, "$entries[2]['severity']", "DEBUG"); + TEST_CHECK(ret == FLB_TRUE); + + // verifies that severity is removed from jsonPayload + ret = mp_kv_exists(res_data, res_size, "$entries[2]['jsonPayload']['severity']"); + TEST_CHECK(ret == FLB_FALSE); + + ret = mp_kv_exists(res_data, res_size, "$entries[3]['severity']"); + TEST_CHECK(ret == FLB_FALSE); + + flb_sds_destroy(res_data); +} + +static void cb_check_source_location_common_case(void *ctx, int ffd, + int res_ret, void *res_data, size_t res_size, + void *data) +{ + int ret; + + /* sourceLocation_file */ + ret = mp_kv_cmp(res_data, res_size, "$entries[0]['sourceLocation']['file']", "test_file"); + TEST_CHECK(ret == FLB_TRUE); + + /* sourceLocation_line */ + ret = mp_kv_cmp_integer(res_data, res_size, "$entries[0]['sourceLocation']['line']", 123); + TEST_CHECK(ret == FLB_TRUE); + + /* sourceLocation_function */ + ret = mp_kv_cmp(res_data, res_size, "$entries[0]['sourceLocation']['function']", "test_function"); + TEST_CHECK(ret == FLB_TRUE); + + /* check `sourceLocation` has been removed from jsonPayload */ + ret = mp_kv_exists(res_data, res_size, "$entries[0]['jsonPayload']['logging.googleapis.com/sourceLocation']"); + TEST_CHECK(ret == FLB_FALSE); + + flb_sds_destroy(res_data); +} + +static void cb_check_source_location_common_case_line_in_string(void *ctx, int ffd, + int res_ret, void *res_data, size_t res_size, + void *data) +{ + int ret; + + /* sourceLocation_file */ + ret = mp_kv_cmp(res_data, res_size, "$entries[0]['sourceLocation']['file']", "test_file"); + TEST_CHECK(ret == FLB_TRUE); + + /* sourceLocation_line */ + ret = mp_kv_cmp_integer(res_data, res_size, "$entries[0]['sourceLocation']['line']", 123); + TEST_CHECK(ret == FLB_TRUE); + + /* sourceLocation_function */ + ret = mp_kv_cmp(res_data, res_size, "$entries[0]['sourceLocation']['function']", "test_function"); + TEST_CHECK(ret == FLB_TRUE); + + /* check `sourceLocation` has been removed from jsonPayload */ + ret = mp_kv_exists(res_data, res_size, "$entries[0]['jsonPayload']['logging.googleapis.com/sourceLocation']"); + TEST_CHECK(ret == FLB_FALSE); + + flb_sds_destroy(res_data); +} + +static void cb_check_empty_source_location(void *ctx, int ffd, + int res_ret, void *res_data, size_t res_size, + void *data) +{ + int ret; + + /* sourceLocation_file */ + ret = mp_kv_cmp(res_data, res_size, "$entries[0]['sourceLocation']['file']", ""); + TEST_CHECK(ret == FLB_TRUE); + + /* sourceLocation_line */ + ret = mp_kv_cmp_integer(res_data, res_size, "$entries[0]['sourceLocation']['line']", 0); + TEST_CHECK(ret == FLB_TRUE); + + /* sourceLocation_function */ + ret = mp_kv_cmp(res_data, res_size, "$entries[0]['sourceLocation']['function']", ""); + TEST_CHECK(ret == FLB_TRUE); + + /* check `sourceLocation` has been removed from jsonPayload */ + ret = mp_kv_exists(res_data, res_size, "$entries[0]['jsonPayload']['logging.googleapis.com/sourceLocation']"); + TEST_CHECK(ret == FLB_FALSE); + + flb_sds_destroy(res_data); +} + +static void cb_check_source_location_in_string(void *ctx, int ffd, + int res_ret, void *res_data, size_t res_size, + void *data) +{ + int ret; + + /* sourceLocation remains in jsonPayload */ + ret = mp_kv_cmp(res_data, res_size, "$entries[0]['jsonPayload']['logging.googleapis.com/sourceLocation']", "some string"); + TEST_CHECK(ret == FLB_TRUE); + + ret = mp_kv_exists(res_data, res_size, "$entries[0]['sourceLocation']"); + TEST_CHECK(ret == FLB_FALSE); + + flb_sds_destroy(res_data); +} + +static void cb_check_source_location_partial_subfields(void *ctx, int ffd, + int res_ret, void *res_data, size_t res_size, + void *data) +{ + int ret; + + /* sourceLocation_file */ + ret = mp_kv_cmp(res_data, res_size, "$entries[0]['sourceLocation']['file']", ""); + TEST_CHECK(ret == FLB_TRUE); + + /* sourceLocation_line */ + ret = mp_kv_cmp_integer(res_data, res_size, "$entries[0]['sourceLocation']['line']", 0); + TEST_CHECK(ret == FLB_TRUE); + + /* sourceLocation_function */ + ret = mp_kv_cmp(res_data, res_size, "$entries[0]['sourceLocation']['function']", "test_function"); + TEST_CHECK(ret == FLB_TRUE); + + + /* check `sourceLocation` has been removed from jsonPayload */ + ret = mp_kv_exists(res_data, res_size, "$entries[0]['jsonPayload']['logging.googleapis.com/sourceLocation']"); + TEST_CHECK(ret == FLB_FALSE); + + flb_sds_destroy(res_data); +} + +static void cb_check_source_location_incorrect_type_subfields(void *ctx, int ffd, + int res_ret, void *res_data, size_t res_size, + void *data) +{ + int ret; + + /* sourceLocation_file */ + ret = mp_kv_cmp(res_data, res_size, "$entries[0]['sourceLocation']['file']", ""); + TEST_CHECK(ret == FLB_TRUE); + + /* sourceLocation_line */ + ret = mp_kv_cmp_integer(res_data, res_size, "$entries[0]['sourceLocation']['line']", 0); + TEST_CHECK(ret == FLB_TRUE); + + /* sourceLocation_function */ + ret = mp_kv_cmp(res_data, res_size, "$entries[0]['sourceLocation']['function']", ""); + TEST_CHECK(ret == FLB_TRUE); + + + /* check `sourceLocation` has been removed from jsonPayload */ + ret = mp_kv_exists(res_data, res_size, "$entries[0]['jsonPayload']['logging.googleapis.com/sourceLocation']"); + TEST_CHECK(ret == FLB_FALSE); + + flb_sds_destroy(res_data); +} + +static void cb_check_source_location_extra_subfields(void *ctx, int ffd, + int res_ret, void *res_data, size_t res_size, + void *data) +{ + int ret; + + /* sourceLocation_file */ + ret = mp_kv_cmp(res_data, res_size, "$entries[0]['sourceLocation']['file']", "test_file"); + TEST_CHECK(ret == FLB_TRUE); + + /* sourceLocation_line */ + ret = mp_kv_cmp_integer(res_data, res_size, "$entries[0]['sourceLocation']['line']", 123); + TEST_CHECK(ret == FLB_TRUE); + + /* sourceLocation_function */ + ret = mp_kv_cmp(res_data, res_size, "$entries[0]['sourceLocation']['function']", "test_function"); + TEST_CHECK(ret == FLB_TRUE); + + /* Preserve extra subfields inside jsonPayload */ + ret = mp_kv_cmp(res_data, res_size, "$entries[0]['jsonPayload']['logging.googleapis.com/sourceLocation']['extra_key1']", "extra_val1"); + TEST_CHECK(ret == FLB_TRUE); + + ret = mp_kv_cmp_integer(res_data, res_size, "$entries[0]['jsonPayload']['logging.googleapis.com/sourceLocation']['extra_key2']", 123); + TEST_CHECK(ret == FLB_TRUE); + + ret = mp_kv_cmp_boolean(res_data, res_size, "$entries[0]['jsonPayload']['logging.googleapis.com/sourceLocation']['extra_key3']", true); + TEST_CHECK(ret == FLB_TRUE); + + flb_sds_destroy(res_data); +} + +static void cb_check_http_request_common_case(void *ctx, int ffd, + int res_ret, void *res_data, size_t res_size, + void *data) +{ + int ret; + + ret = mp_kv_cmp(res_data, res_size, "$entries[0]['httpRequest']['requestMethod']", "test_requestMethod"); + TEST_CHECK(ret == FLB_TRUE); + + ret = mp_kv_cmp(res_data, res_size, "$entries[0]['httpRequest']['requestUrl']", "test_requestUrl"); + TEST_CHECK(ret == FLB_TRUE); + + ret = mp_kv_cmp(res_data, res_size, "$entries[0]['httpRequest']['userAgent']", "test_userAgent"); + TEST_CHECK(ret == FLB_TRUE); + + ret = mp_kv_cmp(res_data, res_size, "$entries[0]['httpRequest']['remoteIp']", "test_remoteIp"); + TEST_CHECK(ret == FLB_TRUE); + + ret = mp_kv_cmp(res_data, res_size, "$entries[0]['httpRequest']['serverIp']", "test_serverIp"); + TEST_CHECK(ret == FLB_TRUE); + + ret = mp_kv_cmp(res_data, res_size, "$entries[0]['httpRequest']['referer']", "test_referer"); + TEST_CHECK(ret == FLB_TRUE); + + ret = mp_kv_cmp(res_data, res_size, "$entries[0]['httpRequest']['protocol']", "test_protocol"); + TEST_CHECK(ret == FLB_TRUE); + + ret = mp_kv_cmp(res_data, res_size, "$entries[0]['httpRequest']['latency']", "0s"); + TEST_CHECK(ret == FLB_TRUE); + + ret = mp_kv_cmp_integer(res_data, res_size, "$entries[0]['httpRequest']['requestSize']", 123); + TEST_CHECK(ret == FLB_TRUE); + + ret = mp_kv_cmp_integer(res_data, res_size, "$entries[0]['httpRequest']['responseSize']", 123); + TEST_CHECK(ret == FLB_TRUE); + + ret = mp_kv_cmp_integer(res_data, res_size, "$entries[0]['httpRequest']['status']", 200); + TEST_CHECK(ret == FLB_TRUE); + + ret = mp_kv_cmp_integer(res_data, res_size, "$entries[0]['httpRequest']['cacheFillBytes']", 123); + TEST_CHECK(ret == FLB_TRUE); + + ret = mp_kv_cmp_boolean(res_data, res_size, "$entries[0]['httpRequest']['cacheLookup']", true); + TEST_CHECK(ret == FLB_TRUE); + + ret = mp_kv_cmp_boolean(res_data, res_size, "$entries[0]['httpRequest']['cacheHit']", true); + TEST_CHECK(ret == FLB_TRUE); + + ret = mp_kv_cmp_boolean(res_data, res_size, "$entries[0]['httpRequest']['cacheValidatedWithOriginServer']", true); + TEST_CHECK(ret == FLB_TRUE); + + /* check `httpRequest` has been removed from jsonPayload */ + ret = mp_kv_exists(res_data, res_size, "$entries[0]['jsonPayload']['logging.googleapis.com/http_request']"); + TEST_CHECK(ret == FLB_FALSE); + + flb_sds_destroy(res_data); +} + +static void cb_check_empty_http_request(void *ctx, int ffd, + int res_ret, void *res_data, size_t res_size, + void *data) +{ + int ret; + + ret = mp_kv_cmp(res_data, res_size, "$entries[0]['httpRequest']['requestMethod']", ""); + TEST_CHECK(ret == FLB_TRUE); + + ret = mp_kv_cmp(res_data, res_size, "$entries[0]['httpRequest']['requestUrl']", ""); + TEST_CHECK(ret == FLB_TRUE); + + ret = mp_kv_cmp(res_data, res_size, "$entries[0]['httpRequest']['userAgent']", ""); + TEST_CHECK(ret == FLB_TRUE); + + ret = mp_kv_cmp(res_data, res_size, "$entries[0]['httpRequest']['remoteIp']", ""); + TEST_CHECK(ret == FLB_TRUE); + + ret = mp_kv_cmp(res_data, res_size, "$entries[0]['httpRequest']['serverIp']", ""); + TEST_CHECK(ret == FLB_TRUE); + + ret = mp_kv_cmp(res_data, res_size, "$entries[0]['httpRequest']['referer']", ""); + TEST_CHECK(ret == FLB_TRUE); + + ret = mp_kv_cmp(res_data, res_size, "$entries[0]['httpRequest']['protocol']", ""); + TEST_CHECK(ret == FLB_TRUE); + + ret = mp_kv_exists(res_data, res_size, "$entries[0]['httpRequest']['latency']"); + TEST_CHECK(ret == FLB_FALSE); + + ret = mp_kv_cmp_integer(res_data, res_size, "$entries[0]['httpRequest']['requestSize']", 0); + TEST_CHECK(ret == FLB_TRUE); + + ret = mp_kv_cmp_integer(res_data, res_size, "$entries[0]['httpRequest']['responseSize']", 0); + TEST_CHECK(ret == FLB_TRUE); + + ret = mp_kv_cmp_integer(res_data, res_size, "$entries[0]['httpRequest']['status']", 0); + TEST_CHECK(ret == FLB_TRUE); + + ret = mp_kv_cmp_integer(res_data, res_size, "$entries[0]['httpRequest']['cacheFillBytes']", 0); + TEST_CHECK(ret == FLB_TRUE); + + ret = mp_kv_cmp_boolean(res_data, res_size, "$entries[0]['httpRequest']['cacheLookup']", false); + TEST_CHECK(ret == FLB_TRUE); + + ret = mp_kv_cmp_boolean(res_data, res_size, "$entries[0]['httpRequest']['cacheHit']", false); + TEST_CHECK(ret == FLB_TRUE); + + ret = mp_kv_cmp_boolean(res_data, res_size, "$entries[0]['httpRequest']['cacheValidatedWithOriginServer']", false); + TEST_CHECK(ret == FLB_TRUE); + + /* check `httpRequest` has been removed from jsonPayload */ + ret = mp_kv_exists(res_data, res_size, "$entries[0]['jsonPayload']['logging.googleapis.com/http_request']"); + TEST_CHECK(ret == FLB_FALSE); + + flb_sds_destroy(res_data); +} + +static void cb_check_http_request_in_string(void *ctx, int ffd, + int res_ret, void *res_data, size_t res_size, + void *data) +{ + int ret; + + ret = mp_kv_cmp(res_data, res_size, "$entries[0]['jsonPayload']['logging.googleapis.com/http_request']", "some string"); + TEST_CHECK(ret == FLB_TRUE); + + ret = mp_kv_exists(res_data, res_size, "$entries[0]['httpRequest']"); + TEST_CHECK(ret == FLB_FALSE); + + flb_sds_destroy(res_data); +} + +static void cb_check_http_request_partial_subfields(void *ctx, int ffd, + int res_ret, void *res_data, size_t res_size, + void *data) +{ + int ret; + + ret = mp_kv_cmp(res_data, res_size, "$entries[0]['httpRequest']['requestMethod']", ""); + TEST_CHECK(ret == FLB_TRUE); + + ret = mp_kv_cmp(res_data, res_size, "$entries[0]['httpRequest']['requestUrl']", ""); + TEST_CHECK(ret == FLB_TRUE); + + ret = mp_kv_cmp(res_data, res_size, "$entries[0]['httpRequest']['userAgent']", ""); + TEST_CHECK(ret == FLB_TRUE); + + ret = mp_kv_cmp(res_data, res_size, "$entries[0]['httpRequest']['remoteIp']", ""); + TEST_CHECK(ret == FLB_TRUE); + + ret = mp_kv_cmp(res_data, res_size, "$entries[0]['httpRequest']['serverIp']", ""); + TEST_CHECK(ret == FLB_TRUE); + + ret = mp_kv_cmp(res_data, res_size, "$entries[0]['httpRequest']['referer']", ""); + TEST_CHECK(ret == FLB_TRUE); + + ret = mp_kv_cmp(res_data, res_size, "$entries[0]['httpRequest']['protocol']", ""); + TEST_CHECK(ret == FLB_TRUE); + + ret = mp_kv_exists(res_data, res_size, "$entries[0]['httpRequest']['latency']"); + TEST_CHECK(ret == FLB_FALSE); + + ret = mp_kv_cmp_integer(res_data, res_size, "$entries[0]['httpRequest']['requestSize']", 0); + TEST_CHECK(ret == FLB_TRUE); + + ret = mp_kv_cmp_integer(res_data, res_size, "$entries[0]['httpRequest']['responseSize']", 0); + TEST_CHECK(ret == FLB_TRUE); + + ret = mp_kv_cmp_integer(res_data, res_size, "$entries[0]['httpRequest']['status']", 0); + TEST_CHECK(ret == FLB_TRUE); + + ret = mp_kv_cmp_integer(res_data, res_size, "$entries[0]['httpRequest']['cacheFillBytes']", 0); + TEST_CHECK(ret == FLB_TRUE); + + ret = mp_kv_cmp_boolean(res_data, res_size, "$entries[0]['httpRequest']['cacheLookup']", true); + TEST_CHECK(ret == FLB_TRUE); + + ret = mp_kv_cmp_boolean(res_data, res_size, "$entries[0]['httpRequest']['cacheHit']", true); + TEST_CHECK(ret == FLB_TRUE); + + ret = mp_kv_cmp_boolean(res_data, res_size, "$entries[0]['httpRequest']['cacheValidatedWithOriginServer']", true); + TEST_CHECK(ret == FLB_TRUE); + + /* check `httpRequest` has been removed from jsonPayload */ + ret = mp_kv_exists(res_data, res_size, "$entries[0]['jsonPayload']['logging.googleapis.com/http_request']"); + TEST_CHECK(ret == FLB_FALSE); + + flb_sds_destroy(res_data); +} + +static void cb_check_http_request_incorrect_type_subfields(void *ctx, int ffd, + int res_ret, void *res_data, size_t res_size, + void *data) +{ + int ret; + + ret = mp_kv_cmp(res_data, res_size, "$entries[0]['httpRequest']['requestMethod']", ""); + TEST_CHECK(ret == FLB_TRUE); + + ret = mp_kv_cmp(res_data, res_size, "$entries[0]['httpRequest']['requestUrl']", ""); + TEST_CHECK(ret == FLB_TRUE); + + ret = mp_kv_cmp(res_data, res_size, "$entries[0]['httpRequest']['userAgent']", ""); + TEST_CHECK(ret == FLB_TRUE); + + ret = mp_kv_cmp(res_data, res_size, "$entries[0]['httpRequest']['remoteIp']", ""); + TEST_CHECK(ret == FLB_TRUE); + + ret = mp_kv_cmp(res_data, res_size, "$entries[0]['httpRequest']['serverIp']", ""); + TEST_CHECK(ret == FLB_TRUE); + + ret = mp_kv_cmp(res_data, res_size, "$entries[0]['httpRequest']['referer']", ""); + TEST_CHECK(ret == FLB_TRUE); + + ret = mp_kv_cmp(res_data, res_size, "$entries[0]['httpRequest']['protocol']", ""); + TEST_CHECK(ret == FLB_TRUE); + + ret = mp_kv_exists(res_data, res_size, "$entries[0]['httpRequest']['latency']"); + TEST_CHECK(ret == FLB_FALSE); + + ret = mp_kv_cmp_integer(res_data, res_size, "$entries[0]['httpRequest']['requestSize']", 0); + TEST_CHECK(ret == FLB_TRUE); + + ret = mp_kv_cmp_integer(res_data, res_size, "$entries[0]['httpRequest']['responseSize']", 0); + TEST_CHECK(ret == FLB_TRUE); + + ret = mp_kv_cmp_integer(res_data, res_size, "$entries[0]['httpRequest']['status']", 0); + TEST_CHECK(ret == FLB_TRUE); + + ret = mp_kv_cmp_integer(res_data, res_size, "$entries[0]['httpRequest']['cacheFillBytes']", 0); + TEST_CHECK(ret == FLB_TRUE); + + ret = mp_kv_cmp_boolean(res_data, res_size, "$entries[0]['httpRequest']['cacheLookup']", false); + TEST_CHECK(ret == FLB_TRUE); + + ret = mp_kv_cmp_boolean(res_data, res_size, "$entries[0]['httpRequest']['cacheHit']", false); + TEST_CHECK(ret == FLB_TRUE); + + ret = mp_kv_cmp_boolean(res_data, res_size, "$entries[0]['httpRequest']['cacheValidatedWithOriginServer']", false); + TEST_CHECK(ret == FLB_TRUE); + + /* check `httpRequest` has been removed from jsonPayload */ + ret = mp_kv_exists(res_data, res_size, "$entries[0]['jsonPayload']['logging.googleapis.com/http_request']"); + TEST_CHECK(ret == FLB_FALSE); + + flb_sds_destroy(res_data); +} + +static void cb_check_http_request_extra_subfields(void *ctx, int ffd, + int res_ret, void *res_data, size_t res_size, + void *data) +{ + int ret; + + ret = mp_kv_cmp(res_data, res_size, "$entries[0]['httpRequest']['requestMethod']", "test_requestMethod"); + TEST_CHECK(ret == FLB_TRUE); + + ret = mp_kv_cmp(res_data, res_size, "$entries[0]['httpRequest']['requestUrl']", "test_requestUrl"); + TEST_CHECK(ret == FLB_TRUE); + + ret = mp_kv_cmp(res_data, res_size, "$entries[0]['httpRequest']['userAgent']", "test_userAgent"); + TEST_CHECK(ret == FLB_TRUE); + + ret = mp_kv_cmp(res_data, res_size, "$entries[0]['httpRequest']['remoteIp']", "test_remoteIp"); + TEST_CHECK(ret == FLB_TRUE); + + ret = mp_kv_cmp(res_data, res_size, "$entries[0]['httpRequest']['serverIp']", "test_serverIp"); + TEST_CHECK(ret == FLB_TRUE); + + ret = mp_kv_cmp(res_data, res_size, "$entries[0]['httpRequest']['referer']", "test_referer"); + TEST_CHECK(ret == FLB_TRUE); + + ret = mp_kv_cmp(res_data, res_size, "$entries[0]['httpRequest']['protocol']", "test_protocol"); + TEST_CHECK(ret == FLB_TRUE); + + ret = mp_kv_cmp(res_data, res_size, "$entries[0]['httpRequest']['latency']", "0s"); + TEST_CHECK(ret == FLB_TRUE); + + ret = mp_kv_cmp_integer(res_data, res_size, "$entries[0]['httpRequest']['requestSize']", 123); + TEST_CHECK(ret == FLB_TRUE); + + ret = mp_kv_cmp_integer(res_data, res_size, "$entries[0]['httpRequest']['responseSize']", 123); + TEST_CHECK(ret == FLB_TRUE); + + ret = mp_kv_cmp_integer(res_data, res_size, "$entries[0]['httpRequest']['status']", 200); + TEST_CHECK(ret == FLB_TRUE); + + ret = mp_kv_cmp_integer(res_data, res_size, "$entries[0]['httpRequest']['cacheFillBytes']", 123); + TEST_CHECK(ret == FLB_TRUE); + + ret = mp_kv_cmp_boolean(res_data, res_size, "$entries[0]['httpRequest']['cacheLookup']", true); + TEST_CHECK(ret == FLB_TRUE); + + ret = mp_kv_cmp_boolean(res_data, res_size, "$entries[0]['httpRequest']['cacheHit']", true); + TEST_CHECK(ret == FLB_TRUE); + + ret = mp_kv_cmp_boolean(res_data, res_size, "$entries[0]['httpRequest']['cacheValidatedWithOriginServer']", true); + TEST_CHECK(ret == FLB_TRUE); + + /* Preserve extra subfields inside jsonPayload */ + ret = mp_kv_cmp(res_data, res_size, "$entries[0]['jsonPayload']['logging.googleapis.com/http_request']['extra_key1']", "extra_val1"); + TEST_CHECK(ret == FLB_TRUE); + + ret = mp_kv_cmp_integer(res_data, res_size, "$entries[0]['jsonPayload']['logging.googleapis.com/http_request']['extra_key2']", 123); + TEST_CHECK(ret == FLB_TRUE); + + ret = mp_kv_cmp_boolean(res_data, res_size, "$entries[0]['jsonPayload']['logging.googleapis.com/http_request']['extra_key3']", true); + TEST_CHECK(ret == FLB_TRUE); + + flb_sds_destroy(res_data); +} + +static void cb_check_http_request_lantency_common_case(void *ctx, int ffd, + int res_ret, void *res_data, size_t res_size, + void *data) +{ + int ret; + + ret = mp_kv_cmp(res_data, res_size, "$entries[0]['httpRequest']['latency']", "100.00s"); + TEST_CHECK(ret == FLB_TRUE); + + /* check `httpRequest` has been removed from jsonPayload */ + ret = mp_kv_exists(res_data, res_size, "$entries[0]['jsonPayload']['logging.googleapis.com/http_request']"); + TEST_CHECK(ret == FLB_FALSE); + + flb_sds_destroy(res_data); +} + +static void cb_check_http_request_latency_incorrect_format(void *ctx, int ffd, + int res_ret, void *res_data, size_t res_size, + void *data) +{ + int ret; + + ret = mp_kv_exists(res_data, res_size, "$entries[0]['httpRequest']['latency']"); + TEST_CHECK(ret == FLB_FALSE); + + /* check `httpRequest` has been removed from jsonPayload */ + ret = mp_kv_exists(res_data, res_size, "$entries[0]['jsonPayload']['logging.googleapis.com/http_request']"); + TEST_CHECK(ret == FLB_FALSE); + + flb_sds_destroy(res_data); +} + +static void cb_check_timestamp_format_object_common_case(void *ctx, int ffd, + int res_ret, void *res_data, size_t res_size, + void *data) +{ + int ret; + + ret = mp_kv_cmp(res_data, res_size, "$entries[0]['timestamp']", "2020-07-21T16:40:42.000012345Z"); + TEST_CHECK(ret == FLB_TRUE); + + /* check `timestamp` has been removed from jsonPayload */ + ret = mp_kv_exists(res_data, res_size, "$entries[0]['jsonPayload']['timestamp']"); + TEST_CHECK(ret == FLB_FALSE); + + flb_sds_destroy(res_data); +} + +static void cb_check_timestamp_format_object_not_a_map(void *ctx, int ffd, + int res_ret, void *res_data, size_t res_size, + void *data) +{ + int ret; + + ret = mp_kv_cmp(res_data, res_size, "$entries[0]['timestamp']", "2020-07-21T16:40:00.000000000Z"); + TEST_CHECK(ret == FLB_TRUE); + + ret = mp_kv_cmp(res_data, res_size, "$entries[0]['jsonPayload']['timestamp']", "string"); + TEST_CHECK(ret == FLB_TRUE); + + flb_sds_destroy(res_data); +} + +static void cb_check_timestamp_format_object_missing_subfield(void *ctx, int ffd, + int res_ret, void *res_data, + size_t res_size, + void *data) +{ + int ret; + + ret = mp_kv_cmp(res_data, res_size, "$entries[0]['timestamp']", "2020-07-21T16:40:00.000000000Z"); + TEST_CHECK(ret == FLB_TRUE); + + ret = mp_kv_cmp(res_data, res_size, "$entries[0]['jsonPayload']['timestamp']['nanos']", "12345"); + TEST_CHECK(ret == FLB_TRUE); + + flb_sds_destroy(res_data); +} + +static void cb_check_timestamp_format_object_incorrect_subfields(void *ctx, int ffd, + int res_ret, void *res_data, + size_t res_size, + void *data) +{ + int ret; + + ret = mp_kv_cmp(res_data, res_size, "$entries[0]['timestamp']", "2020-07-21T16:40:00.000000000Z"); + TEST_CHECK(ret == FLB_TRUE); + + /* check `timestamp` has been removed from jsonPayload */ + ret = mp_kv_exists(res_data, res_size, "$entries[0]['jsonPayload']['timestamp']"); + TEST_CHECK(ret == FLB_FALSE); + + flb_sds_destroy(res_data); +} + +static void cb_check_timestamp_format_duo_fields_common_case(void *ctx, int ffd, + int res_ret, void *res_data, size_t res_size, + void *data) +{ + int ret; + + ret = mp_kv_cmp(res_data, res_size, "$entries[0]['timestamp']", "2020-07-21T16:40:42.000012345Z"); + TEST_CHECK(ret == FLB_TRUE); + + /* check `timestampSeconds` has been removed from jsonPayload */ + ret = mp_kv_exists(res_data, res_size, "$entries[0]['jsonPayload']['timestampSeconds']"); + TEST_CHECK(ret == FLB_FALSE); + + /* check `timestampNanos` has been removed from jsonPayload */ + ret = mp_kv_exists(res_data, res_size, "$entries[0]['jsonPayload']['timestampNanos']"); + TEST_CHECK(ret == FLB_FALSE); + + flb_sds_destroy(res_data); +} + +static void cb_check_timestamp_format_duo_fields_missing_nanos(void *ctx, int ffd, + int res_ret, void *res_data, size_t res_size, + void *data) +{ + int ret; + + ret = mp_kv_cmp(res_data, res_size, "$entries[0]['timestamp']", "2020-07-21T16:40:00.000000000Z"); + TEST_CHECK(ret == FLB_TRUE); + + ret = mp_kv_cmp(res_data, res_size, "$entries[0]['jsonPayload']['timestampSeconds']", "1595349642"); + TEST_CHECK(ret == FLB_TRUE); + + flb_sds_destroy(res_data); +} + +static void cb_check_timestamp_format_duo_fields_incorrect_type(void *ctx, int ffd, + int res_ret, void *res_data, size_t res_size, + void *data) +{ + int ret; + + ret = mp_kv_cmp(res_data, res_size, "$entries[0]['timestamp']", "2020-07-21T16:40:00.000000000Z"); + TEST_CHECK(ret == FLB_TRUE); + + /* check `timestampSeconds` has been removed from jsonPayload */ + ret = mp_kv_exists(res_data, res_size, "$entries[0]['jsonPayload']['timestampSeconds']"); + TEST_CHECK(ret == FLB_FALSE); + + /* check `timestampNanos` has been removed from jsonPayload */ + ret = mp_kv_exists(res_data, res_size, "$entries[0]['jsonPayload']['timestampNanos']"); + TEST_CHECK(ret == FLB_FALSE); + + flb_sds_destroy(res_data); +} + +void flb_test_monitored_resource_common() +{ + int ret; + int size = sizeof(MONITORED_RESOURCE_COMMON_CASE) - 1; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + + /* Lib input mode */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + /* Stackdriver output */ + out_ffd = flb_output(ctx, (char *) "stackdriver", NULL); + flb_output_set(ctx, out_ffd, + "match", "test", + "google_service_credentials", SERVICE_CREDENTIALS, + "resource", "monitored_resource", + NULL); + + /* Enable test mode */ + ret = flb_output_set_test(ctx, out_ffd, "formatter", + cb_check_monitored_resource, + NULL, NULL); + + /* Start */ + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + flb_lib_push(ctx, in_ffd, (char *) MONITORED_RESOURCE_COMMON_CASE, size); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_monitored_resource_priority_higher_than_local_resource_id() +{ + int ret; + int size = sizeof(MONITORED_RESOURCE_PRIORITY_HIGHER_THAN_LOCAL_RESOURCE_ID) - 1; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + + /* Lib input mode */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + /* Stackdriver output */ + out_ffd = flb_output(ctx, (char *) "stackdriver", NULL); + flb_output_set(ctx, out_ffd, + "match", "test", + "google_service_credentials", SERVICE_CREDENTIALS, + "resource", "k8s_container", + "k8s_cluster_name", "test_cluster_name", + "k8s_cluster_location", "test_cluster_location", + NULL); + + /* Enable test mode */ + ret = flb_output_set_test(ctx, out_ffd, "formatter", + cb_check_monitored_resource_priority_higher_than_local_resource_id, + NULL, NULL); + + /* Start */ + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + flb_lib_push(ctx, in_ffd, (char *) MONITORED_RESOURCE_PRIORITY_HIGHER_THAN_LOCAL_RESOURCE_ID, size); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_monitored_resource_priority_higher_than_gce_instance() +{ + int ret; + int size = sizeof(MONITORED_RESOURCE_PRIORITY_HIGHER_THAN_GCE_INSTANCE) - 1; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + + /* Lib input mode */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + /* Stackdriver output */ + out_ffd = flb_output(ctx, (char *) "stackdriver", NULL); + flb_output_set(ctx, out_ffd, + "match", "test", + "google_service_credentials", SERVICE_CREDENTIALS, + "resource", "gce_instance", + NULL); + + /* Enable test mode */ + ret = flb_output_set_test(ctx, out_ffd, "formatter", + cb_check_monitored_resource_priority_higher_than_gce_instance, + NULL, NULL); + + /* Start */ + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + flb_lib_push(ctx, in_ffd, (char *) MONITORED_RESOURCE_PRIORITY_HIGHER_THAN_GCE_INSTANCE, size); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_resource_global() +{ + int ret; + int size = sizeof(JSON) - 1; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + + /* Lib input mode */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + /* Stackdriver output */ + out_ffd = flb_output(ctx, (char *) "stackdriver", NULL); + flb_output_set(ctx, out_ffd, + "match", "test", + "google_service_credentials", SERVICE_CREDENTIALS, + "resource", "global", + NULL); + + /* Enable test mode */ + ret = flb_output_set_test(ctx, out_ffd, "formatter", + cb_check_global_resource, + NULL, NULL); + + /* Start */ + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + flb_lib_push(ctx, in_ffd, (char *) JSON, size); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_trace_no_autoformat() +{ + int ret; + int size = sizeof(TRACE_COMMON_CASE) - 1; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + + /* Lib input mode */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + /* Stackdriver output */ + out_ffd = flb_output(ctx, (char *) "stackdriver", NULL); + flb_output_set(ctx, out_ffd, + "match", "test", + "resource", "gce_instance", + "trace_key", "trace", + NULL); + + /* Enable test mode */ + ret = flb_output_set_test(ctx, out_ffd, "formatter", + cb_check_trace_no_autoformat, + NULL, NULL); + + /* Start */ + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + flb_lib_push(ctx, in_ffd, (char *) TRACE_COMMON_CASE, size); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_trace_stackdriver_autoformat() +{ + int ret; + int size = sizeof(TRACE_COMMON_CASE) - 1; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + + /* Lib input mode */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + /* Stackdriver output */ + out_ffd = flb_output(ctx, (char *) "stackdriver", NULL); + flb_output_set(ctx, out_ffd, + "match", "test", + "resource", "gce_instance", + "trace_key", "trace", + "autoformat_stackdriver_trace", "true", + NULL); + + /* Enable test mode */ + ret = flb_output_set_test(ctx, out_ffd, "formatter", + cb_check_trace_stackdriver_autoformat, + NULL, NULL); + + /* Start */ + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + flb_lib_push(ctx, in_ffd, (char *) TRACE_COMMON_CASE, size); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_span_id() +{ + int ret; + int size = sizeof(SPAN_ID_COMMON_CASE) - 1; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + + /* Lib input mode */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + /* Stackdriver output */ + out_ffd = flb_output(ctx, (char *) "stackdriver", NULL); + flb_output_set(ctx, out_ffd, + "match", "test", + "resource", "gce_instance", + NULL); + + /* Enable test mode */ + ret = flb_output_set_test(ctx, out_ffd, "formatter", + cb_check_span_id, + NULL, NULL); + + /* Start */ + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + flb_lib_push(ctx, in_ffd, (char *) SPAN_ID_COMMON_CASE, size); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_trace_sampled_true() +{ + int ret; + int size = sizeof(TRACE_SAMPLED_CASE_TRUE) - 1; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + + /* Lib input mode */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + /* Stackdriver output */ + out_ffd = flb_output(ctx, (char *) "stackdriver", NULL); + flb_output_set(ctx, out_ffd, + "match", "test", + "resource", "gce_instance", + NULL); + + /* Enable test mode */ + ret = flb_output_set_test(ctx, out_ffd, "formatter", + cb_check_trace_sampled_true, + NULL, NULL); + + /* Start */ + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + flb_lib_push(ctx, in_ffd, (char *) TRACE_SAMPLED_CASE_TRUE, size); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_trace_sampled_false() +{ + int ret; + int size = sizeof(TRACE_SAMPLED_CASE_FALSE) - 1; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + + /* Lib input mode */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + /* Stackdriver output */ + out_ffd = flb_output(ctx, (char *) "stackdriver", NULL); + flb_output_set(ctx, out_ffd, + "match", "test", + "resource", "gce_instance", + NULL); + + /* Enable test mode */ + ret = flb_output_set_test(ctx, out_ffd, "formatter", + cb_check_trace_sampled_false, + NULL, NULL); + + /* Start */ + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + flb_lib_push(ctx, in_ffd, (char *) TRACE_SAMPLED_CASE_FALSE, size); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_set_metadata_server() +{ + int ret; + int size = sizeof(JSON) - 1; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + + /* Lib input mode */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + /* Stackdriver output */ + out_ffd = flb_output(ctx, (char *) "stackdriver", NULL); + flb_output_set(ctx, out_ffd, + "match", "test", + "resource", "gce_instance", + "metadata_server", "http://metadata.google.internal", + NULL); + + /* Enable test mode */ + ret = flb_output_set_test(ctx, out_ffd, "formatter", + cb_check_gce_instance, + NULL, NULL); + + /* Start */ + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + flb_lib_push(ctx, in_ffd, (char *) JSON, size); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_log_name_override() +{ + int ret; + int size = sizeof(LOG_NAME_OVERRIDE) - 1; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + + /* Lib input mode */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + /* Stackdriver output */ + out_ffd = flb_output(ctx, (char *) "stackdriver", NULL); + flb_output_set(ctx, out_ffd, + "match", "test", + "resource", "gce_instance", + "log_name_key", "custom_log_name_key", + NULL); + + /* Enable test mode */ + ret = flb_output_set_test(ctx, out_ffd, "formatter", + cb_check_log_name_override, + NULL, NULL); + + /* Start */ + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + flb_lib_push(ctx, in_ffd, (char *) LOG_NAME_OVERRIDE, size); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_log_name_no_override() +{ + int ret; + int size = sizeof(LOG_NAME_NO_OVERRIDE) - 1; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + + /* Lib input mode */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + /* Stackdriver output */ + out_ffd = flb_output(ctx, (char *) "stackdriver", NULL); + flb_output_set(ctx, out_ffd, + "match", "test", + "resource", "gce_instance", + "log_name_key", "custom_log_name_key", + NULL); + + /* Enable test mode */ + ret = flb_output_set_test(ctx, out_ffd, "formatter", + cb_check_log_name_no_override, + NULL, NULL); + + /* Start */ + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + flb_lib_push(ctx, in_ffd, (char *) LOG_NAME_NO_OVERRIDE, size); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_resource_global_custom_prefix() +{ + /* configuring tag_prefix for non-k8s resource type should have no effect at all */ + int ret; + int size = sizeof(JSON) - 1; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + + /* Lib input mode */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + /* Stackdriver output */ + out_ffd = flb_output(ctx, (char *) "stackdriver", NULL); + flb_output_set(ctx, out_ffd, + "match", "test", + "google_service_credentials", SERVICE_CREDENTIALS, + "resource", "global", + "tag_prefix", "custom_tag.", + NULL); + + /* Enable test mode */ + ret = flb_output_set_test(ctx, out_ffd, "formatter", + cb_check_global_resource, + NULL, NULL); + + /* Start */ + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + flb_lib_push(ctx, in_ffd, (char *) JSON, size); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_resource_generic_node_creds() +{ + int ret; + int size = sizeof(JSON) - 1; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + + /* Lib input mode */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + /* Stackdriver output */ + out_ffd = flb_output(ctx, (char *) "stackdriver", NULL); + flb_output_set(ctx, out_ffd, + "match", "test", + "google_service_credentials", SERVICE_CREDENTIALS, + "resource", "generic_node", + "location", "fluent", + "namespace", "test", + "node_id", "333222111", + NULL); + + /* Enable test mode */ + ret = flb_output_set_test(ctx, out_ffd, "formatter", + cb_check_generic_node_creds, + NULL, NULL); + + /* Start */ + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + flb_lib_push(ctx, in_ffd, (char *) JSON, size); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_resource_generic_node_metadata() +{ + int ret; + int size = sizeof(JSON) - 1; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + + /* Lib input mode */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + /* Stackdriver output */ + out_ffd = flb_output(ctx, (char *) "stackdriver", NULL); + flb_output_set(ctx, out_ffd, + "match", "test", + "resource", "generic_node", + "location", "fluent", + "namespace", "test", + "node_id", "333222111", + NULL); + + /* Enable test mode */ + ret = flb_output_set_test(ctx, out_ffd, "formatter", + cb_check_generic_node_metadata, + NULL, NULL); + + /* Start */ + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + flb_lib_push(ctx, in_ffd, (char *) JSON, size); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_resource_generic_task_creds() +{ + int ret; + int size = sizeof(JSON) - 1; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + + /* Lib input mode */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + /* Stackdriver output */ + out_ffd = flb_output(ctx, (char *) "stackdriver", NULL); + flb_output_set(ctx, out_ffd, + "match", "test", + "google_service_credentials", SERVICE_CREDENTIALS, + "resource", "generic_task", + "location", "fluent", + "namespace", "test", + "job", "test-job", + "task_id", "333222111", + NULL); + + /* Enable test mode */ + ret = flb_output_set_test(ctx, out_ffd, "formatter", + cb_check_generic_task_creds, + NULL, NULL); + + /* Start */ + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + flb_lib_push(ctx, in_ffd, (char *) JSON, size); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_resource_generic_task_metadata() +{ + int ret; + int size = sizeof(JSON) - 1; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + + /* Lib input mode */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + /* Stackdriver output */ + out_ffd = flb_output(ctx, (char *) "stackdriver", NULL); + flb_output_set(ctx, out_ffd, + "match", "test", + "resource", "generic_task", + "location", "fluent", + "namespace", "test", + "job", "test-job", + "task_id", "333222111", + NULL); + + /* Enable test mode */ + ret = flb_output_set_test(ctx, out_ffd, "formatter", + cb_check_generic_task_metadata, + NULL, NULL); + + /* Start */ + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + flb_lib_push(ctx, in_ffd, (char *) JSON, size); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_resource_gce_instance() +{ + int ret; + int size = sizeof(JSON) - 1; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + + /* Lib input mode */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + /* Stackdriver output */ + out_ffd = flb_output(ctx, (char *) "stackdriver", NULL); + flb_output_set(ctx, out_ffd, + "match", "test", + "resource", "gce_instance", + NULL); + + /* Enable test mode */ + ret = flb_output_set_test(ctx, out_ffd, "formatter", + cb_check_gce_instance, + NULL, NULL); + + /* Start */ + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + flb_lib_push(ctx, in_ffd, (char *) JSON, size); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_insert_id_common_case() +{ + int ret; + int size = sizeof(INSERTID_COMMON_CASE) - 1; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + + /* Lib input mode */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + /* Stackdriver output */ + out_ffd = flb_output(ctx, (char *) "stackdriver", NULL); + flb_output_set(ctx, out_ffd, + "match", "test", + "resource", "gce_instance", + NULL); + + /* Enable test mode */ + ret = flb_output_set_test(ctx, out_ffd, "formatter", + cb_check_insert_id_common_case, + NULL, NULL); + + /* Start */ + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + flb_lib_push(ctx, in_ffd, (char *) INSERTID_COMMON_CASE, size); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_empty_insert_id() +{ + int ret; + int size = sizeof(EMPTY_INSERTID) - 1; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + + /* Lib input mode */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + /* Stackdriver output */ + out_ffd = flb_output(ctx, (char *) "stackdriver", NULL); + flb_output_set(ctx, out_ffd, + "match", "test", + "resource", "gce_instance", + NULL); + + /* Enable test mode */ + ret = flb_output_set_test(ctx, out_ffd, "formatter", + cb_check_empty_insert_id, + NULL, NULL); + + /* Start */ + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + flb_lib_push(ctx, in_ffd, (char *) EMPTY_INSERTID, size); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_insert_id_incorrect_type() +{ + int ret; + int size = sizeof(INSERTID_INCORRECT_TYPE_INT) - 1; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + + /* Lib input mode */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + /* Stackdriver output */ + out_ffd = flb_output(ctx, (char *) "stackdriver", NULL); + flb_output_set(ctx, out_ffd, + "match", "test", + "resource", "gce_instance", + NULL); + + /* Enable test mode */ + ret = flb_output_set_test(ctx, out_ffd, "formatter", + cb_check_insert_id_incorrect_type, + NULL, NULL); + + /* Start */ + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + flb_lib_push(ctx, in_ffd, (char *) INSERTID_INCORRECT_TYPE_INT, size); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_operation_common() +{ + int ret; + int size = sizeof(OPERATION_COMMON_CASE) - 1; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + + /* Lib input mode */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + /* Stackdriver output */ + out_ffd = flb_output(ctx, (char *) "stackdriver", NULL); + flb_output_set(ctx, out_ffd, + "match", "test", + "resource", "gce_instance", + NULL); + + /* Enable test mode */ + ret = flb_output_set_test(ctx, out_ffd, "formatter", + cb_check_operation_common_case, + NULL, NULL); + + /* Start */ + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + flb_lib_push(ctx, in_ffd, (char *) OPERATION_COMMON_CASE, size); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_empty_operation() +{ + int ret; + int size = sizeof(EMPTY_OPERATION) - 1; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + + /* Lib input mode */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + /* Stackdriver output */ + out_ffd = flb_output(ctx, (char *) "stackdriver", NULL); + flb_output_set(ctx, out_ffd, + "match", "test", + "resource", "gce_instance", + NULL); + + /* Enable test mode */ + ret = flb_output_set_test(ctx, out_ffd, "formatter", + cb_check_empty_operation, + NULL, NULL); + + /* Start */ + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + flb_lib_push(ctx, in_ffd, (char *) EMPTY_OPERATION, size); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_operation_in_string() +{ + int ret; + int size = sizeof(OPERATION_IN_STRING) - 1; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + + /* Lib input mode */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + /* Stackdriver output */ + out_ffd = flb_output(ctx, (char *) "stackdriver", NULL); + flb_output_set(ctx, out_ffd, + "match", "test", + "resource", "gce_instance", + NULL); + + /* Enable test mode */ + ret = flb_output_set_test(ctx, out_ffd, "formatter", + cb_check_operation_in_string, + NULL, NULL); + + /* Start */ + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + flb_lib_push(ctx, in_ffd, (char *) OPERATION_IN_STRING, size); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_operation_partial_subfields() +{ + int ret; + int size = sizeof(PARTIAL_SUBFIELDS) - 1; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + + /* Lib input mode */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + /* Stackdriver output */ + out_ffd = flb_output(ctx, (char *) "stackdriver", NULL); + flb_output_set(ctx, out_ffd, + "match", "test", + "resource", "gce_instance", + NULL); + + /* Enable test mode */ + ret = flb_output_set_test(ctx, out_ffd, "formatter", + cb_check_operation_partial_subfields, + NULL, NULL); + + /* Start */ + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + flb_lib_push(ctx, in_ffd, (char *) PARTIAL_SUBFIELDS, size); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_operation_incorrect_type_subfields() +{ + int ret; + int size = sizeof(SUBFIELDS_IN_INCORRECT_TYPE) - 1; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + + /* Lib input mode */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + /* Stackdriver output */ + out_ffd = flb_output(ctx, (char *) "stackdriver", NULL); + flb_output_set(ctx, out_ffd, + "match", "test", + "resource", "gce_instance", + NULL); + + /* Enable test mode */ + ret = flb_output_set_test(ctx, out_ffd, "formatter", + cb_check_operation_incorrect_type_subfields, + NULL, NULL); + + /* Start */ + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + flb_lib_push(ctx, in_ffd, (char *) SUBFIELDS_IN_INCORRECT_TYPE, size); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_operation_extra_subfields() +{ + int ret; + int size = sizeof(EXTRA_SUBFIELDS_EXISTED) - 1; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + + /* Lib input mode */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + /* Stackdriver output */ + out_ffd = flb_output(ctx, (char *) "stackdriver", NULL); + flb_output_set(ctx, out_ffd, + "match", "test", + "resource", "gce_instance", + NULL); + + /* Enable test mode */ + ret = flb_output_set_test(ctx, out_ffd, "formatter", + cb_check_operation_extra_subfields, + NULL, NULL); + + /* Start */ + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + flb_lib_push(ctx, in_ffd, (char *) EXTRA_SUBFIELDS_EXISTED, size); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_resource_k8s_container_common() +{ + int ret; + int size = sizeof(K8S_CONTAINER_COMMON) - 1; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + + /* Lib input mode */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + /* Stackdriver output */ + out_ffd = flb_output(ctx, (char *) "stackdriver", NULL); + flb_output_set(ctx, out_ffd, + "match", "test", + "resource", "k8s_container", + "google_service_credentials", SERVICE_CREDENTIALS, + "k8s_cluster_name", "test_cluster_name", + "k8s_cluster_location", "test_cluster_location", + NULL); + + /* Enable test mode */ + ret = flb_output_set_test(ctx, out_ffd, "formatter", + cb_check_k8s_container_resource, + NULL, NULL); + + /* Start */ + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + flb_lib_push(ctx, in_ffd, (char *) K8S_CONTAINER_COMMON, size); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_resource_k8s_container_multi_tag_value() +{ + int ret; + int size_one = sizeof(K8S_CONTAINER_COMMON) - 1; + int size_two = sizeof(K8S_CONTAINER_COMMON_DIFF_TAGS) - 1; + + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + + /* Lib input mode */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + /* Stackdriver output */ + out_ffd = flb_output(ctx, (char *) "stackdriver", NULL); + flb_output_set(ctx, out_ffd, + "match", "test", + "resource", "k8s_container", + "google_service_credentials", SERVICE_CREDENTIALS, + "k8s_cluster_name", "test_cluster_name", + "k8s_cluster_location", "test_cluster_location", + NULL); + + /* Enable test mode */ + ret = flb_output_set_test(ctx, out_ffd, "formatter", + cb_check_k8s_container_resource, + NULL, NULL); + + /* Start */ + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + flb_lib_push(ctx, in_ffd, (char *) K8S_CONTAINER_COMMON_DIFF_TAGS, size_one); + + /* Enable test mode */ + ret = flb_output_set_test(ctx, out_ffd, "formatter", + cb_check_k8s_container_resource_diff_tag, + NULL, NULL); + + /* Ingest data sample */ + flb_lib_push(ctx, in_ffd, (char *) K8S_CONTAINER_COMMON_DIFF_TAGS, size_two); + + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_resource_k8s_container_custom_tag_prefix() +{ + int ret; + int size = sizeof(K8S_CONTAINER_NO_LOCAL_RESOURCE_ID) - 1; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + + /* Lib input mode */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + flb_input_set(ctx, in_ffd, "tag", "kube_custom_tag.testnamespace.testpod.testctr", NULL); + + /* Stackdriver output */ + out_ffd = flb_output(ctx, (char *) "stackdriver", NULL); + flb_output_set(ctx, out_ffd, + "match", "kube_custom_tag.*", + "resource", "k8s_container", + "google_service_credentials", SERVICE_CREDENTIALS, + "k8s_cluster_name", "test_cluster_name", + "k8s_cluster_location", "test_cluster_location", + "tag_prefix", "kube_custom_tag.", + NULL); + + /* Enable test mode */ + ret = flb_output_set_test(ctx, out_ffd, "formatter", + cb_check_k8s_container_resource, + NULL, NULL); + + /* Start */ + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + flb_lib_push(ctx, in_ffd, (char *) K8S_CONTAINER_NO_LOCAL_RESOURCE_ID, size); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_resource_k8s_container_custom_tag_prefix_with_dot() +{ + int ret; + int size = sizeof(K8S_CONTAINER_NO_LOCAL_RESOURCE_ID) - 1; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + + /* Lib input mode */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + flb_input_set(ctx, in_ffd, "tag", "kube.custom.tag.testnamespace.testpod.testctr", NULL); + + /* Stackdriver output */ + out_ffd = flb_output(ctx, (char *) "stackdriver", NULL); + flb_output_set(ctx, out_ffd, + "match", "kube.custom.tag.*", + "resource", "k8s_container", + "google_service_credentials", SERVICE_CREDENTIALS, + "k8s_cluster_name", "test_cluster_name", + "k8s_cluster_location", "test_cluster_location", + "tag_prefix", "kube.custom.tag.", + NULL); + + /* Enable test mode */ + ret = flb_output_set_test(ctx, out_ffd, "formatter", + cb_check_k8s_container_resource, + NULL, NULL); + + /* Start */ + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + flb_lib_push(ctx, in_ffd, (char *) K8S_CONTAINER_NO_LOCAL_RESOURCE_ID, size); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_resource_k8s_container_default_tag_regex() +{ + int ret; + int size = sizeof(K8S_CONTAINER_NO_LOCAL_RESOURCE_ID) - 1; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + + /* Lib input mode */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + flb_input_set(ctx, in_ffd, "tag", + "kube.var.log.containers.apache-logs-annotated_default_apache-aeeccc7a9f00f6e4e066aeff0434cf80621215071f1b20a51e8340aa7c35eac6.log", NULL); + + /* Stackdriver output */ + out_ffd = flb_output(ctx, (char *) "stackdriver", NULL); + flb_output_set(ctx, out_ffd, + "match", "kube.custom.tag.*", + "resource", "k8s_container", + "google_service_credentials", SERVICE_CREDENTIALS, + "k8s_cluster_name", "test_cluster_name", + "k8s_cluster_location", "test_cluster_location", + "tag_prefix", "kube.var.log.containers.", + NULL); + + /* Enable test mode */ + ret = flb_output_set_test(ctx, out_ffd, "formatter", + cb_check_k8s_container_resource_default_regex, + NULL, NULL); + + /* Start */ + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + flb_lib_push(ctx, in_ffd, (char *) K8S_CONTAINER_NO_LOCAL_RESOURCE_ID, size); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_resource_k8s_container_custom_k8s_regex() +{ + int ret; + int size = sizeof(K8S_CONTAINER_NO_LOCAL_RESOURCE_ID) - 1; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + + /* Lib input mode */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + flb_input_set(ctx, in_ffd, "tag", + "k8s_container.testnamespace.testpod.testctr", NULL); + + /* Stackdriver output */ + out_ffd = flb_output(ctx, (char *) "stackdriver", NULL); + flb_output_set(ctx, out_ffd, + "match", "k8s_container.*", + "resource", "k8s_container", + "google_service_credentials", SERVICE_CREDENTIALS, + "k8s_cluster_name", "test_cluster_name", + "k8s_cluster_location", "test_cluster_location", + "custom_k8s_regex", "^(?<namespace_name>[^_]+)\\.(?<pod_name>[a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*)\\.(?<container_name>.+)$", + NULL); + + /* Enable test mode */ + ret = flb_output_set_test(ctx, out_ffd, "formatter", + cb_check_k8s_container_resource, + NULL, NULL); + + /* Start */ + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + flb_lib_push(ctx, in_ffd, (char *) K8S_CONTAINER_NO_LOCAL_RESOURCE_ID, size); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_resource_k8s_container_custom_k8s_regex_custom_prefix() +{ + int ret; + int size = sizeof(K8S_CONTAINER_NO_LOCAL_RESOURCE_ID) - 1; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + + /* Lib input mode */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + flb_input_set(ctx, in_ffd, "tag", + "kube.var.log.containers.testnamespace.testpod.testctr", NULL); + + /* Stackdriver output */ + out_ffd = flb_output(ctx, (char *) "stackdriver", NULL); + flb_output_set(ctx, out_ffd, + "match", "kube.var.log.containers.*", + "resource", "k8s_container", + "google_service_credentials", SERVICE_CREDENTIALS, + "k8s_cluster_name", "test_cluster_name", + "k8s_cluster_location", "test_cluster_location", + "tag_prefix", "kube.var.log.containers.", + "custom_k8s_regex", "^(?<namespace_name>[^_]+)\\.(?<pod_name>[a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*)\\.(?<container_name>.+)$", + NULL); + + /* Enable test mode */ + ret = flb_output_set_test(ctx, out_ffd, "formatter", + cb_check_k8s_container_resource, + NULL, NULL); + + /* Start */ + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + flb_lib_push(ctx, in_ffd, (char *) K8S_CONTAINER_NO_LOCAL_RESOURCE_ID, size); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_resource_k8s_node_common() +{ + int ret; + int size = sizeof(K8S_NODE_COMMON) - 1; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + + /* Lib input mode */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + /* Stackdriver output */ + out_ffd = flb_output(ctx, (char *) "stackdriver", NULL); + flb_output_set(ctx, out_ffd, + "match", "test", + "resource", "k8s_node", + "google_service_credentials", SERVICE_CREDENTIALS, + "k8s_cluster_name", "test_cluster_name", + "k8s_cluster_location", "test_cluster_location", + NULL); + + /* Enable test mode */ + ret = flb_output_set_test(ctx, out_ffd, "formatter", + cb_check_k8s_node_resource, + NULL, NULL); + + /* Start */ + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + flb_lib_push(ctx, in_ffd, (char *) K8S_NODE_COMMON, size); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_resource_k8s_pod_common() +{ + int ret; + int size = sizeof(K8S_POD_COMMON) - 1; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + + /* Lib input mode */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + /* Stackdriver output */ + out_ffd = flb_output(ctx, (char *) "stackdriver", NULL); + flb_output_set(ctx, out_ffd, + "match", "test", + "resource", "k8s_pod", + "google_service_credentials", SERVICE_CREDENTIALS, + "k8s_cluster_name", "test_cluster_name", + "k8s_cluster_location", "test_cluster_location", + NULL); + + /* Enable test mode */ + ret = flb_output_set_test(ctx, out_ffd, "formatter", + cb_check_k8s_pod_resource, + NULL, NULL); + + /* Start */ + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + flb_lib_push(ctx, in_ffd, (char *) K8S_POD_COMMON, size); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_default_labels() +{ + int ret; + int size = sizeof(DEFAULT_LABELS) - 1; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + + /* Lib input mode */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + /* Stackdriver output */ + out_ffd = flb_output(ctx, (char *) "stackdriver", NULL); + flb_output_set(ctx, out_ffd, + "match", "test", + "resource", "global", + "google_service_credentials", SERVICE_CREDENTIALS, + NULL); + + /* Enable test mode */ + ret = flb_output_set_test(ctx, out_ffd, "formatter", + cb_check_default_labels, + NULL, NULL); + + /* Start */ + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + flb_lib_push(ctx, in_ffd, (char *) DEFAULT_LABELS, size); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_custom_labels() +{ + int ret; + int size = sizeof(CUSTOM_LABELS) - 1; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + + /* Lib input mode */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + /* Stackdriver output */ + out_ffd = flb_output(ctx, (char *) "stackdriver", NULL); + flb_output_set(ctx, out_ffd, + "match", "test", + "resource", "global", + "google_service_credentials", SERVICE_CREDENTIALS, + "labels_key", "logging.googleapis.com/customlabels", + NULL); + + /* Enable test mode */ + ret = flb_output_set_test(ctx, out_ffd, "formatter", + cb_check_custom_labels, + NULL, NULL); + + /* Start */ + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + flb_lib_push(ctx, in_ffd, (char *) CUSTOM_LABELS, size); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_config_labels_conflict() +{ + int ret; + int size = sizeof(DEFAULT_LABELS) - 1; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + + /* Lib input mode */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + /* Stackdriver output */ + out_ffd = flb_output(ctx, (char *) "stackdriver", NULL); + flb_output_set(ctx, out_ffd, + "match", "test", + "resource", "global", + "google_service_credentials", SERVICE_CREDENTIALS, + "labels", "testB=valC", + NULL); + + /* Enable test mode */ + ret = flb_output_set_test(ctx, out_ffd, "formatter", + cb_check_config_labels_conflict, + NULL, NULL); + + /* Start */ + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + flb_lib_push(ctx, in_ffd, (char *) DEFAULT_LABELS, size); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_config_labels_no_conflict() +{ + int ret; + int size = sizeof(DEFAULT_LABELS) - 1; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + + /* Lib input mode */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + /* Stackdriver output */ + out_ffd = flb_output(ctx, (char *) "stackdriver", NULL); + flb_output_set(ctx, out_ffd, + "match", "test", + "resource", "global", + "google_service_credentials", SERVICE_CREDENTIALS, + "labels", "testC=valC", + NULL); + + /* Enable test mode */ + ret = flb_output_set_test(ctx, out_ffd, "formatter", + cb_check_config_labels_no_conflict, + NULL, NULL); + + /* Start */ + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + flb_lib_push(ctx, in_ffd, (char *) DEFAULT_LABELS, size); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_default_labels_k8s_resource_type() +{ + int ret; + int size = sizeof(DEFAULT_LABELS_K8S_RESOURCE_TYPE) - 1; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + + /* Lib input mode */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + /* Stackdriver output */ + out_ffd = flb_output(ctx, (char *) "stackdriver", NULL); + flb_output_set(ctx, out_ffd, + "match", "test", + "resource", "k8s_container", + "google_service_credentials", SERVICE_CREDENTIALS, + "k8s_cluster_name", "test_cluster_name", + "k8s_cluster_location", "test_cluster_location", + NULL); + + /* Enable test mode */ + ret = flb_output_set_test(ctx, out_ffd, "formatter", + cb_check_default_labels_k8s_resource_type, + NULL, NULL); + + /* Start */ + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + flb_lib_push(ctx, in_ffd, (char *) DEFAULT_LABELS_K8S_RESOURCE_TYPE, size); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_resource_labels_one_field() +{ + int ret; + int size = sizeof(ONE_FIELD) - 1; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + + /* Lib input mode */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + /* Stackdriver output */ + out_ffd = flb_output(ctx, (char *) "stackdriver", NULL); + flb_output_set(ctx, out_ffd, + "match", "test", + "google_service_credentials", SERVICE_CREDENTIALS, + "resource_labels", "keyB=$keyA", + NULL); + + /* Enable test mode */ + ret = flb_output_set_test(ctx, out_ffd, "formatter", + cb_check_resource_labels_one_field, + NULL, NULL); + + /* Start */ + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + flb_lib_push(ctx, in_ffd, (char *) ONE_FIELD, size); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_resource_labels_plaintext() +{ + int ret; + int size = sizeof(ONE_FIELD) - 1; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + + /* Lib input mode */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + /* Stackdriver output */ + out_ffd = flb_output(ctx, (char *) "stackdriver", NULL); + flb_output_set(ctx, out_ffd, + "match", "test", + "google_service_credentials", SERVICE_CREDENTIALS, + "resource_labels", "keyB=plaintext", + NULL); + + /* Enable test mode */ + ret = flb_output_set_test(ctx, out_ffd, "formatter", + cb_check_resource_labels_plaintext, + NULL, NULL); + + /* Start */ + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + flb_lib_push(ctx, in_ffd, (char *) ONE_FIELD, size); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_resource_labels_multiple_fields() +{ + int ret; + int size = sizeof(MULTIPLE_FIELDS) - 1; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + + /* Lib input mode */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + /* Stackdriver output */ + out_ffd = flb_output(ctx, (char *) "stackdriver", NULL); + flb_output_set(ctx, out_ffd, + "match", "test", + "google_service_credentials", SERVICE_CREDENTIALS, + "resource_labels", "keyD=$keyB,keyC=$keyA", + NULL); + + /* Enable test mode */ + ret = flb_output_set_test(ctx, out_ffd, "formatter", + cb_check_multiple_fields_mapped, + NULL, NULL); + + /* Start */ + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + flb_lib_push(ctx, in_ffd, (char *) MULTIPLE_FIELDS, size); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_resource_labels_nested_fields() +{ + int ret; + int size = sizeof(NESTED_FIELDS) - 1; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + + /* Lib input mode */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + /* Stackdriver output */ + out_ffd = flb_output(ctx, (char *) "stackdriver", NULL); + flb_output_set(ctx, out_ffd, + "match", "test", + "google_service_credentials", SERVICE_CREDENTIALS, + "resource_labels", "keyD=$toplevel['keyB'],keyC=$toplevel['keyA']", + NULL); + + /* Enable test mode */ + ret = flb_output_set_test(ctx, out_ffd, "formatter", + cb_check_nested_fields_mapped, + NULL, NULL); + + /* Start */ + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + flb_lib_push(ctx, in_ffd, (char *) NESTED_FIELDS, size); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_resource_labels_layered_nested_fields() +{ + int ret; + int size = sizeof(LAYERED_NESTED_FIELDS) - 1; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + + /* Lib input mode */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + /* Stackdriver output */ + out_ffd = flb_output(ctx, (char *) "stackdriver", NULL); + flb_output_set(ctx, out_ffd, + "match", "test", + "google_service_credentials", SERVICE_CREDENTIALS, + "resource_labels", + "keyD=$toplevel['keyB'],keyC=$toplevel['midlevel']['keyA']", + NULL); + + /* Enable test mode */ + ret = flb_output_set_test(ctx, out_ffd, "formatter", + cb_check_layered_nested_fields_mapped, + NULL, NULL); + + /* Start */ + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + flb_lib_push(ctx, in_ffd, (char *) LAYERED_NESTED_FIELDS, size); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_resource_labels_original_does_not_exist() +{ + int ret; + int size = sizeof(ONE_FIELD) - 1; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + + /* Lib input mode */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + /* Stackdriver output */ + out_ffd = flb_output(ctx, (char *) "stackdriver", NULL); + flb_output_set(ctx, out_ffd, + "match", "test", + "google_service_credentials", SERVICE_CREDENTIALS, + "resource_labels", "keyC=$keyB", + NULL); + + /* Enable test mode */ + ret = flb_output_set_test(ctx, out_ffd, "formatter", + cb_check_resource_labels_original_does_not_exist, + NULL, NULL); + + /* Start */ + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + flb_lib_push(ctx, in_ffd, (char *) ONE_FIELD, size); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_resource_labels_nested_original_does_not_exist() +{ + int ret; + int size = sizeof(ONE_FIELD) - 1; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + + /* Lib input mode */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + /* Stackdriver output */ + out_ffd = flb_output(ctx, (char *) "stackdriver", NULL); + flb_output_set(ctx, out_ffd, + "match", "test", + "google_service_credentials", SERVICE_CREDENTIALS, + "resource_labels", "keyC=$keyY['keyZ']", + NULL); + + /* Enable test mode */ + ret = flb_output_set_test(ctx, out_ffd, "formatter", + cb_check_resource_labels_original_does_not_exist, + NULL, NULL); + + /* Start */ + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + flb_lib_push(ctx, in_ffd, (char *) ONE_FIELD, size); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_resource_labels_nested_original_partially_exists() +{ + int ret; + int size = sizeof(NESTED_FIELDS) - 1; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + + /* Lib input mode */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + /* Stackdriver output */ + out_ffd = flb_output(ctx, (char *) "stackdriver", NULL); + flb_output_set(ctx, out_ffd, + "match", "test", + "google_service_credentials", SERVICE_CREDENTIALS, + "resource_labels", "keyC=$toplevel['keyZ']", + NULL); + + /* Enable test mode */ + ret = flb_output_set_test(ctx, out_ffd, "formatter", + cb_check_resource_labels_nested_original_partially_exists, + NULL, NULL); + + /* Start */ + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + flb_lib_push(ctx, in_ffd, (char *) NESTED_FIELDS, size); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_resource_labels_one_field_with_spaces() +{ + int ret; + int size = sizeof(ONE_FIELD) - 1; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + + /* Lib input mode */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + /* Stackdriver output */ + out_ffd = flb_output(ctx, (char *) "stackdriver", NULL); + flb_output_set(ctx, out_ffd, + "match", "test", + "google_service_credentials", SERVICE_CREDENTIALS, + "resource_labels", "keyB = $keyA", + NULL); + + /* Enable test mode */ + ret = flb_output_set_test(ctx, out_ffd, "formatter", + cb_check_resource_labels_one_field, + NULL, NULL); + + /* Start */ + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + flb_lib_push(ctx, in_ffd, (char *) ONE_FIELD, size); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_resource_labels_multiple_fields_with_spaces() +{ + int ret; + int size = sizeof(MULTIPLE_FIELDS) - 1; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + + /* Lib input mode */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + /* Stackdriver output */ + out_ffd = flb_output(ctx, (char *) "stackdriver", NULL); + flb_output_set(ctx, out_ffd, + "match", "test", + "google_service_credentials", SERVICE_CREDENTIALS, + "resource_labels", "keyD = $keyB, keyC = $keyA", + NULL); + + /* Enable test mode */ + ret = flb_output_set_test(ctx, out_ffd, "formatter", + cb_check_multiple_fields_mapped, + NULL, NULL); + + /* Start */ + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + flb_lib_push(ctx, in_ffd, (char *) MULTIPLE_FIELDS, size); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_resource_labels_empty_input() +{ + int ret; + int size = sizeof(ONE_FIELD) - 1; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + + /* Lib input mode */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + /* Stackdriver output */ + out_ffd = flb_output(ctx, (char *) "stackdriver", NULL); + flb_output_set(ctx, out_ffd, + "match", "test", + "google_service_credentials", SERVICE_CREDENTIALS, + "resource_labels", "", + NULL); + + /* Enable test mode */ + ret = flb_output_set_test(ctx, out_ffd, "formatter", + cb_check_resource_labels_original_does_not_exist, + NULL, NULL); + + /* Start */ + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + flb_lib_push(ctx, in_ffd, (char *) ONE_FIELD, size); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_resource_labels_duplicate_assignment() +{ + int ret; + int size = sizeof(MULTIPLE_FIELDS) - 1; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + + /* Lib input mode */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + /* Stackdriver output */ + out_ffd = flb_output(ctx, (char *) "stackdriver", NULL); + flb_output_set(ctx, out_ffd, + "match", "test", + "google_service_credentials", SERVICE_CREDENTIALS, + "resource_labels", "keyC=$keyA,keyC=$keyB", + NULL); + + /* Enable test mode */ + ret = flb_output_set_test(ctx, out_ffd, "formatter", + cb_check_resource_labels_duplicate_assignment, + NULL, NULL); + + /* Start */ + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + flb_lib_push(ctx, in_ffd, (char *) MULTIPLE_FIELDS, size); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_resource_labels_project_id_not_overridden() +{ + int ret; + int size = sizeof(ONE_FIELD) - 1; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + + /* Lib input mode */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + /* Stackdriver output */ + out_ffd = flb_output(ctx, (char *) "stackdriver", NULL); + flb_output_set(ctx, out_ffd, + "match", "test", + "google_service_credentials", SERVICE_CREDENTIALS, + "resource_labels", "project_id=$keyA", + NULL); + + /* Enable test mode */ + ret = flb_output_set_test(ctx, out_ffd, "formatter", + cb_check_resource_labels_project_id_not_overridden, + NULL, NULL); + + /* Start */ + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + flb_lib_push(ctx, in_ffd, (char *) ONE_FIELD, size); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_resource_labels_has_priority() +{ + int ret; + int size = sizeof(ONE_FIELD) - 1; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + + /* Lib input mode */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + /* Stackdriver output */ + out_ffd = flb_output(ctx, (char *) "stackdriver", NULL); + flb_output_set(ctx, out_ffd, + "match", "test", + "resource", "k8s_container", + "google_service_credentials", SERVICE_CREDENTIALS, + "k8s_cluster_name", "test_cluster_name", + "k8s_cluster_location", "test_cluster_location", + "resource_labels", + "cluster_name=name_from_labels,location=loc_from_labels", + NULL); + + /* Enable test mode */ + ret = flb_output_set_test(ctx, out_ffd, "formatter", + cb_check_resource_labels_k8s, + NULL, NULL); + + /* Start */ + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + flb_lib_push(ctx, in_ffd, (char *) ONE_FIELD, size); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_resource_labels_fallsback_when_required_not_specified() +{ + int ret; + int size = sizeof(K8S_CONTAINER_COMMON) - 1; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + + /* Lib input mode */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + /* Stackdriver output */ + out_ffd = flb_output(ctx, (char *) "stackdriver", NULL); + flb_output_set(ctx, out_ffd, + "match", "test", + "resource", "k8s_container", + "google_service_credentials", SERVICE_CREDENTIALS, + "k8s_cluster_name", "test_cluster_name", + "k8s_cluster_location", "test_cluster_location", + "resource_labels", "keyB=$keyA", + NULL); + + /* Enable test mode */ + ret = flb_output_set_test(ctx, out_ffd, "formatter", + cb_check_k8s_container_resource, + NULL, NULL); + + /* Start */ + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + flb_lib_push(ctx, in_ffd, (char *) K8S_CONTAINER_COMMON, size); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_resource_labels_fallsback_when_required_partially_specified() +{ + int ret; + int size = sizeof(K8S_CONTAINER_COMMON) - 1; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + + /* Lib input mode */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + /* Stackdriver output */ + out_ffd = flb_output(ctx, (char *) "stackdriver", NULL); + flb_output_set(ctx, out_ffd, + "match", "test", + "resource", "k8s_container", + "google_service_credentials", SERVICE_CREDENTIALS, + "k8s_cluster_name", "test_cluster_name", + "k8s_cluster_location", "test_cluster_location", + "resource_labels", "location=cluster_loc", + NULL); + + /* Enable test mode */ + ret = flb_output_set_test(ctx, out_ffd, "formatter", + cb_check_k8s_container_resource, + NULL, NULL); + + /* Start */ + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + flb_lib_push(ctx, in_ffd, (char *) K8S_CONTAINER_COMMON, size); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_resource_labels_k8s_container() +{ + int ret; + int size = sizeof(ONE_FIELD) - 1; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + + /* Lib input mode */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + /* Stackdriver output */ + out_ffd = flb_output(ctx, (char *) "stackdriver", NULL); + flb_output_set(ctx, out_ffd, + "match", "test", + "resource", "k8s_container", + "google_service_credentials", SERVICE_CREDENTIALS, + "resource_labels", + "cluster_name=name_from_labels,location=loc_from_labels", + NULL); + + /* Enable test mode */ + ret = flb_output_set_test(ctx, out_ffd, "formatter", + cb_check_resource_labels_k8s, + NULL, NULL); + + /* Start */ + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + flb_lib_push(ctx, in_ffd, (char *) ONE_FIELD, size); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_resource_labels_k8s_node() +{ + int ret; + int size = sizeof(ONE_FIELD) - 1; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + + /* Lib input mode */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + /* Stackdriver output */ + out_ffd = flb_output(ctx, (char *) "stackdriver", NULL); + flb_output_set(ctx, out_ffd, + "match", "test", + "resource", "k8s_node", + "google_service_credentials", SERVICE_CREDENTIALS, + "resource_labels", + "cluster_name=name_from_labels,location=loc_from_labels", + NULL); + + /* Enable test mode */ + ret = flb_output_set_test(ctx, out_ffd, "formatter", + cb_check_resource_labels_k8s, + NULL, NULL); + + /* Start */ + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + flb_lib_push(ctx, in_ffd, (char *) ONE_FIELD, size); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_resource_labels_k8s_pod() +{ + int ret; + int size = sizeof(ONE_FIELD) - 1; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + + /* Lib input mode */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + /* Stackdriver output */ + out_ffd = flb_output(ctx, (char *) "stackdriver", NULL); + flb_output_set(ctx, out_ffd, + "match", "test", + "resource", "k8s_pod", + "google_service_credentials", SERVICE_CREDENTIALS, + "resource_labels", + "cluster_name=name_from_labels,location=loc_from_labels", + NULL); + + /* Enable test mode */ + ret = flb_output_set_test(ctx, out_ffd, "formatter", + cb_check_resource_labels_k8s, + NULL, NULL); + + /* Start */ + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + flb_lib_push(ctx, in_ffd, (char *) ONE_FIELD, size); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_resource_labels_generic_node() +{ + int ret; + int size = sizeof(ONE_FIELD) - 1; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + + /* Lib input mode */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + /* Stackdriver output */ + out_ffd = flb_output(ctx, (char *) "stackdriver", NULL); + flb_output_set(ctx, out_ffd, + "match", "test", + "google_service_credentials", SERVICE_CREDENTIALS, + "resource", "generic_node", + "resource_labels", + "location=fluent,namespace=test,node_id=333222111", + NULL); + + /* Enable test mode */ + ret = flb_output_set_test(ctx, out_ffd, "formatter", + cb_check_generic_node_creds, + NULL, NULL); + + /* Start */ + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + flb_lib_push(ctx, in_ffd, (char *) ONE_FIELD, size); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_resource_labels_generic_task() +{ + int ret; + int size = sizeof(ONE_FIELD) - 1; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + + /* Lib input mode */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + /* Stackdriver output */ + out_ffd = flb_output(ctx, (char *) "stackdriver", NULL); + flb_output_set(ctx, out_ffd, + "match", "test", + "google_service_credentials", SERVICE_CREDENTIALS, + "resource", "generic_task", + "resource_labels", + "location=fluent,namespace=test,job=test-job,task_id=333222111", + NULL); + + /* Enable test mode */ + ret = flb_output_set_test(ctx, out_ffd, "formatter", + cb_check_generic_task_creds, + NULL, NULL); + + /* Start */ + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + flb_lib_push(ctx, in_ffd, (char *) ONE_FIELD, size); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_custom_labels_k8s_resource_type() +{ + int ret; + int size = sizeof(CUSTOM_LABELS_K8S_RESOURCE_TYPE) - 1; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + + /* Lib input mode */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + /* Stackdriver output */ + out_ffd = flb_output(ctx, (char *) "stackdriver", NULL); + flb_output_set(ctx, out_ffd, + "match", "test", + "resource", "k8s_container", + "google_service_credentials", SERVICE_CREDENTIALS, + "labels_key", "logging.googleapis.com/customlabels", + "k8s_cluster_name", "test_cluster_name", + "k8s_cluster_location", "test_cluster_location", + NULL); + + /* Enable test mode */ + ret = flb_output_set_test(ctx, out_ffd, "formatter", + cb_check_custom_labels_k8s_resource_type, + NULL, NULL); + + /* Start */ + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + flb_lib_push(ctx, in_ffd, (char *) CUSTOM_LABELS_K8S_RESOURCE_TYPE, size); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_resource_k8s_container_no_local_resource_id() +{ + int ret; + int size = sizeof(K8S_CONTAINER_NO_LOCAL_RESOURCE_ID) - 1; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + + /* Lib input mode */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + flb_input_set(ctx, in_ffd, "tag", + "k8s_container.testnamespace.testpod.testctr", NULL); + + /* Stackdriver output */ + out_ffd = flb_output(ctx, (char *) "stackdriver", NULL); + flb_output_set(ctx, out_ffd, + "match", "k8s_container.*", + "resource", "k8s_container", + "google_service_credentials", SERVICE_CREDENTIALS, + "k8s_cluster_name", "test_cluster_name", + "k8s_cluster_location", "test_cluster_location", + NULL); + + /* Enable test mode */ + ret = flb_output_set_test(ctx, out_ffd, "formatter", + cb_check_k8s_container_resource, + NULL, NULL); + + /* Start */ + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + flb_lib_push(ctx, in_ffd, (char *) K8S_CONTAINER_NO_LOCAL_RESOURCE_ID, size); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_resource_k8s_node_no_local_resource_id() +{ + int ret; + int size = sizeof(K8S_NODE_NO_LOCAL_RESOURCE_ID) - 1; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + + /* Lib input mode */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + flb_input_set(ctx, in_ffd, "tag", "k8s_node.testnode", NULL); + + /* Stackdriver output */ + out_ffd = flb_output(ctx, (char *) "stackdriver", NULL); + flb_output_set(ctx, out_ffd, + "match", "k8s_node.*", + "resource", "k8s_node", + "google_service_credentials", SERVICE_CREDENTIALS, + "k8s_cluster_name", "test_cluster_name", + "k8s_cluster_location", "test_cluster_location", + NULL); + + /* Enable test mode */ + ret = flb_output_set_test(ctx, out_ffd, "formatter", + cb_check_k8s_node_resource, + NULL, NULL); + + /* Start */ + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + flb_lib_push(ctx, in_ffd, (char *) K8S_NODE_NO_LOCAL_RESOURCE_ID, size); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_resource_k8s_node_custom_k8s_regex_with_dot() +{ + int ret; + int size = sizeof(K8S_NODE_LOCAL_RESOURCE_ID_WITH_DOT) - 1; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + + /* Lib input mode */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + /* Stackdriver output */ + out_ffd = flb_output(ctx, (char *) "stackdriver", NULL); + flb_output_set(ctx, out_ffd, + "match", "test", + "resource", "k8s_node", + "google_service_credentials", SERVICE_CREDENTIALS, + "k8s_cluster_name", "test_cluster_name", + "k8s_cluster_location", "test_cluster_location", + "custom_k8s_regex", "^(?<node_name>.*)$", + NULL); + + /* Enable test mode */ + ret = flb_output_set_test(ctx, out_ffd, "formatter", + cb_check_k8s_node_custom_k8s_regex, + NULL, NULL); + + /* Start */ + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + flb_lib_push(ctx, in_ffd, (char *) K8S_NODE_LOCAL_RESOURCE_ID_WITH_DOT, size); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_resource_k8s_node_custom_k8s_regex_with_long_tag() +{ + int ret; + int size = sizeof(K8S_NODE_LOCAL_RESOURCE_ID_WITH_DOT) - 1; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + + /* Lib input mode */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + flb_input_set(ctx, in_ffd, "tag", "tagWithLongLen", NULL); + + /* Stackdriver output */ + out_ffd = flb_output(ctx, (char *) "stackdriver", NULL); + flb_output_set(ctx, out_ffd, + "match", "tagWithLongLen", + "resource", "k8s_node", + "google_service_credentials", SERVICE_CREDENTIALS, + "k8s_cluster_name", "test_cluster_name", + "k8s_cluster_location", "test_cluster_location", + "custom_k8s_regex", "^(?<node_name>.*)$", + NULL); + + /* Enable test mode */ + ret = flb_output_set_test(ctx, out_ffd, "formatter", + cb_check_k8s_node_custom_k8s_regex, + NULL, NULL); + + /* Start */ + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + flb_lib_push(ctx, in_ffd, (char *) K8S_NODE_LOCAL_RESOURCE_ID_WITH_DOT, size); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_resource_k8s_pod_no_local_resource_id() +{ + int ret; + int size = sizeof(K8S_POD_NO_LOCAL_RESOURCE_ID) - 1; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + + /* Lib input mode */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + flb_input_set(ctx, in_ffd, "tag", "k8s_pod.testnamespace.testpod", NULL); + + /* Stackdriver output */ + out_ffd = flb_output(ctx, (char *) "stackdriver", NULL); + flb_output_set(ctx, out_ffd, + "match", "k8s_pod.*", + "resource", "k8s_pod", + "google_service_credentials", SERVICE_CREDENTIALS, + "k8s_cluster_name", "test_cluster_name", + "k8s_cluster_location", "test_cluster_location", + NULL); + + /* Enable test mode */ + ret = flb_output_set_test(ctx, out_ffd, "formatter", + cb_check_k8s_pod_resource, + NULL, NULL); + + /* Start */ + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + flb_lib_push(ctx, in_ffd, (char *) K8S_POD_NO_LOCAL_RESOURCE_ID, size); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_multi_entries_severity() +{ + int ret; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + ret = flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + TEST_CHECK_(ret == 0, "setting service options"); + + /* Tail input mode */ + in_ffd = flb_input(ctx, (char *) "tail", NULL); + ret = flb_input_set(ctx, in_ffd, + "Path", STACKDRIVER_DATA_PATH "/stackdriver_multi_entries_severity.log", + "tag", "test", + "read_from_head", "true", + NULL); + TEST_CHECK_(ret == 0, "setting input options"); + + /* Stackdriver output */ + out_ffd = flb_output(ctx, (char *) "stackdriver", NULL); + ret = flb_output_set(ctx, out_ffd, + "match", "test", + "resource", "gce_instance", + "google_service_credentials", SERVICE_CREDENTIALS, + "severity_key", "severity", + NULL); + TEST_CHECK_(ret == 0, "setting output options"); + + /* Enable test mode */ + ret = flb_output_set_test(ctx, out_ffd, "formatter", + cb_check_multi_entries_severity, + NULL, NULL); + + /* Start */ + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_source_location_common_case() +{ + int ret; + int size = sizeof(SOURCELOCATION_COMMON_CASE) - 1; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + + /* Lib input mode */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + /* Stackdriver output */ + out_ffd = flb_output(ctx, (char *) "stackdriver", NULL); + flb_output_set(ctx, out_ffd, + "match", "test", + "resource", "gce_instance", + NULL); + + /* Enable test mode */ + ret = flb_output_set_test(ctx, out_ffd, "formatter", + cb_check_source_location_common_case, + NULL, NULL); + + /* Start */ + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + flb_lib_push(ctx, in_ffd, (char *) SOURCELOCATION_COMMON_CASE, size); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_source_location_line_in_string() +{ + int ret; + int size = sizeof(SOURCELOCATION_COMMON_CASE_LINE_IN_STRING) - 1; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + + /* Lib input mode */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + /* Stackdriver output */ + out_ffd = flb_output(ctx, (char *) "stackdriver", NULL); + flb_output_set(ctx, out_ffd, + "match", "test", + "resource", "gce_instance", + NULL); + + /* Enable test mode */ + ret = flb_output_set_test(ctx, out_ffd, "formatter", + cb_check_source_location_common_case_line_in_string, + NULL, NULL); + + /* Start */ + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + flb_lib_push(ctx, in_ffd, (char *) SOURCELOCATION_COMMON_CASE_LINE_IN_STRING, size); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_empty_source_location() +{ + int ret; + int size = sizeof(EMPTY_SOURCELOCATION) - 1; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + + /* Lib input mode */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + /* Stackdriver output */ + out_ffd = flb_output(ctx, (char *) "stackdriver", NULL); + flb_output_set(ctx, out_ffd, + "match", "test", + "resource", "gce_instance", + NULL); + + /* Enable test mode */ + ret = flb_output_set_test(ctx, out_ffd, "formatter", + cb_check_empty_source_location, + NULL, NULL); + + /* Start */ + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + flb_lib_push(ctx, in_ffd, (char *) EMPTY_SOURCELOCATION, size); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_source_location_in_string() +{ + int ret; + int size = sizeof(SOURCELOCATION_IN_STRING) - 1; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + + /* Lib input mode */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + /* Stackdriver output */ + out_ffd = flb_output(ctx, (char *) "stackdriver", NULL); + flb_output_set(ctx, out_ffd, + "match", "test", + "resource", "gce_instance", + NULL); + + /* Enable test mode */ + ret = flb_output_set_test(ctx, out_ffd, "formatter", + cb_check_source_location_in_string, + NULL, NULL); + + /* Start */ + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + flb_lib_push(ctx, in_ffd, (char *) SOURCELOCATION_IN_STRING, size); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_source_location_partial_subfields() +{ + int ret; + int size = sizeof(PARTIAL_SOURCELOCATION) - 1; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + + /* Lib input mode */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + /* Stackdriver output */ + out_ffd = flb_output(ctx, (char *) "stackdriver", NULL); + flb_output_set(ctx, out_ffd, + "match", "test", + "resource", "gce_instance", + NULL); + + /* Enable test mode */ + ret = flb_output_set_test(ctx, out_ffd, "formatter", + cb_check_source_location_partial_subfields, + NULL, NULL); + + /* Start */ + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + flb_lib_push(ctx, in_ffd, (char *) PARTIAL_SOURCELOCATION, size); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_source_location_incorrect_type_subfields() +{ + int ret; + int size = sizeof(SOURCELOCATION_SUBFIELDS_IN_INCORRECT_TYPE) - 1; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + + /* Lib input mode */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + /* Stackdriver output */ + out_ffd = flb_output(ctx, (char *) "stackdriver", NULL); + flb_output_set(ctx, out_ffd, + "match", "test", + "resource", "gce_instance", + NULL); + + /* Enable test mode */ + ret = flb_output_set_test(ctx, out_ffd, "formatter", + cb_check_source_location_incorrect_type_subfields, + NULL, NULL); + + /* Start */ + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + flb_lib_push(ctx, in_ffd, (char *) SOURCELOCATION_SUBFIELDS_IN_INCORRECT_TYPE, size); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_source_location_extra_subfields() +{ + int ret; + int size = sizeof(SOURCELOCATION_EXTRA_SUBFIELDS_EXISTED) - 1; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + + /* Lib input mode */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + /* Stackdriver output */ + out_ffd = flb_output(ctx, (char *) "stackdriver", NULL); + flb_output_set(ctx, out_ffd, + "match", "test", + "resource", "gce_instance", + NULL); + + /* Enable test mode */ + ret = flb_output_set_test(ctx, out_ffd, "formatter", + cb_check_source_location_extra_subfields, + NULL, NULL); + + /* Start */ + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + flb_lib_push(ctx, in_ffd, (char *) SOURCELOCATION_EXTRA_SUBFIELDS_EXISTED, size); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_http_request_common_case() +{ + int ret; + int size = sizeof(HTTPREQUEST_COMMON_CASE) - 1; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + + /* Lib input mode */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + /* Stackdriver output */ + out_ffd = flb_output(ctx, (char *) "stackdriver", NULL); + flb_output_set(ctx, out_ffd, + "match", "test", + "resource", "gce_instance", + NULL); + + /* Enable test mode */ + ret = flb_output_set_test(ctx, out_ffd, "formatter", + cb_check_http_request_common_case, + NULL, NULL); + + /* Start */ + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + flb_lib_push(ctx, in_ffd, (char *) HTTPREQUEST_COMMON_CASE, size); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_empty_http_request() +{ + int ret; + int size = sizeof(EMPTY_HTTPREQUEST) - 1; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + + /* Lib input mode */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + /* Stackdriver output */ + out_ffd = flb_output(ctx, (char *) "stackdriver", NULL); + flb_output_set(ctx, out_ffd, + "match", "test", + "resource", "gce_instance", + NULL); + + /* Enable test mode */ + ret = flb_output_set_test(ctx, out_ffd, "formatter", + cb_check_empty_http_request, + NULL, NULL); + + /* Start */ + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + flb_lib_push(ctx, in_ffd, (char *) EMPTY_HTTPREQUEST, size); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_http_request_in_string() +{ + int ret; + int size = sizeof(HTTPREQUEST_IN_STRING) - 1; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + + /* Lib input mode */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + /* Stackdriver output */ + out_ffd = flb_output(ctx, (char *) "stackdriver", NULL); + flb_output_set(ctx, out_ffd, + "match", "test", + "resource", "gce_instance", + NULL); + + /* Enable test mode */ + ret = flb_output_set_test(ctx, out_ffd, "formatter", + cb_check_http_request_in_string, + NULL, NULL); + + /* Start */ + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + flb_lib_push(ctx, in_ffd, (char *) HTTPREQUEST_IN_STRING, size); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_http_request_partial_subfields() +{ + int ret; + int size = sizeof(PARTIAL_HTTPREQUEST) - 1; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + + /* Lib input mode */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + /* Stackdriver output */ + out_ffd = flb_output(ctx, (char *) "stackdriver", NULL); + flb_output_set(ctx, out_ffd, + "match", "test", + "resource", "gce_instance", + NULL); + + /* Enable test mode */ + ret = flb_output_set_test(ctx, out_ffd, "formatter", + cb_check_http_request_partial_subfields, + NULL, NULL); + + /* Start */ + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + flb_lib_push(ctx, in_ffd, (char *) PARTIAL_HTTPREQUEST, size); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_http_request_incorrect_type_subfields() +{ + int ret; + int size = sizeof(HTTPREQUEST_SUBFIELDS_IN_INCORRECT_TYPE) - 1; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + + /* Lib input mode */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + /* Stackdriver output */ + out_ffd = flb_output(ctx, (char *) "stackdriver", NULL); + flb_output_set(ctx, out_ffd, + "match", "test", + "resource", "gce_instance", + NULL); + + /* Enable test mode */ + ret = flb_output_set_test(ctx, out_ffd, "formatter", + cb_check_http_request_incorrect_type_subfields, + NULL, NULL); + + /* Start */ + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + flb_lib_push(ctx, in_ffd, (char *) HTTPREQUEST_SUBFIELDS_IN_INCORRECT_TYPE, size); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_http_request_extra_subfields() +{ + int ret; + int size = sizeof(HTTPREQUEST_EXTRA_SUBFIELDS_EXISTED) - 1; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + + /* Lib input mode */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + /* Stackdriver output */ + out_ffd = flb_output(ctx, (char *) "stackdriver", NULL); + flb_output_set(ctx, out_ffd, + "match", "test", + "resource", "gce_instance", + NULL); + + /* Enable test mode */ + ret = flb_output_set_test(ctx, out_ffd, "formatter", + cb_check_http_request_extra_subfields, + NULL, NULL); + + /* Start */ + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + flb_lib_push(ctx, in_ffd, (char *) HTTPREQUEST_EXTRA_SUBFIELDS_EXISTED, size); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_http_request_latency_common_case() +{ + int ret; + int size = sizeof(HTTPREQUEST_LATENCY_COMMON_CASE) - 1; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + + /* Lib input mode */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + /* Stackdriver output */ + out_ffd = flb_output(ctx, (char *) "stackdriver", NULL); + flb_output_set(ctx, out_ffd, + "match", "test", + "resource", "gce_instance", + NULL); + + /* Enable test mode */ + ret = flb_output_set_test(ctx, out_ffd, "formatter", + cb_check_http_request_lantency_common_case, + NULL, NULL); + + /* Start */ + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + flb_lib_push(ctx, in_ffd, (char *) HTTPREQUEST_LATENCY_COMMON_CASE, size); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_http_request_latency_invalid_spaces() +{ + int ret; + int size = sizeof(HTTPREQUEST_LATENCY_INVALID_SPACES) - 1; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + + /* Lib input mode */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + /* Stackdriver output */ + out_ffd = flb_output(ctx, (char *) "stackdriver", NULL); + flb_output_set(ctx, out_ffd, + "match", "test", + "resource", "gce_instance", + NULL); + + /* Enable test mode */ + ret = flb_output_set_test(ctx, out_ffd, "formatter", + cb_check_http_request_latency_incorrect_format, + NULL, NULL); + + /* Start */ + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + flb_lib_push(ctx, in_ffd, (char *) HTTPREQUEST_LATENCY_INVALID_SPACES, size); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_http_request_latency_invalid_string() +{ + int ret; + int size = sizeof(HTTPREQUEST_LATENCY_INVALID_STRING) - 1; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + + /* Lib input mode */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + /* Stackdriver output */ + out_ffd = flb_output(ctx, (char *) "stackdriver", NULL); + flb_output_set(ctx, out_ffd, + "match", "test", + "resource", "gce_instance", + NULL); + + /* Enable test mode */ + ret = flb_output_set_test(ctx, out_ffd, "formatter", + cb_check_http_request_latency_incorrect_format, + NULL, NULL); + + /* Start */ + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + flb_lib_push(ctx, in_ffd, (char *) HTTPREQUEST_LATENCY_INVALID_STRING, size); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_http_request_latency_invalid_end() +{ + int ret; + int size = sizeof(HTTPREQUEST_LATENCY_INVALID_END) - 1; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + + /* Lib input mode */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + /* Stackdriver output */ + out_ffd = flb_output(ctx, (char *) "stackdriver", NULL); + flb_output_set(ctx, out_ffd, + "match", "test", + "resource", "gce_instance", + NULL); + + /* Enable test mode */ + ret = flb_output_set_test(ctx, out_ffd, "formatter", + cb_check_http_request_latency_incorrect_format, + NULL, NULL); + + /* Start */ + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + flb_lib_push(ctx, in_ffd, (char *) HTTPREQUEST_LATENCY_INVALID_END, size); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_timestamp_format_object_common() +{ + int ret; + int size = sizeof(TIMESTAMP_FORMAT_OBJECT_COMMON_CASE) - 1; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + + /* Lib input mode */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + /* Stackdriver output */ + out_ffd = flb_output(ctx, (char *) "stackdriver", NULL); + flb_output_set(ctx, out_ffd, + "match", "test", + "resource", "gce_instance", + NULL); + + /* Enable test mode */ + ret = flb_output_set_test(ctx, out_ffd, "formatter", + cb_check_timestamp_format_object_common_case, + NULL, NULL); + + /* Start */ + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + flb_lib_push(ctx, in_ffd, (char *) TIMESTAMP_FORMAT_OBJECT_COMMON_CASE, size); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_timestamp_format_object_not_a_map() +{ + int ret; + int size = sizeof(TIMESTAMP_FORMAT_OBJECT_NOT_A_MAP) - 1; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + + /* Lib input mode */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + /* Stackdriver output */ + out_ffd = flb_output(ctx, (char *) "stackdriver", NULL); + flb_output_set(ctx, out_ffd, + "match", "test", + "resource", "gce_instance", + NULL); + + /* Enable test mode */ + ret = flb_output_set_test(ctx, out_ffd, "formatter", + cb_check_timestamp_format_object_not_a_map, + NULL, NULL); + + /* Start */ + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + flb_lib_push(ctx, in_ffd, (char *) TIMESTAMP_FORMAT_OBJECT_NOT_A_MAP, size); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_timestamp_format_object_missing_subfield() +{ + int ret; + int size = sizeof(TIMESTAMP_FORMAT_OBJECT_MISSING_SUBFIELD) - 1; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + + /* Lib input mode */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + /* Stackdriver output */ + out_ffd = flb_output(ctx, (char *) "stackdriver", NULL); + flb_output_set(ctx, out_ffd, + "match", "test", + "resource", "gce_instance", + NULL); + + /* Enable test mode */ + ret = flb_output_set_test(ctx, out_ffd, "formatter", + cb_check_timestamp_format_object_missing_subfield, + NULL, NULL); + + /* Start */ + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + flb_lib_push(ctx, in_ffd, (char *) TIMESTAMP_FORMAT_OBJECT_MISSING_SUBFIELD, size); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_timestamp_format_object_incorrect_subfields() +{ + int ret; + int size = sizeof(TIMESTAMP_FORMAT_OBJECT_INCORRECT_TYPE_SUBFIELDS) - 1; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + + /* Lib input mode */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + /* Stackdriver output */ + out_ffd = flb_output(ctx, (char *) "stackdriver", NULL); + flb_output_set(ctx, out_ffd, + "match", "test", + "resource", "gce_instance", + NULL); + + /* Enable test mode */ + ret = flb_output_set_test(ctx, out_ffd, "formatter", + cb_check_timestamp_format_object_incorrect_subfields, + NULL, NULL); + + /* Start */ + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + flb_lib_push(ctx, in_ffd, (char *) TIMESTAMP_FORMAT_OBJECT_INCORRECT_TYPE_SUBFIELDS, size); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_timestamp_format_duo_fields_common_case() +{ + int ret; + int size = sizeof(TIMESTAMP_FORMAT_DUO_FIELDS_COMMON_CASE) - 1; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + + /* Lib input mode */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + /* Stackdriver output */ + out_ffd = flb_output(ctx, (char *) "stackdriver", NULL); + flb_output_set(ctx, out_ffd, + "match", "test", + "resource", "gce_instance", + NULL); + + /* Enable test mode */ + ret = flb_output_set_test(ctx, out_ffd, "formatter", + cb_check_timestamp_format_duo_fields_common_case, + NULL, NULL); + + /* Start */ + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + flb_lib_push(ctx, in_ffd, (char *) TIMESTAMP_FORMAT_DUO_FIELDS_COMMON_CASE, size); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_timestamp_format_duo_fields_missing_nanos() +{ + int ret; + int size = sizeof(TIMESTAMP_FORMAT_DUO_FIELDS_MISSING_NANOS) - 1; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + + /* Lib input mode */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + /* Stackdriver output */ + out_ffd = flb_output(ctx, (char *) "stackdriver", NULL); + flb_output_set(ctx, out_ffd, + "match", "test", + "resource", "gce_instance", + NULL); + + /* Enable test mode */ + ret = flb_output_set_test(ctx, out_ffd, "formatter", + cb_check_timestamp_format_duo_fields_missing_nanos, + NULL, NULL); + + /* Start */ + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + flb_lib_push(ctx, in_ffd, (char *) TIMESTAMP_FORMAT_DUO_FIELDS_MISSING_NANOS, size); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_timestamp_format_duo_fields_incorrect_type() +{ + int ret; + int size = sizeof(TIMESTAMP_FORMAT_DUO_FIELDS_INCORRECT_TYPE) - 1; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + + /* Lib input mode */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + /* Stackdriver output */ + out_ffd = flb_output(ctx, (char *) "stackdriver", NULL); + flb_output_set(ctx, out_ffd, + "match", "test", + "resource", "gce_instance", + NULL); + + /* Enable test mode */ + ret = flb_output_set_test(ctx, out_ffd, "formatter", + cb_check_timestamp_format_duo_fields_incorrect_type, + NULL, NULL); + + /* Start */ + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + flb_lib_push(ctx, in_ffd, (char *) TIMESTAMP_FORMAT_DUO_FIELDS_INCORRECT_TYPE, size); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + +/* Test list */ +TEST_LIST = { + {"severity_multi_entries", flb_test_multi_entries_severity }, + {"resource_global", flb_test_resource_global }, + {"resource_global_custom_prefix", flb_test_resource_global_custom_prefix }, + {"resource_gce_instance", flb_test_resource_gce_instance }, + + /* generic resources */ + {"resource_generic_node_creds", flb_test_resource_generic_node_creds}, + {"resource_generic_node_metadata", flb_test_resource_generic_node_metadata}, + {"resource_generic_task_creds", flb_test_resource_generic_task_creds}, + {"resource_generic_task_metadata", flb_test_resource_generic_task_metadata}, + + /* test trace */ + {"trace_no_autoformat", flb_test_trace_no_autoformat}, + {"trace_stackdriver_autoformat", flb_test_trace_stackdriver_autoformat}, + + /* test span id */ + {"span_id", flb_test_span_id}, + + /* test trace sampled */ + {"trace_sampled_true", flb_test_trace_sampled_true}, + {"trace_sampled_false", flb_test_trace_sampled_false}, + + /* test metadata server */ + {"set_metadata_server", flb_test_set_metadata_server}, + + /* test log name */ + {"log_name_override", flb_test_log_name_override}, + {"log_name_no_override", flb_test_log_name_no_override}, + + /* test insertId */ + {"insertId_common_case", flb_test_insert_id_common_case}, + {"empty_insertId", flb_test_empty_insert_id}, + {"insertId_incorrect_type_int", flb_test_insert_id_incorrect_type}, + + /* test operation */ + {"operation_common_case", flb_test_operation_common}, + {"empty_operation", flb_test_empty_operation}, + {"operation_not_a_map", flb_test_operation_in_string}, + {"operation_partial_subfields", flb_test_operation_partial_subfields}, + {"operation_subfields_in_incorrect_type", flb_test_operation_incorrect_type_subfields}, + {"operation_extra_subfields_exist", flb_test_operation_extra_subfields}, + + /* test sourceLocation */ + {"sourceLocation_common_case", flb_test_source_location_common_case}, + {"sourceLocation_line_in_string", flb_test_source_location_line_in_string}, + {"empty_sourceLocation", flb_test_empty_source_location}, + {"sourceLocation_not_a_map", flb_test_source_location_in_string}, + {"sourceLocation_partial_subfields", flb_test_source_location_partial_subfields}, + {"sourceLocation_subfields_in_incorrect_type", flb_test_source_location_incorrect_type_subfields}, + {"sourceLocation_extra_subfields_exist", flb_test_source_location_extra_subfields}, + + /* test monitored resource */ + {"monitored_resource_common", flb_test_monitored_resource_common}, + {"monitored_resource_priority_higher_than_local_resource_id", flb_test_monitored_resource_priority_higher_than_local_resource_id}, + {"monitored_resource_priority_higher_than_gce_instance", flb_test_monitored_resource_priority_higher_than_gce_instance}, + + /* test k8s */ + {"resource_k8s_container_common", flb_test_resource_k8s_container_common }, + {"resource_k8s_container_no_local_resource_id", flb_test_resource_k8s_container_no_local_resource_id }, + {"resource_k8s_container_multi_tag_value", flb_test_resource_k8s_container_multi_tag_value } , + {"resource_k8s_container_custom_tag_prefix", flb_test_resource_k8s_container_custom_tag_prefix }, + {"resource_k8s_container_custom_tag_prefix_with_dot", flb_test_resource_k8s_container_custom_tag_prefix_with_dot }, + {"resource_k8s_container_default_tag_regex", flb_test_resource_k8s_container_default_tag_regex }, + {"resource_k8s_container_custom_k8s_regex", flb_test_resource_k8s_container_custom_k8s_regex }, + {"resource_k8s_container_custom_k8s_regex_custom_prefix", flb_test_resource_k8s_container_custom_k8s_regex_custom_prefix }, + {"resource_k8s_node_common", flb_test_resource_k8s_node_common }, + {"resource_k8s_node_no_local_resource_id", flb_test_resource_k8s_node_no_local_resource_id }, + {"resource_k8s_node_custom_k8s_regex_with_dot", flb_test_resource_k8s_node_custom_k8s_regex_with_dot }, + {"resource_k8s_node_custom_k8s_regex_with_long_tag", flb_test_resource_k8s_node_custom_k8s_regex_with_long_tag }, + {"resource_k8s_pod_common", flb_test_resource_k8s_pod_common }, + {"resource_k8s_pod_no_local_resource_id", flb_test_resource_k8s_pod_no_local_resource_id }, + {"default_labels", flb_test_default_labels }, + {"custom_labels", flb_test_custom_labels }, + {"config_labels_conflict", flb_test_config_labels_conflict }, + {"config_labels_no_conflict", flb_test_config_labels_no_conflict }, + {"default_labels_k8s_resource_type", flb_test_default_labels_k8s_resource_type }, + {"custom_labels_k8s_resource_type", flb_test_custom_labels_k8s_resource_type }, + + /* test resource labels api */ + {"resource_labels_one_field", flb_test_resource_labels_one_field }, + {"resource_labels_plaintext", flb_test_resource_labels_plaintext }, + {"resource_labels_multiple_fields", flb_test_resource_labels_multiple_fields }, + {"resource_labels_nested_fields", flb_test_resource_labels_nested_fields }, + {"resource_labels_layered_nested_fields", flb_test_resource_labels_layered_nested_fields }, + {"resource_labels_original_does_not_exist", flb_test_resource_labels_original_does_not_exist }, + {"resource_labels_nested_original_does_not_exist", flb_test_resource_labels_nested_original_does_not_exist }, + {"resource_labels_nested_original_partially_exists", flb_test_resource_labels_nested_original_partially_exists }, + {"resource_labels_one_field_with_spaces", flb_test_resource_labels_one_field_with_spaces }, + {"resource_labels_multiple_fields_with_spaces", flb_test_resource_labels_multiple_fields_with_spaces }, + {"resource_labels_empty_input", flb_test_resource_labels_empty_input }, + {"resource_labels_duplicate_assignment", flb_test_resource_labels_duplicate_assignment }, + {"resource_labels_project_id_not_overridden", flb_test_resource_labels_project_id_not_overridden }, + {"resource_labels_has_priority", flb_test_resource_labels_has_priority }, + {"resource_labels_fallsback_when_required_not_specified", flb_test_resource_labels_fallsback_when_required_not_specified }, + {"resource_labels_fallsback_when_required_partially_specified", flb_test_resource_labels_fallsback_when_required_partially_specified }, + {"resource_labels_k8s_container", flb_test_resource_labels_k8s_container }, + {"resource_labels_k8s_node", flb_test_resource_labels_k8s_node }, + {"resource_labels_k8s_pod", flb_test_resource_labels_k8s_pod }, + {"resource_labels_generic_node", flb_test_resource_labels_generic_node }, + {"resource_labels_generic_task", flb_test_resource_labels_generic_task }, + + /* test httpRequest */ + {"httpRequest_common_case", flb_test_http_request_common_case}, + {"empty_httpRequest", flb_test_empty_http_request}, + {"httpRequest_not_a_map", flb_test_http_request_in_string}, + {"httpRequest_partial_subfields", flb_test_http_request_partial_subfields}, + {"httpRequest_subfields_in_incorret_type", flb_test_http_request_incorrect_type_subfields}, + {"httpRequest_extra_subfields_exist", flb_test_http_request_extra_subfields}, + {"httpRequest_common_latency", flb_test_http_request_latency_common_case}, + {"httpRequest_latency_incorrect_spaces", flb_test_http_request_latency_invalid_spaces}, + {"httpRequest_latency_incorrect_string", flb_test_http_request_latency_invalid_string}, + {"httpRequest_latency_incorrect_end", flb_test_http_request_latency_invalid_end}, + + /* test timestamp */ + {"timestamp_format_object_common_case", flb_test_timestamp_format_object_common}, + {"timestamp_format_object_not_a_map", flb_test_timestamp_format_object_not_a_map}, + {"timestamp_format_object_missing_subfield", flb_test_timestamp_format_object_missing_subfield}, + {"timestamp_format_object_incorrect_type_subfields", flb_test_timestamp_format_object_incorrect_subfields}, + + {"timestamp_format_duo_fields_common_case", flb_test_timestamp_format_duo_fields_common_case}, + {"timestamp_format_duo_fields_missing_nanos", flb_test_timestamp_format_duo_fields_missing_nanos}, + {"timestamp_format_duo_fields_incorrect_type", flb_test_timestamp_format_duo_fields_incorrect_type}, + + {NULL, NULL} +}; diff --git a/src/fluent-bit/tests/runtime/out_stdout.c b/src/fluent-bit/tests/runtime/out_stdout.c new file mode 100644 index 000000000..26182fbe9 --- /dev/null +++ b/src/fluent-bit/tests/runtime/out_stdout.c @@ -0,0 +1,138 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +#include <fluent-bit.h> +#include "flb_tests_runtime.h" + +/* Test data */ +#include "data/common/json_invalid.h" /* JSON_INVALID */ +#include "data/common/json_long.h" /* JSON_LONG */ +#include "data/common/json_small.h" /* JSON_SMALL */ + +/* Test functions */ +void flb_test_stdout_json_invalid(void); +void flb_test_stdout_json_long(void); +void flb_test_stdout_json_small(void); + +/* Test list */ +TEST_LIST = { + {"json_invalid", flb_test_stdout_json_invalid }, + {"json_long", flb_test_stdout_json_long }, + {"json_small", flb_test_stdout_json_small }, + {NULL, NULL} +}; + + +void flb_test_stdout_json_invalid(void) +{ + int i; + int ret; + int total; + int bytes; + char *p = (char *) JSON_INVALID; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + ctx = flb_create(); + flb_service_set(ctx, "Flush", "1", "Grace", "1", "Log_Level", "error", NULL); + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + out_ffd = flb_output(ctx, (char *) "stdout", NULL); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, "match", "test", NULL); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + total = 0; + for (i = 0; i < (int) sizeof(JSON_INVALID) - 1; i++) { + bytes = flb_lib_push(ctx, in_ffd, p + i, 1); + TEST_CHECK(bytes == 1); + total++; + } + + sleep(1); /* waiting flush */ + + flb_stop(ctx); + flb_destroy(ctx); +} + +/* It writes a very long JSON map (> 100KB) byte by byte */ +void flb_test_stdout_json_long(void) +{ + int i; + int ret; + int total; + int bytes; + char *p = (char *) JSON_LONG; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + ctx = flb_create(); + flb_service_set(ctx, "Flush", "1", "Grace", "1", "Log_Level", "error", NULL); + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + out_ffd = flb_output(ctx, (char *) "stdout", NULL); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, "match", "test", NULL); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + total = 0; + for (i = 0; i < (int) sizeof(JSON_LONG) - 1; i++) { + bytes = flb_lib_push(ctx, in_ffd, p + i, 1); + TEST_CHECK(bytes == 1); + total++; + } + + sleep(1); /* waiting flush */ + + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_stdout_json_small(void) +{ + int i; + int ret; + int total; + int bytes; + char *p = (char *) JSON_SMALL; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + ctx = flb_create(); + flb_service_set(ctx, "Flush", "1", "Grace", "1", "Log_Level", "error", NULL); + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + out_ffd = flb_output(ctx, (char *) "stdout", NULL); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, "match", "test", NULL); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + total = 0; + for (i = 0; i < (int) sizeof(JSON_SMALL) - 1; i++) { + bytes = flb_lib_push(ctx, in_ffd, p + i, 1); + TEST_CHECK(bytes == 1); + total++; + } + + sleep(1); /* waiting flush */ + + flb_stop(ctx); + flb_destroy(ctx); +} diff --git a/src/fluent-bit/tests/runtime/out_syslog.c b/src/fluent-bit/tests/runtime/out_syslog.c new file mode 100644 index 000000000..aa9edab33 --- /dev/null +++ b/src/fluent-bit/tests/runtime/out_syslog.c @@ -0,0 +1,1644 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2019-2022 The Fluent Bit Authors + * Copyright (C) 2015-2018 Treasure Data Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <fluent-bit.h> +#include <fluent-bit/flb_compat.h> +#include <fluent-bit/flb_time.h> +#include <fluent-bit/flb_pack.h> +#include <fluent-bit/flb_socket.h> +#include <sys/types.h> +#include <sys/stat.h> +#include <fcntl.h> +#include "flb_tests_runtime.h" + +struct test_ctx { + flb_ctx_t *flb; /* Fluent Bit library context */ + int i_ffd; /* Input fd */ + int f_ffd; /* Filter fd (unused) */ + int o_ffd; /* Output fd */ +}; + + +pthread_mutex_t result_mutex = PTHREAD_MUTEX_INITIALIZER; +int num_output = 0; +static int get_output_num() +{ + int ret; + pthread_mutex_lock(&result_mutex); + ret = num_output; + pthread_mutex_unlock(&result_mutex); + + return ret; +} + +static void set_output_num(int num) +{ + pthread_mutex_lock(&result_mutex); + num_output = num; + pthread_mutex_unlock(&result_mutex); +} + +static void clear_output_num() +{ + set_output_num(0); +} + +struct str_list { + size_t size; + char **lists; +}; + +/* Callback to check expected results */ +static void cb_check_str_list(void *ctx, int ffd, int res_ret, + void *res_data, size_t res_size, void *data) +{ + char *p; + flb_sds_t out_line = res_data; + int num = get_output_num(); + size_t i; + struct str_list *l = (struct str_list *)data; + + if (!TEST_CHECK(res_data != NULL)) { + TEST_MSG("res_data is NULL"); + return; + } + + if (!TEST_CHECK(l != NULL)) { + TEST_MSG("l is NULL"); + flb_sds_destroy(out_line); + return; + } + + if(!TEST_CHECK(res_ret == 0)) { + TEST_MSG("callback ret=%d", res_ret); + } + if (!TEST_CHECK(res_data != NULL)) { + TEST_MSG("res_data is NULL"); + flb_sds_destroy(out_line); + return; + } + + for (i=0; i<l->size; i++) { + p = strstr(out_line, l->lists[i]); + if (!TEST_CHECK(p != NULL)) { + TEST_MSG(" Got :%s\n expect:%s", out_line, l->lists[i]); + } + } + set_output_num(num+1); + + flb_sds_destroy(out_line); +} + +static struct test_ctx *test_ctx_create() +{ + int i_ffd; + int o_ffd; + struct test_ctx *ctx = NULL; + + ctx = flb_malloc(sizeof(struct test_ctx)); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("malloc failed"); + flb_errno(); + return NULL; + } + + /* Service config */ + ctx->flb = flb_create(); + flb_service_set(ctx->flb, + "Flush", "0.200000000", + "Grace", "1", + "Log_Level", "error", + NULL); + + /* Input */ + i_ffd = flb_input(ctx->flb, (char *) "lib", NULL); + TEST_CHECK(i_ffd >= 0); + ctx->i_ffd = i_ffd; + + /* Output */ + o_ffd = flb_output(ctx->flb, (char *) "syslog", NULL); + ctx->o_ffd = o_ffd; + + return ctx; +} + +static void test_ctx_destroy(struct test_ctx *ctx) +{ + TEST_CHECK(ctx != NULL); + + sleep(1); + flb_stop(ctx->flb); + flb_destroy(ctx->flb); + flb_free(ctx); +} + +void flb_test_syslog_rfc5424() +{ + struct test_ctx *ctx; + int ret; + int num; + + char *buf = "[1, {\"msg\":\"hello world\"}]"; + size_t size = strlen(buf); + + char *expected_strs[] = {"hello world", "1970-01-01T00:00:01.000000Z"}; + struct str_list expected = { + .size = sizeof(expected_strs)/sizeof(char*), + .lists = &expected_strs[0], + }; + + clear_output_num(); + + ctx = test_ctx_create(); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("test_ctx_create failed"); + exit(EXIT_FAILURE); + } + + ret = flb_output_set(ctx->flb, ctx->o_ffd, + "match", "*", + "syslog_format", "rfc5424", + "syslog_message_key", "msg", + NULL); + TEST_CHECK(ret == 0); + + ret = flb_output_set_test(ctx->flb, ctx->o_ffd, + "formatter", cb_check_str_list, + &expected, NULL); + TEST_CHECK(ret == 0); + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + ret = flb_lib_push(ctx->flb, ctx->i_ffd, (char *) buf, size); + TEST_CHECK(ret >= 0); + + /* waiting to flush */ + flb_time_msleep(500); + + num = get_output_num(); + if (!TEST_CHECK(num > 0)) { + TEST_MSG("no outputs"); + } + + test_ctx_destroy(ctx); +} + +void flb_test_severity_key_rfc5424() +{ + struct test_ctx *ctx; + int ret; + int num; + + char *buf = "[1, {\"msg\":\"hello world\", \"s_key\":\"5\"}]"; + size_t size = strlen(buf); + + char *expected_strs[] = {"hello world", "1970-01-01T00:00:01.000000Z", "<13>" /* 1(user-level messages) * 8 + 5(severity) */, + "<13>1 1970-01-01T00:00:01.000000Z - - - - - hello world"}; + struct str_list expected = { + .size = sizeof(expected_strs)/sizeof(char*), + .lists = &expected_strs[0], + }; + + clear_output_num(); + + ctx = test_ctx_create(); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("test_ctx_create failed"); + exit(EXIT_FAILURE); + } + + ret = flb_output_set(ctx->flb, ctx->o_ffd, + "match", "*", + "syslog_format", "rfc5424", + "syslog_message_key", "msg", + "syslog_severity_key", "s_key", + NULL); + TEST_CHECK(ret == 0); + + ret = flb_output_set_test(ctx->flb, ctx->o_ffd, + "formatter", cb_check_str_list, + &expected, NULL); + TEST_CHECK(ret == 0); + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + ret = flb_lib_push(ctx->flb, ctx->i_ffd, (char *) buf, size); + TEST_CHECK(ret >= 0); + + /* waiting to flush */ + flb_time_msleep(500); + + num = get_output_num(); + if (!TEST_CHECK(num > 0)) { + TEST_MSG("no outputs"); + } + + test_ctx_destroy(ctx); +} + + +void flb_test_severity_preset_rfc5424() +{ + struct test_ctx *ctx; + int ret; + int num; + + char *buf = "[1, {\"msg\":\"hello world\", \"s_key\":\"5\"}]"; + size_t size = strlen(buf); + + char *expected_strs[] = {"hello world", "1970-01-01T00:00:01.000000Z", "<13>" /* 1(user-level messages) * 8 + 5(severity) */, + "<13>1 1970-01-01T00:00:01.000000Z - - - - - hello world"}; + struct str_list expected = { + .size = sizeof(expected_strs)/sizeof(char*), + .lists = &expected_strs[0], + }; + + clear_output_num(); + + ctx = test_ctx_create(); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("test_ctx_create failed"); + exit(EXIT_FAILURE); + } + + ret = flb_output_set(ctx->flb, ctx->o_ffd, + "match", "*", + "syslog_format", "rfc5424", + "syslog_message_key", "msg", + "syslog_severity_preset", "5", + NULL); + TEST_CHECK(ret == 0); + + ret = flb_output_set_test(ctx->flb, ctx->o_ffd, + "formatter", cb_check_str_list, + &expected, NULL); + TEST_CHECK(ret == 0); + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + ret = flb_lib_push(ctx->flb, ctx->i_ffd, (char *) buf, size); + TEST_CHECK(ret >= 0); + + /* waiting to flush */ + flb_time_msleep(500); + + num = get_output_num(); + if (!TEST_CHECK(num > 0)) { + TEST_MSG("no outputs"); + } + + test_ctx_destroy(ctx); +} + +void flb_test_severity_key_rfc3164() +{ + struct test_ctx *ctx; + int ret; + int num; + + char *buf = "[1, {\"msg\":\"hello world\", \"s_key\":\"5\"}]"; + size_t size = strlen(buf); + + char *expected_strs[] = {"hello world", "Jan 1 00:00:01", "<13>" /* 1(user-level messages) * 8 + 5(severity) */, + "<13>Jan 1 00:00:01 hello world"}; + struct str_list expected = { + .size = sizeof(expected_strs)/sizeof(char*), + .lists = &expected_strs[0], + }; + + clear_output_num(); + + ctx = test_ctx_create(); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("test_ctx_create failed"); + exit(EXIT_FAILURE); + } + + ret = flb_output_set(ctx->flb, ctx->o_ffd, + "match", "*", + "syslog_format", "rfc3164", + "syslog_message_key", "msg", + "syslog_severity_key", "s_key", + NULL); + TEST_CHECK(ret == 0); + + ret = flb_output_set_test(ctx->flb, ctx->o_ffd, + "formatter", cb_check_str_list, + &expected, NULL); + TEST_CHECK(ret == 0); + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + ret = flb_lib_push(ctx->flb, ctx->i_ffd, (char *) buf, size); + TEST_CHECK(ret >= 0); + + /* waiting to flush */ + flb_time_msleep(500); + + num = get_output_num(); + if (!TEST_CHECK(num > 0)) { + TEST_MSG("no outputs"); + } + + test_ctx_destroy(ctx); +} + +void flb_test_severity_preset_rfc3164() +{ + struct test_ctx *ctx; + int ret; + int num; + + char *buf = "[1, {\"msg\":\"hello world\", \"s_key\":\"5\"}]"; + size_t size = strlen(buf); + + char *expected_strs[] = {"hello world", "Jan 1 00:00:01", "<13>" /* 1(user-level messages) * 8 + 5(severity) */, + "<13>Jan 1 00:00:01 hello world"}; + struct str_list expected = { + .size = sizeof(expected_strs)/sizeof(char*), + .lists = &expected_strs[0], + }; + + clear_output_num(); + + ctx = test_ctx_create(); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("test_ctx_create failed"); + exit(EXIT_FAILURE); + } + + ret = flb_output_set(ctx->flb, ctx->o_ffd, + "match", "*", + "syslog_format", "rfc3164", + "syslog_message_key", "msg", + "syslog_severity_preset", "5", + NULL); + TEST_CHECK(ret == 0); + + ret = flb_output_set_test(ctx->flb, ctx->o_ffd, + "formatter", cb_check_str_list, + &expected, NULL); + TEST_CHECK(ret == 0); + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + ret = flb_lib_push(ctx->flb, ctx->i_ffd, (char *) buf, size); + TEST_CHECK(ret >= 0); + + /* waiting to flush */ + flb_time_msleep(500); + + num = get_output_num(); + if (!TEST_CHECK(num > 0)) { + TEST_MSG("no outputs"); + } + + test_ctx_destroy(ctx); +} + +void flb_test_facility_key_rfc5424() +{ + struct test_ctx *ctx; + int ret; + int num; + + char *buf = "[1, {\"msg\":\"hello world\", \"f_key\":\"13\"}]"; + size_t size = strlen(buf); + + char *expected_strs[] = {"hello world", "1970-01-01T00:00:01.000000Z", "<110>" /* 13(log audit) * 8 + 6(default severity) */, + "<110>1 1970-01-01T00:00:01.000000Z - - - - - hello world"}; + struct str_list expected = { + .size = sizeof(expected_strs)/sizeof(char*), + .lists = &expected_strs[0], + }; + + clear_output_num(); + + ctx = test_ctx_create(); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("test_ctx_create failed"); + exit(EXIT_FAILURE); + } + + ret = flb_output_set(ctx->flb, ctx->o_ffd, + "match", "*", + "syslog_format", "rfc5424", + "syslog_message_key", "msg", + "syslog_facility_key", "f_key", + NULL); + TEST_CHECK(ret == 0); + + ret = flb_output_set_test(ctx->flb, ctx->o_ffd, + "formatter", cb_check_str_list, + &expected, NULL); + TEST_CHECK(ret == 0); + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + ret = flb_lib_push(ctx->flb, ctx->i_ffd, (char *) buf, size); + TEST_CHECK(ret >= 0); + + /* waiting to flush */ + flb_time_msleep(500); + + num = get_output_num(); + if (!TEST_CHECK(num > 0)) { + TEST_MSG("no outputs"); + } + + test_ctx_destroy(ctx); +} + +void flb_test_facility_preset_rfc5424() +{ + struct test_ctx *ctx; + int ret; + int num; + + char *buf = "[1, {\"msg\":\"hello world\", \"f_key\":\"13\"}]"; + size_t size = strlen(buf); + + char *expected_strs[] = {"hello world", "1970-01-01T00:00:01.000000Z", "<110>" /* 13(log audit) * 8 + 6(default severity) */, + "<110>1 1970-01-01T00:00:01.000000Z - - - - - hello world"}; + struct str_list expected = { + .size = sizeof(expected_strs)/sizeof(char*), + .lists = &expected_strs[0], + }; + + clear_output_num(); + + ctx = test_ctx_create(); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("test_ctx_create failed"); + exit(EXIT_FAILURE); + } + + ret = flb_output_set(ctx->flb, ctx->o_ffd, + "match", "*", + "syslog_format", "rfc5424", + "syslog_message_key", "msg", + "syslog_facility_preset", "13", + NULL); + TEST_CHECK(ret == 0); + + ret = flb_output_set_test(ctx->flb, ctx->o_ffd, + "formatter", cb_check_str_list, + &expected, NULL); + TEST_CHECK(ret == 0); + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + ret = flb_lib_push(ctx->flb, ctx->i_ffd, (char *) buf, size); + TEST_CHECK(ret >= 0); + + /* waiting to flush */ + flb_time_msleep(500); + + num = get_output_num(); + if (!TEST_CHECK(num > 0)) { + TEST_MSG("no outputs"); + } + + test_ctx_destroy(ctx); +} + +void flb_test_facility_key_rfc3164() +{ + struct test_ctx *ctx; + int ret; + int num; + + char *buf = "[1, {\"msg\":\"hello world\", \"f_key\":\"13\"}]"; + size_t size = strlen(buf); + + char *expected_strs[] = {"hello world", "Jan 1 00:00:01", "<110>" /* 13(log audit) * 8 + 6(default severity) */, + "<110>Jan 1 00:00:01 hello world"}; + struct str_list expected = { + .size = sizeof(expected_strs)/sizeof(char*), + .lists = &expected_strs[0], + }; + + clear_output_num(); + + ctx = test_ctx_create(); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("test_ctx_create failed"); + exit(EXIT_FAILURE); + } + + ret = flb_output_set(ctx->flb, ctx->o_ffd, + "match", "*", + "syslog_format", "rfc3164", + "syslog_message_key", "msg", + "syslog_facility_key", "f_key", + NULL); + TEST_CHECK(ret == 0); + + ret = flb_output_set_test(ctx->flb, ctx->o_ffd, + "formatter", cb_check_str_list, + &expected, NULL); + TEST_CHECK(ret == 0); + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + ret = flb_lib_push(ctx->flb, ctx->i_ffd, (char *) buf, size); + TEST_CHECK(ret >= 0); + + /* waiting to flush */ + flb_time_msleep(500); + + num = get_output_num(); + if (!TEST_CHECK(num > 0)) { + TEST_MSG("no outputs"); + } + + test_ctx_destroy(ctx); +} + +void flb_test_facility_preset_rfc3164() +{ + struct test_ctx *ctx; + int ret; + int num; + + char *buf = "[1, {\"msg\":\"hello world\", \"f_key\":\"13\"}]"; + size_t size = strlen(buf); + + char *expected_strs[] = {"hello world", "Jan 1 00:00:01", "<110>" /* 13(log audit) * 8 + 6(default severity) */, + "<110>Jan 1 00:00:01 hello world"}; + struct str_list expected = { + .size = sizeof(expected_strs)/sizeof(char*), + .lists = &expected_strs[0], + }; + + clear_output_num(); + + ctx = test_ctx_create(); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("test_ctx_create failed"); + exit(EXIT_FAILURE); + } + + ret = flb_output_set(ctx->flb, ctx->o_ffd, + "match", "*", + "syslog_format", "rfc3164", + "syslog_message_key", "msg", + "syslog_facility_preset", "13", + NULL); + TEST_CHECK(ret == 0); + + ret = flb_output_set_test(ctx->flb, ctx->o_ffd, + "formatter", cb_check_str_list, + &expected, NULL); + TEST_CHECK(ret == 0); + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + ret = flb_lib_push(ctx->flb, ctx->i_ffd, (char *) buf, size); + TEST_CHECK(ret >= 0); + + /* waiting to flush */ + flb_time_msleep(500); + + num = get_output_num(); + if (!TEST_CHECK(num > 0)) { + TEST_MSG("no outputs"); + } + + test_ctx_destroy(ctx); +} + +void flb_test_severity_facility_key_rfc5424() +{ + struct test_ctx *ctx; + int ret; + int num; + + char *buf = "[1, {\"msg\":\"hello world\", \"f_key\":\"13\", \"s_key\":\"5\"}]"; + size_t size = strlen(buf); + + char *expected_strs[] = {"hello world", "1970-01-01T00:00:01.000000Z", "<109>" /* 13(log audit) * 8 + 5(severity) */, + "<109>1 1970-01-01T00:00:01.000000Z - - - - - hello world"}; + struct str_list expected = { + .size = sizeof(expected_strs)/sizeof(char*), + .lists = &expected_strs[0], + }; + + clear_output_num(); + + ctx = test_ctx_create(); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("test_ctx_create failed"); + exit(EXIT_FAILURE); + } + + ret = flb_output_set(ctx->flb, ctx->o_ffd, + "match", "*", + "syslog_format", "rfc5424", + "syslog_message_key", "msg", + "syslog_severity_key", "s_key", + "syslog_facility_key", "f_key", + NULL); + TEST_CHECK(ret == 0); + + ret = flb_output_set_test(ctx->flb, ctx->o_ffd, + "formatter", cb_check_str_list, + &expected, NULL); + TEST_CHECK(ret == 0); + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + ret = flb_lib_push(ctx->flb, ctx->i_ffd, (char *) buf, size); + TEST_CHECK(ret >= 0); + + /* waiting to flush */ + flb_time_msleep(500); + + num = get_output_num(); + if (!TEST_CHECK(num > 0)) { + TEST_MSG("no outputs"); + } + + test_ctx_destroy(ctx); +} + +void flb_test_severity_facility_key_rfc3164() +{ + struct test_ctx *ctx; + int ret; + int num; + + char *buf = "[1, {\"msg\":\"hello world\", \"f_key\":\"13\", \"s_key\":\"5\"}]"; + size_t size = strlen(buf); + + char *expected_strs[] = {"hello world", "Jan 1 00:00:01", "<109>" /* 13(log audit) * 8 + 5(severity) */, + "<109>Jan 1 00:00:01 hello world"}; + struct str_list expected = { + .size = sizeof(expected_strs)/sizeof(char*), + .lists = &expected_strs[0], + }; + + clear_output_num(); + + ctx = test_ctx_create(); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("test_ctx_create failed"); + exit(EXIT_FAILURE); + } + + ret = flb_output_set(ctx->flb, ctx->o_ffd, + "match", "*", + "syslog_format", "rfc3164", + "syslog_message_key", "msg", + "syslog_severity_key", "s_key", + "syslog_facility_key", "f_key", + NULL); + TEST_CHECK(ret == 0); + + ret = flb_output_set_test(ctx->flb, ctx->o_ffd, + "formatter", cb_check_str_list, + &expected, NULL); + TEST_CHECK(ret == 0); + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + ret = flb_lib_push(ctx->flb, ctx->i_ffd, (char *) buf, size); + TEST_CHECK(ret >= 0); + + /* waiting to flush */ + flb_time_msleep(500); + + num = get_output_num(); + if (!TEST_CHECK(num > 0)) { + TEST_MSG("no outputs"); + } + + test_ctx_destroy(ctx); +} + +void flb_test_hostname_key_rfc5424() +{ + struct test_ctx *ctx; + int ret; + int num; + + char *buf = "[1, {\"msg\":\"hello world\", \"h_key\":\"localhost\"}]"; + size_t size = strlen(buf); + + char *expected_strs[] = {"hello world", "1970-01-01T00:00:01.000000Z", "localhost", + "<14>1 1970-01-01T00:00:01.000000Z localhost - - - - hello world"}; + struct str_list expected = { + .size = sizeof(expected_strs)/sizeof(char*), + .lists = &expected_strs[0], + }; + + clear_output_num(); + + ctx = test_ctx_create(); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("test_ctx_create failed"); + exit(EXIT_FAILURE); + } + + ret = flb_output_set(ctx->flb, ctx->o_ffd, + "match", "*", + "syslog_format", "rfc5424", + "syslog_message_key", "msg", + "syslog_hostname_key", "h_key", + NULL); + TEST_CHECK(ret == 0); + + ret = flb_output_set_test(ctx->flb, ctx->o_ffd, + "formatter", cb_check_str_list, + &expected, NULL); + TEST_CHECK(ret == 0); + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + ret = flb_lib_push(ctx->flb, ctx->i_ffd, (char *) buf, size); + TEST_CHECK(ret >= 0); + + /* waiting to flush */ + flb_time_msleep(500); + + num = get_output_num(); + if (!TEST_CHECK(num > 0)) { + TEST_MSG("no outputs"); + } + + test_ctx_destroy(ctx); +} + +void flb_test_hostname_preset_rfc5424() +{ + struct test_ctx *ctx; + int ret; + int num; + + char *buf = "[1, {\"msg\":\"hello world\", \"h_key\":\"localhost\"}]"; + size_t size = strlen(buf); + + char *expected_strs[] = {"hello world", "1970-01-01T00:00:01.000000Z", "localhost", + "<14>1 1970-01-01T00:00:01.000000Z localhost - - - - hello world"}; + struct str_list expected = { + .size = sizeof(expected_strs)/sizeof(char*), + .lists = &expected_strs[0], + }; + + clear_output_num(); + + ctx = test_ctx_create(); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("test_ctx_create failed"); + exit(EXIT_FAILURE); + } + + ret = flb_output_set(ctx->flb, ctx->o_ffd, + "match", "*", + "syslog_format", "rfc5424", + "syslog_message_key", "msg", + "syslog_hostname_preset", "localhost", + NULL); + TEST_CHECK(ret == 0); + + ret = flb_output_set_test(ctx->flb, ctx->o_ffd, + "formatter", cb_check_str_list, + &expected, NULL); + TEST_CHECK(ret == 0); + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + ret = flb_lib_push(ctx->flb, ctx->i_ffd, (char *) buf, size); + TEST_CHECK(ret >= 0); + + /* waiting to flush */ + flb_time_msleep(500); + + num = get_output_num(); + if (!TEST_CHECK(num > 0)) { + TEST_MSG("no outputs"); + } + + test_ctx_destroy(ctx); +} + +void flb_test_hostname_key_rfc3164() +{ + struct test_ctx *ctx; + int ret; + int num; + + char *buf = "[1, {\"msg\":\"hello world\", \"h_key\":\"localhost\"}]"; + size_t size = strlen(buf); + + char *expected_strs[] = {"hello world", "Jan 1 00:00:01", "localhost", + "<14>Jan 1 00:00:01 localhost hello world"}; + struct str_list expected = { + .size = sizeof(expected_strs)/sizeof(char*), + .lists = &expected_strs[0], + }; + + clear_output_num(); + + ctx = test_ctx_create(); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("test_ctx_create failed"); + exit(EXIT_FAILURE); + } + + ret = flb_output_set(ctx->flb, ctx->o_ffd, + "match", "*", + "syslog_format", "rfc3164", + "syslog_message_key", "msg", + "syslog_hostname_key", "h_key", + NULL); + TEST_CHECK(ret == 0); + + ret = flb_output_set_test(ctx->flb, ctx->o_ffd, + "formatter", cb_check_str_list, + &expected, NULL); + TEST_CHECK(ret == 0); + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + ret = flb_lib_push(ctx->flb, ctx->i_ffd, (char *) buf, size); + TEST_CHECK(ret >= 0); + + /* waiting to flush */ + flb_time_msleep(500); + + num = get_output_num(); + if (!TEST_CHECK(num > 0)) { + TEST_MSG("no outputs"); + } + + test_ctx_destroy(ctx); +} + +void flb_test_hostname_preset_rfc3164() +{ + struct test_ctx *ctx; + int ret; + int num; + + char *buf = "[1, {\"msg\":\"hello world\", \"h_key\":\"localhost\"}]"; + size_t size = strlen(buf); + + char *expected_strs[] = {"hello world", "Jan 1 00:00:01", "localhost", + "<14>Jan 1 00:00:01 localhost hello world"}; + struct str_list expected = { + .size = sizeof(expected_strs)/sizeof(char*), + .lists = &expected_strs[0], + }; + + clear_output_num(); + + ctx = test_ctx_create(); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("test_ctx_create failed"); + exit(EXIT_FAILURE); + } + + ret = flb_output_set(ctx->flb, ctx->o_ffd, + "match", "*", + "syslog_format", "rfc3164", + "syslog_message_key", "msg", + "syslog_hostname_preset", "localhost", + NULL); + TEST_CHECK(ret == 0); + + ret = flb_output_set_test(ctx->flb, ctx->o_ffd, + "formatter", cb_check_str_list, + &expected, NULL); + TEST_CHECK(ret == 0); + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + ret = flb_lib_push(ctx->flb, ctx->i_ffd, (char *) buf, size); + TEST_CHECK(ret >= 0); + + /* waiting to flush */ + flb_time_msleep(500); + + num = get_output_num(); + if (!TEST_CHECK(num > 0)) { + TEST_MSG("no outputs"); + } + + test_ctx_destroy(ctx); +} + +void flb_test_appname_key_rfc5424() +{ + struct test_ctx *ctx; + int ret; + int num; + + char *buf = "[1, {\"msg\":\"hello world\", \"a_key\":\"fluent-bit\"}]"; + size_t size = strlen(buf); + + char *expected_strs[] = {"hello world", "1970-01-01T00:00:01.000000Z", "fluent-bit", + "<14>1 1970-01-01T00:00:01.000000Z - fluent-bit - - - hello world"}; + struct str_list expected = { + .size = sizeof(expected_strs)/sizeof(char*), + .lists = &expected_strs[0], + }; + + clear_output_num(); + + ctx = test_ctx_create(); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("test_ctx_create failed"); + exit(EXIT_FAILURE); + } + + ret = flb_output_set(ctx->flb, ctx->o_ffd, + "match", "*", + "syslog_format", "rfc5424", + "syslog_message_key", "msg", + "syslog_appname_key", "a_key", + NULL); + TEST_CHECK(ret == 0); + + ret = flb_output_set_test(ctx->flb, ctx->o_ffd, + "formatter", cb_check_str_list, + &expected, NULL); + TEST_CHECK(ret == 0); + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + ret = flb_lib_push(ctx->flb, ctx->i_ffd, (char *) buf, size); + TEST_CHECK(ret >= 0); + + /* waiting to flush */ + flb_time_msleep(500); + + num = get_output_num(); + if (!TEST_CHECK(num > 0)) { + TEST_MSG("no outputs"); + } + + test_ctx_destroy(ctx); +} + +void flb_test_appname_preset_rfc5424() +{ + struct test_ctx *ctx; + int ret; + int num; + + char *buf = "[1, {\"msg\":\"hello world\", \"a_key\":\"fluent-bit\"}]"; + size_t size = strlen(buf); + + char *expected_strs[] = {"hello world", "1970-01-01T00:00:01.000000Z", "fluent-bit", + "<14>1 1970-01-01T00:00:01.000000Z - fluent-bit - - - hello world"}; + struct str_list expected = { + .size = sizeof(expected_strs)/sizeof(char*), + .lists = &expected_strs[0], + }; + + clear_output_num(); + + ctx = test_ctx_create(); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("test_ctx_create failed"); + exit(EXIT_FAILURE); + } + + ret = flb_output_set(ctx->flb, ctx->o_ffd, + "match", "*", + "syslog_format", "rfc5424", + "syslog_message_key", "msg", + "syslog_appname_preset", "fluent-bit", + NULL); + TEST_CHECK(ret == 0); + + ret = flb_output_set_test(ctx->flb, ctx->o_ffd, + "formatter", cb_check_str_list, + &expected, NULL); + TEST_CHECK(ret == 0); + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + ret = flb_lib_push(ctx->flb, ctx->i_ffd, (char *) buf, size); + TEST_CHECK(ret >= 0); + + /* waiting to flush */ + flb_time_msleep(500); + + num = get_output_num(); + if (!TEST_CHECK(num > 0)) { + TEST_MSG("no outputs"); + } + + test_ctx_destroy(ctx); +} + +void flb_test_appname_key_rfc3164() +{ + struct test_ctx *ctx; + int ret; + int num; + + char *buf = "[1, {\"msg\":\"hello world\", \"a_key\":\"fluent-bit\"}]"; + size_t size = strlen(buf); + + char *expected_strs[] = {"hello world", "Jan 1 00:00:01", "fluent-bit", + "<14>Jan 1 00:00:01 fluent-bit: hello world"}; + struct str_list expected = { + .size = sizeof(expected_strs)/sizeof(char*), + .lists = &expected_strs[0], + }; + + clear_output_num(); + + ctx = test_ctx_create(); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("test_ctx_create failed"); + exit(EXIT_FAILURE); + } + + ret = flb_output_set(ctx->flb, ctx->o_ffd, + "match", "*", + "syslog_format", "rfc3164", + "syslog_message_key", "msg", + "syslog_appname_key", "a_key", + NULL); + TEST_CHECK(ret == 0); + + ret = flb_output_set_test(ctx->flb, ctx->o_ffd, + "formatter", cb_check_str_list, + &expected, NULL); + TEST_CHECK(ret == 0); + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + ret = flb_lib_push(ctx->flb, ctx->i_ffd, (char *) buf, size); + TEST_CHECK(ret >= 0); + + /* waiting to flush */ + flb_time_msleep(500); + + num = get_output_num(); + if (!TEST_CHECK(num > 0)) { + TEST_MSG("no outputs"); + } + + test_ctx_destroy(ctx); +} + +void flb_test_appname_preset_rfc3164() +{ + struct test_ctx *ctx; + int ret; + int num; + + char *buf = "[1, {\"msg\":\"hello world\", \"a_key\":\"fluent-bit\"}]"; + size_t size = strlen(buf); + + char *expected_strs[] = {"hello world", "Jan 1 00:00:01", "fluent-bit", + "<14>Jan 1 00:00:01 fluent-bit: hello world"}; + struct str_list expected = { + .size = sizeof(expected_strs)/sizeof(char*), + .lists = &expected_strs[0], + }; + + clear_output_num(); + + ctx = test_ctx_create(); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("test_ctx_create failed"); + exit(EXIT_FAILURE); + } + + ret = flb_output_set(ctx->flb, ctx->o_ffd, + "match", "*", + "syslog_format", "rfc3164", + "syslog_message_key", "msg", + "syslog_appname_preset", "fluent-bit", + NULL); + TEST_CHECK(ret == 0); + + ret = flb_output_set_test(ctx->flb, ctx->o_ffd, + "formatter", cb_check_str_list, + &expected, NULL); + TEST_CHECK(ret == 0); + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + ret = flb_lib_push(ctx->flb, ctx->i_ffd, (char *) buf, size); + TEST_CHECK(ret >= 0); + + /* waiting to flush */ + flb_time_msleep(500); + + num = get_output_num(); + if (!TEST_CHECK(num > 0)) { + TEST_MSG("no outputs"); + } + + test_ctx_destroy(ctx); +} + +void flb_test_procid_key_rfc5424() +{ + struct test_ctx *ctx; + int ret; + int num; + + char *buf = "[1, {\"msg\":\"hello world\", \"p_key\":\"1234\"}]"; + size_t size = strlen(buf); + + char *expected_strs[] = {"hello world", "1970-01-01T00:00:01.000000Z", "1234", + "<14>1 1970-01-01T00:00:01.000000Z - - 1234 - - hello world"}; + struct str_list expected = { + .size = sizeof(expected_strs)/sizeof(char*), + .lists = &expected_strs[0], + }; + + clear_output_num(); + + ctx = test_ctx_create(); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("test_ctx_create failed"); + exit(EXIT_FAILURE); + } + + ret = flb_output_set(ctx->flb, ctx->o_ffd, + "match", "*", + "syslog_format", "rfc5424", + "syslog_message_key", "msg", + "syslog_procid_key", "p_key", + NULL); + TEST_CHECK(ret == 0); + + ret = flb_output_set_test(ctx->flb, ctx->o_ffd, + "formatter", cb_check_str_list, + &expected, NULL); + TEST_CHECK(ret == 0); + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + ret = flb_lib_push(ctx->flb, ctx->i_ffd, (char *) buf, size); + TEST_CHECK(ret >= 0); + + /* waiting to flush */ + flb_time_msleep(500); + + num = get_output_num(); + if (!TEST_CHECK(num > 0)) { + TEST_MSG("no outputs"); + } + + test_ctx_destroy(ctx); +} + +void flb_test_procid_preset_rfc5424() +{ + struct test_ctx *ctx; + int ret; + int num; + + char *buf = "[1, {\"msg\":\"hello world\", \"p_key\":\"1234\"}]"; + size_t size = strlen(buf); + + char *expected_strs[] = {"hello world", "1970-01-01T00:00:01.000000Z", "1234", + "<14>1 1970-01-01T00:00:01.000000Z - - 1234 - - hello world"}; + struct str_list expected = { + .size = sizeof(expected_strs)/sizeof(char*), + .lists = &expected_strs[0], + }; + + clear_output_num(); + + ctx = test_ctx_create(); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("test_ctx_create failed"); + exit(EXIT_FAILURE); + } + + ret = flb_output_set(ctx->flb, ctx->o_ffd, + "match", "*", + "syslog_format", "rfc5424", + "syslog_message_key", "msg", + "syslog_procid_preset", "1234", + NULL); + TEST_CHECK(ret == 0); + + ret = flb_output_set_test(ctx->flb, ctx->o_ffd, + "formatter", cb_check_str_list, + &expected, NULL); + TEST_CHECK(ret == 0); + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + ret = flb_lib_push(ctx->flb, ctx->i_ffd, (char *) buf, size); + TEST_CHECK(ret >= 0); + + /* waiting to flush */ + flb_time_msleep(500); + + num = get_output_num(); + if (!TEST_CHECK(num > 0)) { + TEST_MSG("no outputs"); + } + + test_ctx_destroy(ctx); +} + +void flb_test_msgid_key_rfc5424() +{ + struct test_ctx *ctx; + int ret; + int num; + + char *buf = "[1, {\"msg\":\"hello world\", \"m_key\":\"TCPIN\"}]"; + size_t size = strlen(buf); + + char *expected_strs[] = {"hello world", "1970-01-01T00:00:01.000000Z", "TCPIN", + "<14>1 1970-01-01T00:00:01.000000Z - - - TCPIN - hello world"}; + struct str_list expected = { + .size = sizeof(expected_strs)/sizeof(char*), + .lists = &expected_strs[0], + }; + + clear_output_num(); + + ctx = test_ctx_create(); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("test_ctx_create failed"); + exit(EXIT_FAILURE); + } + + ret = flb_output_set(ctx->flb, ctx->o_ffd, + "match", "*", + "syslog_format", "rfc5424", + "syslog_message_key", "msg", + "syslog_msgid_key", "m_key", + NULL); + TEST_CHECK(ret == 0); + + ret = flb_output_set_test(ctx->flb, ctx->o_ffd, + "formatter", cb_check_str_list, + &expected, NULL); + TEST_CHECK(ret == 0); + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + ret = flb_lib_push(ctx->flb, ctx->i_ffd, (char *) buf, size); + TEST_CHECK(ret >= 0); + + /* waiting to flush */ + flb_time_msleep(500); + + num = get_output_num(); + if (!TEST_CHECK(num > 0)) { + TEST_MSG("no outputs"); + } + + test_ctx_destroy(ctx); +} + +void flb_test_msgid_preset_rfc5424() +{ + struct test_ctx *ctx; + int ret; + int num; + + char *buf = "[1, {\"msg\":\"hello world\", \"m_key\":\"TCPIN\"}]"; + size_t size = strlen(buf); + + char *expected_strs[] = {"hello world", "1970-01-01T00:00:01.000000Z", "TCPIN", + "<14>1 1970-01-01T00:00:01.000000Z - - - TCPIN - hello world"}; + struct str_list expected = { + .size = sizeof(expected_strs)/sizeof(char*), + .lists = &expected_strs[0], + }; + + clear_output_num(); + + ctx = test_ctx_create(); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("test_ctx_create failed"); + exit(EXIT_FAILURE); + } + + ret = flb_output_set(ctx->flb, ctx->o_ffd, + "match", "*", + "syslog_format", "rfc5424", + "syslog_message_key", "msg", + "syslog_msgid_preset", "TCPIN", + NULL); + TEST_CHECK(ret == 0); + + ret = flb_output_set_test(ctx->flb, ctx->o_ffd, + "formatter", cb_check_str_list, + &expected, NULL); + TEST_CHECK(ret == 0); + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + ret = flb_lib_push(ctx->flb, ctx->i_ffd, (char *) buf, size); + TEST_CHECK(ret >= 0); + + /* waiting to flush */ + flb_time_msleep(500); + + num = get_output_num(); + if (!TEST_CHECK(num > 0)) { + TEST_MSG("no outputs"); + } + + test_ctx_destroy(ctx); +} + +void flb_test_sd_key_rfc5424() +{ + struct test_ctx *ctx; + int ret; + int num; + + char *buf = "[1, {\"msg\":\"hello world\", \"sd_key\": {\"logtype\": \"access\",\"clustername\": \"mycluster\",\"namespace\": \"mynamespace\"}}]"; + size_t size = strlen(buf); + + char *expected_strs[] = {"hello world", "1970-01-01T00:00:01.000000Z", + "<14>1 1970-01-01T00:00:01.000000Z - - - - [sd_key logtype=\"access\" clustername=\"mycluster\" namespace=\"mynamespace\"] hello world"}; + struct str_list expected = { + .size = sizeof(expected_strs)/sizeof(char*), + .lists = &expected_strs[0], + }; + + clear_output_num(); + + ctx = test_ctx_create(); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("test_ctx_create failed"); + exit(EXIT_FAILURE); + } + + ret = flb_output_set(ctx->flb, ctx->o_ffd, + "match", "*", + "syslog_format", "rfc5424", + "syslog_message_key", "msg", + "syslog_sd_key", "sd_key", + NULL); + TEST_CHECK(ret == 0); + + ret = flb_output_set_test(ctx->flb, ctx->o_ffd, + "formatter", cb_check_str_list, + &expected, NULL); + TEST_CHECK(ret == 0); + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + ret = flb_lib_push(ctx->flb, ctx->i_ffd, (char *) buf, size); + TEST_CHECK(ret >= 0); + + /* waiting to flush */ + flb_time_msleep(500); + + num = get_output_num(); + if (!TEST_CHECK(num > 0)) { + TEST_MSG("no outputs"); + } + + test_ctx_destroy(ctx); +} + +void flb_test_allow_longer_sd_id_rfc5424() +{ + struct test_ctx *ctx; + int ret; + int num; + + char *buf = "[1, {\"msg\":\"hello world\", \"sd_key_that_is_longer_than_32_characters\": {\"logtype_that_is_longer_than_32_characters\": \"access\",\"clustername\": \"mycluster\",\"namespace\": \"mynamespace\"}}]"; + size_t size = strlen(buf); + + char *expected_strs[] = {"hello world", "1970-01-01T00:00:01.000000Z", + "<14>1 1970-01-01T00:00:01.000000Z - - - - [sd_key_that_is_longer_than_32_characters logtype_that_is_longer_than_32_characters=\"access\" clustername=\"mycluster\" namespace=\"mynamespace\"] hello world"}; + struct str_list expected = { + .size = sizeof(expected_strs)/sizeof(char*), + .lists = &expected_strs[0], + }; + + clear_output_num(); + + ctx = test_ctx_create(); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("test_ctx_create failed"); + exit(EXIT_FAILURE); + } + + ret = flb_output_set(ctx->flb, ctx->o_ffd, + "match", "*", + "syslog_format", "rfc5424", + "syslog_message_key", "msg", + "syslog_sd_key", "sd_key_that_is_longer_than_32_characters", + "allow_longer_sd_id", "true", + NULL); + TEST_CHECK(ret == 0); + + ret = flb_output_set_test(ctx->flb, ctx->o_ffd, + "formatter", cb_check_str_list, + &expected, NULL); + TEST_CHECK(ret == 0); + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + ret = flb_lib_push(ctx->flb, ctx->i_ffd, (char *) buf, size); + TEST_CHECK(ret >= 0); + + /* waiting to flush */ + flb_time_msleep(500); + + num = get_output_num(); + if (!TEST_CHECK(num > 0)) { + TEST_MSG("no outputs"); + } + + test_ctx_destroy(ctx); +} + +void flb_test_malformed_longer_sd_id_rfc5424() +{ + struct test_ctx *ctx; + int ret; + int num; + + char *buf = "[1, {\"msg\":\"hello world\", \"sd_key_that_is_longer_than_32_characters\": {\"logtype_that_is_longer_than_32_characters\": \"access\",\"clustername\": \"mycluster\",\"namespace\": \"mynamespace\"}}]"; + size_t size = strlen(buf); + + char *expected_strs[] = {"hello world", "1970-01-01T00:00:01.000000Z", + "<14>1 1970-01-01T00:00:01.000000Z - - - - [sd_key_that_is_longer_than_32_ch logtype_that_is_longer_than_32_c=\"access\" clustername=\"mycluster\" namespace=\"mynamespace\"] hello world"}; + struct str_list expected = { + .size = sizeof(expected_strs)/sizeof(char*), + .lists = &expected_strs[0], + }; + + clear_output_num(); + + ctx = test_ctx_create(); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("test_ctx_create failed"); + exit(EXIT_FAILURE); + } + + ret = flb_output_set(ctx->flb, ctx->o_ffd, + "match", "*", + "syslog_format", "rfc5424", + "syslog_message_key", "msg", + "syslog_sd_key", "sd_key_that_is_longer_than_32_characters", + "allow_longer_sd_id", "false", + NULL); + TEST_CHECK(ret == 0); + + ret = flb_output_set_test(ctx->flb, ctx->o_ffd, + "formatter", cb_check_str_list, + &expected, NULL); + TEST_CHECK(ret == 0); + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + ret = flb_lib_push(ctx->flb, ctx->i_ffd, (char *) buf, size); + TEST_CHECK(ret >= 0); + + /* waiting to flush */ + flb_time_msleep(500); + + num = get_output_num(); + if (!TEST_CHECK(num > 0)) { + TEST_MSG("no outputs"); + } + + test_ctx_destroy(ctx); +} + +TEST_LIST = { + /* rfc3164 */ + /* procid_key, msgid_key, sd_key are not supported */ + {"format_severity_key_rfc3164", flb_test_severity_key_rfc3164}, + {"format_facility_key_rfc3164", flb_test_facility_key_rfc3164}, + {"format_severity_facility_key_rfc3164", flb_test_severity_facility_key_rfc3164}, + {"format_hostname_key_rfc3164", flb_test_hostname_key_rfc3164}, + {"format_appname_key_rfc3164", flb_test_appname_key_rfc3164}, + {"format_severity_preset_rfc3164", flb_test_severity_preset_rfc3164}, + {"format_facility_preset_rfc3164", flb_test_facility_preset_rfc5424}, + {"format_hostname_preset_rfc3164", flb_test_hostname_preset_rfc5424}, + {"format_appname_preset_rfc3164", flb_test_appname_preset_rfc3164}, + + /* rfc5424 (Default) */ + {"format_syslog_rfc5424", flb_test_syslog_rfc5424}, + {"format_severity_key_rfc5424", flb_test_severity_key_rfc5424}, + {"format_facility_key_rfc5424", flb_test_facility_key_rfc5424}, + {"format_severity_facility_key_rfc5424", flb_test_severity_facility_key_rfc5424}, + {"format_hostname_key_rfc5424", flb_test_hostname_key_rfc5424}, + {"format_appname_key_rfc5424", flb_test_appname_key_rfc5424}, + {"format_procid_key_rfc5424", flb_test_procid_key_rfc5424}, + {"format_msgid_key_rfc5424", flb_test_msgid_key_rfc5424}, + {"format_sd_key_rfc5424", flb_test_sd_key_rfc5424}, + {"format_severity_preset_rfc5424", flb_test_severity_preset_rfc5424}, + {"format_facility_preset_rfc5424", flb_test_facility_preset_rfc5424}, + {"format_hostname_preset_rfc5424", flb_test_hostname_preset_rfc5424}, + {"format_appname_preset_rfc5424", flb_test_appname_preset_rfc5424}, + {"format_procid_preset_rfc5424", flb_test_procid_preset_rfc5424}, + {"format_msgid_preset_rfc5424", flb_test_msgid_preset_rfc5424}, + {"allow_longer_sd_id_rfc5424", flb_test_allow_longer_sd_id_rfc5424}, + {"malformed_longer_sd_id_rfc5424", flb_test_malformed_longer_sd_id_rfc5424}, + {NULL, NULL} +}; + diff --git a/src/fluent-bit/tests/runtime/out_tcp.c b/src/fluent-bit/tests/runtime/out_tcp.c new file mode 100644 index 000000000..98f323ba3 --- /dev/null +++ b/src/fluent-bit/tests/runtime/out_tcp.c @@ -0,0 +1,1004 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2019-2022 The Fluent Bit Authors + * Copyright (C) 2015-2018 Treasure Data Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <fluent-bit.h> +#include <fluent-bit/flb_sds.h> +#include <fluent-bit/flb_time.h> +#include <fluent-bit/flb_stream.h> +#include <fluent-bit/flb_upstream.h> +#include <fluent-bit/flb_downstream.h> +#include <string.h> +#include <float.h> +#include <math.h> +#include <msgpack.h> +#include "flb_tests_runtime.h" + +#define DEFAULT_IO_TIMEOUT 10 +#define DEFAULT_HOST "127.0.0.1" +#define DEFAULT_PORT "5170" + +#define TLS_CERTIFICATE_HOSTNAME "leo.vcap.me" +#define TLS_CERTIFICATE_FILENAME FLB_TESTS_DATA_PATH "/data/tls/certificate.pem" +#define TLS_PRIVATE_KEY_FILENAME FLB_TESTS_DATA_PATH "/data/tls/private_key.pem" + +struct test_ctx { + flb_ctx_t *flb; /* Fluent Bit library context */ + int i_ffd; /* Input fd */ + int f_ffd; /* Filter fd (unused) */ + int o_ffd; /* Output fd */ +}; + + +pthread_mutex_t result_mutex = PTHREAD_MUTEX_INITIALIZER; +int num_output = 0; +static int get_output_num() +{ + int ret; + pthread_mutex_lock(&result_mutex); + ret = num_output; + pthread_mutex_unlock(&result_mutex); + + return ret; +} + +static void set_output_num(int num) +{ + pthread_mutex_lock(&result_mutex); + num_output = num; + pthread_mutex_unlock(&result_mutex); +} + +static void clear_output_num() +{ + set_output_num(0); +} + +struct str_list { + size_t size; + char **lists; +}; + +/* Callback to check expected results */ +static void cb_check_str_list(void *ctx, int ffd, int res_ret, + void *res_data, size_t res_size, void *data) +{ + char *p; + flb_sds_t out_line = res_data; + int num = get_output_num(); + size_t i; + struct str_list *l = (struct str_list *)data; + + if (!TEST_CHECK(l != NULL)) { + TEST_MSG("l is NULL"); + flb_sds_destroy(out_line); + return; + } + + if(!TEST_CHECK(res_ret == 0)) { + TEST_MSG("callback ret=%d", res_ret); + } + if (!TEST_CHECK(res_data != NULL)) { + TEST_MSG("res_data is NULL"); + flb_sds_destroy(out_line); + return; + } + + for (i=0; i<l->size; i++) { + p = strstr(out_line, l->lists[i]); + if (!TEST_CHECK(p != NULL)) { + TEST_MSG(" Got :%s\n expect:%s", out_line, l->lists[i]); + } + } + set_output_num(num+1); + + flb_sds_destroy(out_line); +} + +static int msgpack_strncmp(char* str, size_t str_len, msgpack_object obj) +{ + int ret = -1; + + if (str == NULL) { + flb_error("str is NULL"); + return -1; + } + + switch (obj.type) { + case MSGPACK_OBJECT_STR: + if (obj.via.str.size != str_len) { + return -1; + } + ret = strncmp(str, obj.via.str.ptr, str_len); + break; + case MSGPACK_OBJECT_POSITIVE_INTEGER: + { + unsigned long val = strtoul(str, NULL, 10); + if (val == (unsigned long)obj.via.u64) { + ret = 0; + } + } + break; + case MSGPACK_OBJECT_NEGATIVE_INTEGER: + { + long long val = strtoll(str, NULL, 10); + if (val == (unsigned long)obj.via.i64) { + ret = 0; + } + } + break; + case MSGPACK_OBJECT_FLOAT32: + case MSGPACK_OBJECT_FLOAT64: + { + double val = strtod(str, NULL); + if (fabs(val - obj.via.f64) < DBL_EPSILON) { + ret = 0; + } + } + break; + case MSGPACK_OBJECT_BOOLEAN: + if (obj.via.boolean) { + if (str_len != 4 /*true*/) { + return -1; + } + ret = strncasecmp(str, "true", 4); + } + else { + if (str_len != 5 /*false*/) { + return -1; + } + ret = strncasecmp(str, "false", 5); + } + break; + default: + flb_error("not supported"); + } + + return ret; +} + +/* Callback to check expected results */ +static void cb_check_msgpack_kv(void *ctx, int ffd, int res_ret, + void *res_data, size_t res_size, void *data) +{ + msgpack_unpacked result; + msgpack_object obj; + size_t off = 0; + struct str_list *l = (struct str_list *)data; + int i_map; + int map_size; + int i_list; + int num = get_output_num(); + + if (!TEST_CHECK(res_data != NULL)) { + TEST_MSG("res_data is NULL"); + return; + } + + if (!TEST_CHECK(data != NULL)) { + flb_error("data is NULL"); + return; + } + + /* Iterate each item array and apply rules */ + msgpack_unpacked_init(&result); + while (msgpack_unpack_next(&result, res_data, res_size, &off) == MSGPACK_UNPACK_SUCCESS) { + obj = result.data; + /* + msgpack_object_print(stdout, obj); + */ + if (obj.type != MSGPACK_OBJECT_ARRAY || obj.via.array.size != 2) { + flb_error("array error. type = %d", obj.type); + continue; + } + obj = obj.via.array.ptr[1]; + if (obj.type != MSGPACK_OBJECT_MAP) { + flb_error("map error. type = %d", obj.type); + continue; + } + map_size = obj.via.map.size; + for (i_map=0; i_map<map_size; i_map++) { + if (obj.via.map.ptr[i_map].key.type != MSGPACK_OBJECT_STR) { + flb_error("key is not string. type =%d", obj.via.map.ptr[i_map].key.type); + continue; + } + for (i_list=0; i_list< l->size/2; i_list++) { + if (msgpack_strncmp(l->lists[i_list*2], strlen(l->lists[i_list*2]), + obj.via.map.ptr[i_map].key) == 0 && + msgpack_strncmp(l->lists[i_list*2+1], strlen(l->lists[i_list*2+1]), + obj.via.map.ptr[i_map].val) == 0) { + num++; + } + } + } + } + set_output_num(num); + + msgpack_unpacked_destroy(&result); + + return ; +} + +static struct test_ctx *test_ctx_create() +{ + int i_ffd; + int o_ffd; + struct test_ctx *ctx = NULL; + + ctx = flb_malloc(sizeof(struct test_ctx)); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("malloc failed"); + flb_errno(); + return NULL; + } + + /* Service config */ + ctx->flb = flb_create(); + flb_service_set(ctx->flb, + "Flush", "0.200000000", + "Grace", "1", + "Log_Level", "error", + NULL); + + /* Input */ + i_ffd = flb_input(ctx->flb, (char *) "lib", NULL); + TEST_CHECK(i_ffd >= 0); + ctx->i_ffd = i_ffd; + + /* Output */ + o_ffd = flb_output(ctx->flb, (char *) "tcp", NULL); + ctx->o_ffd = o_ffd; + + return ctx; +} + +static void test_ctx_destroy(struct test_ctx *ctx) +{ + TEST_CHECK(ctx != NULL); + + sleep(1); + flb_stop(ctx->flb); + flb_destroy(ctx->flb); + flb_free(ctx); +} + +void flb_test_tcp_with_tls() +{ + struct flb_net_setup downstream_net_setup; + struct flb_connection *client_connection; + size_t out_buf_size; + char in_buf[256]; + struct flb_downstream *downstream; + const char *out_buf; + unsigned short int port; + struct test_ctx *ctx; + int ret; + struct flb_tls *tls; + + port = strtoul(DEFAULT_PORT, NULL, 10); + + out_buf = "[1, {\"msg\":\"hello world\"}]"; + out_buf_size = strlen(out_buf); + + memset(in_buf, 0, sizeof(in_buf)); + + ctx = test_ctx_create(); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("test_ctx_create failed"); + exit(EXIT_FAILURE); + } + + ret = flb_output_set(ctx->flb, ctx->o_ffd, + "match", "*", + "format", "json", + "host", DEFAULT_HOST, + "port", DEFAULT_PORT, + "tls", "on", + "tls.verify", "no", + "tls.vhost", TLS_CERTIFICATE_HOSTNAME, + NULL); + TEST_CHECK(ret == 0); + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + ret = flb_tls_init(); + TEST_CHECK(ret == 0); + + tls = flb_tls_create(FLB_TLS_SERVER_MODE, + FLB_FALSE, + FLB_TRUE, + TLS_CERTIFICATE_HOSTNAME, + NULL, + NULL, + TLS_CERTIFICATE_FILENAME, + TLS_PRIVATE_KEY_FILENAME, + NULL); + + TEST_CHECK(tls != NULL); + + if (tls != NULL) { + /* Ingest data sample */ + ret = flb_lib_push(ctx->flb, ctx->i_ffd, (char *) out_buf, out_buf_size); + TEST_CHECK(ret >= 0); + + flb_net_setup_init(&downstream_net_setup); + + downstream_net_setup.io_timeout = DEFAULT_IO_TIMEOUT; + + downstream = flb_downstream_create(FLB_TRANSPORT_TCP, + FLB_IO_TCP | FLB_IO_TLS, + DEFAULT_HOST, + port, + tls, + ctx->flb->config, + &downstream_net_setup); + + TEST_CHECK(downstream != NULL); + + if (downstream != NULL) { + flb_stream_disable_async_mode(&downstream->base); + + flb_net_socket_blocking(downstream->server_fd); + + client_connection = flb_downstream_conn_get(downstream); + + TEST_CHECK(client_connection != NULL); + + ret = flb_io_net_read(client_connection, + (void *) in_buf, + sizeof(in_buf)); + + TEST_CHECK(ret > 0); + + if (ret > 0) { + TEST_CHECK(strstr(in_buf, "hello world") != NULL); + } + } + } + + sleep(1); + + flb_stop(ctx->flb); + flb_downstream_destroy(downstream); + flb_tls_destroy(tls); + flb_destroy(ctx->flb); + flb_free(ctx); +} + +void flb_test_format_msgpack() +{ + struct test_ctx *ctx; + int ret; + int num; + + char *buf1 = "[1, {\"msg\":\"hello world\", \"val\":1000, \"nval\":-10000, \"bool\":true, \"float\":1.234}]"; + size_t size1 = strlen(buf1); + + char *expected_strs[] = {"msg", "hello world", "val", "1000", "nval", "-10000", "bool", "true", "float", "1.234"}; + struct str_list expected = { + .size = sizeof(expected_strs)/sizeof(char*), + .lists = &expected_strs[0], + }; + + clear_output_num(); + + ctx = test_ctx_create(); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("test_ctx_create failed"); + exit(EXIT_FAILURE); + } + + ret = flb_output_set(ctx->flb, ctx->o_ffd, + "match", "*", + "format", "msgpack", + NULL); + TEST_CHECK(ret == 0); + + ret = flb_output_set_test(ctx->flb, ctx->o_ffd, + "formatter", cb_check_msgpack_kv, + &expected, NULL); + TEST_CHECK(ret == 0); + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + ret = flb_lib_push(ctx->flb, ctx->i_ffd, (char *) buf1, size1); + TEST_CHECK(ret >= 0); + + /* waiting to flush */ + flb_time_msleep(500); + + num = get_output_num(); + if (!TEST_CHECK(num == expected.size / 2)) { + TEST_MSG("got %d, expected %lu", num, expected.size/2); + } + + test_ctx_destroy(ctx); +} + +void flb_test_format_json() +{ + struct test_ctx *ctx; + int ret; + int num; + + char *buf1 = "[1, {\"msg\":\"hello world\"}]"; + size_t size1 = strlen(buf1); + char *buf2 = "[2, {\"msg\":\"hello world\"}]"; + size_t size2 = strlen(buf2); + + char *expected_strs[] = {"[{\"date\":1.0,\"msg\":\"hello world\"},{\"date\":2.0,\"msg\":\"hello world\"}]"}; + struct str_list expected = { + .size = sizeof(expected_strs)/sizeof(char*), + .lists = &expected_strs[0], + }; + + clear_output_num(); + + ctx = test_ctx_create(); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("test_ctx_create failed"); + exit(EXIT_FAILURE); + } + + ret = flb_output_set(ctx->flb, ctx->o_ffd, + "match", "*", + "format", "json", + NULL); + TEST_CHECK(ret == 0); + + ret = flb_output_set_test(ctx->flb, ctx->o_ffd, + "formatter", cb_check_str_list, + &expected, NULL); + TEST_CHECK(ret == 0); + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + ret = flb_lib_push(ctx->flb, ctx->i_ffd, (char *) buf1, size1); + TEST_CHECK(ret >= 0); + ret = flb_lib_push(ctx->flb, ctx->i_ffd, (char *) buf2, size2); + TEST_CHECK(ret >= 0); + + /* waiting to flush */ + flb_time_msleep(500); + + num = get_output_num(); + if (!TEST_CHECK(num > 0)) { + TEST_MSG("no outputs"); + } + + test_ctx_destroy(ctx); +} + +void flb_test_format_json_stream() +{ + struct test_ctx *ctx; + int ret; + int num; + + char *buf1 = "[1, {\"msg\":\"hello world\"}]"; + size_t size1 = strlen(buf1); + char *buf2 = "[2, {\"msg\":\"hello world\"}]"; + size_t size2 = strlen(buf2); + + char *expected_strs[] = {"{\"date\":1.0,\"msg\":\"hello world\"}{\"date\":2.0,\"msg\":\"hello world\"}"}; + struct str_list expected = { + .size = sizeof(expected_strs)/sizeof(char*), + .lists = &expected_strs[0], + }; + + clear_output_num(); + + ctx = test_ctx_create(); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("test_ctx_create failed"); + exit(EXIT_FAILURE); + } + + ret = flb_output_set(ctx->flb, ctx->o_ffd, + "match", "*", + "format", "json_stream", + NULL); + TEST_CHECK(ret == 0); + + ret = flb_output_set_test(ctx->flb, ctx->o_ffd, + "formatter", cb_check_str_list, + &expected, NULL); + TEST_CHECK(ret == 0); + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + ret = flb_lib_push(ctx->flb, ctx->i_ffd, (char *) buf1, size1); + TEST_CHECK(ret >= 0); + ret = flb_lib_push(ctx->flb, ctx->i_ffd, (char *) buf2, size2); + TEST_CHECK(ret >= 0); + + /* waiting to flush */ + flb_time_msleep(500); + + num = get_output_num(); + if (!TEST_CHECK(num > 0)) { + TEST_MSG("no outputs"); + } + + test_ctx_destroy(ctx); +} + +void flb_test_format_json_lines() +{ + struct test_ctx *ctx; + int ret; + int num; + + char *buf1 = "[1, {\"msg\":\"hello world\"}]"; + size_t size1 = strlen(buf1); + char *buf2 = "[2, {\"msg\":\"hello world\"}]"; + size_t size2 = strlen(buf2); + + char *expected_strs[] = {"{\"date\":1.0,\"msg\":\"hello world\"}\n{\"date\":2.0,\"msg\":\"hello world\"}"}; + struct str_list expected = { + .size = sizeof(expected_strs)/sizeof(char*), + .lists = &expected_strs[0], + }; + + clear_output_num(); + + ctx = test_ctx_create(); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("test_ctx_create failed"); + exit(EXIT_FAILURE); + } + + ret = flb_output_set(ctx->flb, ctx->o_ffd, + "match", "*", + "format", "json_lines", + NULL); + TEST_CHECK(ret == 0); + + ret = flb_output_set_test(ctx->flb, ctx->o_ffd, + "formatter", cb_check_str_list, + &expected, NULL); + TEST_CHECK(ret == 0); + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + ret = flb_lib_push(ctx->flb, ctx->i_ffd, (char *) buf1, size1); + TEST_CHECK(ret >= 0); + ret = flb_lib_push(ctx->flb, ctx->i_ffd, (char *) buf2, size2); + TEST_CHECK(ret >= 0); + + /* waiting to flush */ + flb_time_msleep(500); + + num = get_output_num(); + if (!TEST_CHECK(num > 0)) { + TEST_MSG("no outputs"); + } + + test_ctx_destroy(ctx); +} + +void flb_test_set_json_date_key() +{ + struct test_ctx *ctx; + int ret; + int num; + + char *buf1 = "[1, {\"msg\":\"hello world\"}]"; + size_t size1 = strlen(buf1); + + char *expected_strs[] = {"{\"timestamp\":1.0,\"msg\":\"hello world\"}"}; + struct str_list expected = { + .size = sizeof(expected_strs)/sizeof(char*), + .lists = &expected_strs[0], + }; + + clear_output_num(); + + ctx = test_ctx_create(); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("test_ctx_create failed"); + exit(EXIT_FAILURE); + } + + ret = flb_output_set(ctx->flb, ctx->o_ffd, + "match", "*", + "format", "json", + "json_date_key", "timestamp", + NULL); + TEST_CHECK(ret == 0); + + ret = flb_output_set_test(ctx->flb, ctx->o_ffd, + "formatter", cb_check_str_list, + &expected, NULL); + TEST_CHECK(ret == 0); + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + ret = flb_lib_push(ctx->flb, ctx->i_ffd, (char *) buf1, size1); + TEST_CHECK(ret >= 0); + + /* waiting to flush */ + flb_time_msleep(500); + + num = get_output_num(); + if (!TEST_CHECK(num > 0)) { + TEST_MSG("no outputs"); + } + + test_ctx_destroy(ctx); +} + +void flb_test_disable_json_date_key() +{ + struct test_ctx *ctx; + int ret; + int num; + + char *buf1 = "[1, {\"msg\":\"hello world\"}]"; + size_t size1 = strlen(buf1); + + char *expected_strs[] = {"{\"msg\":\"hello world\"}"}; + struct str_list expected = { + .size = sizeof(expected_strs)/sizeof(char*), + .lists = &expected_strs[0], + }; + + clear_output_num(); + + ctx = test_ctx_create(); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("test_ctx_create failed"); + exit(EXIT_FAILURE); + } + + ret = flb_output_set(ctx->flb, ctx->o_ffd, + "match", "*", + "format", "json", + "json_date_key", "false", + NULL); + TEST_CHECK(ret == 0); + + ret = flb_output_set_test(ctx->flb, ctx->o_ffd, + "formatter", cb_check_str_list, + &expected, NULL); + TEST_CHECK(ret == 0); + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + ret = flb_lib_push(ctx->flb, ctx->i_ffd, (char *) buf1, size1); + TEST_CHECK(ret >= 0); + + /* waiting to flush */ + flb_time_msleep(500); + + num = get_output_num(); + if (!TEST_CHECK(num > 0)) { + TEST_MSG("no outputs"); + } + + test_ctx_destroy(ctx); +} + +void flb_test_json_date_format_epoch() +{ + struct test_ctx *ctx; + int ret; + int num; + + char *buf1 = "[1, {\"msg\":\"hello world\"}]"; + size_t size1 = strlen(buf1); + + char *expected_strs[] = {"{\"date\":1,\"msg\":\"hello world\"}"}; + struct str_list expected = { + .size = sizeof(expected_strs)/sizeof(char*), + .lists = &expected_strs[0], + }; + + clear_output_num(); + + ctx = test_ctx_create(); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("test_ctx_create failed"); + exit(EXIT_FAILURE); + } + + ret = flb_output_set(ctx->flb, ctx->o_ffd, + "match", "*", + "format", "json", + "json_date_format", "epoch", + NULL); + TEST_CHECK(ret == 0); + + ret = flb_output_set_test(ctx->flb, ctx->o_ffd, + "formatter", cb_check_str_list, + &expected, NULL); + TEST_CHECK(ret == 0); + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + ret = flb_lib_push(ctx->flb, ctx->i_ffd, (char *) buf1, size1); + TEST_CHECK(ret >= 0); + + /* waiting to flush */ + flb_time_msleep(500); + + num = get_output_num(); + if (!TEST_CHECK(num > 0)) { + TEST_MSG("no outputs"); + } + + test_ctx_destroy(ctx); +} + +void flb_test_json_date_format_iso8601() +{ + struct test_ctx *ctx; + int ret; + int num; + + char *buf1 = "[1, {\"msg\":\"hello world\"}]"; + size_t size1 = strlen(buf1); + + char *expected_strs[] = {"\"msg\":\"hello world\"", "\"date\":\"1970-01-01T00:00:01.000000Z\""}; + struct str_list expected = { + .size = sizeof(expected_strs)/sizeof(char*), + .lists = &expected_strs[0], + }; + + clear_output_num(); + + ctx = test_ctx_create(); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("test_ctx_create failed"); + exit(EXIT_FAILURE); + } + + ret = flb_output_set(ctx->flb, ctx->o_ffd, + "match", "*", + "format", "json", + "json_date_format", "iso8601", + NULL); + TEST_CHECK(ret == 0); + + ret = flb_output_set_test(ctx->flb, ctx->o_ffd, + "formatter", cb_check_str_list, + &expected, NULL); + TEST_CHECK(ret == 0); + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + ret = flb_lib_push(ctx->flb, ctx->i_ffd, (char *) buf1, size1); + TEST_CHECK(ret >= 0); + + /* waiting to flush */ + flb_time_msleep(500); + + num = get_output_num(); + if (!TEST_CHECK(num > 0)) { + TEST_MSG("no outputs"); + } + + test_ctx_destroy(ctx); +} + +void flb_test_json_date_format_java_sql_timestamp() +{ + struct test_ctx *ctx; + int ret; + int num; + + char *buf1 = "[1, {\"msg\":\"hello world\"}]"; + size_t size1 = strlen(buf1); + + char *expected_strs[] = {"\"msg\":\"hello world\"", "\"date\":\"1970-01-01 00:00:01.000000\""}; + struct str_list expected = { + .size = sizeof(expected_strs)/sizeof(char*), + .lists = &expected_strs[0], + }; + + clear_output_num(); + + ctx = test_ctx_create(); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("test_ctx_create failed"); + exit(EXIT_FAILURE); + } + + ret = flb_output_set(ctx->flb, ctx->o_ffd, + "match", "*", + "format", "json", + "json_date_format", "java_sql_timestamp", + NULL); + TEST_CHECK(ret == 0); + + ret = flb_output_set_test(ctx->flb, ctx->o_ffd, + "formatter", cb_check_str_list, + &expected, NULL); + TEST_CHECK(ret == 0); + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + ret = flb_lib_push(ctx->flb, ctx->i_ffd, (char *) buf1, size1); + TEST_CHECK(ret >= 0); + + /* waiting to flush */ + flb_time_msleep(500); + + num = get_output_num(); + if (!TEST_CHECK(num > 0)) { + TEST_MSG("no outputs"); + } + + test_ctx_destroy(ctx); +} + +#define WAIT_STOP (5+1) /* pause in flb_engine_stop and buffer period */ + +static void cb_check_basic(void *ctx, int ffd, + int res_ret, void *res_data, size_t res_size, + void *data) +{ + char *p; + flb_sds_t out_js = res_data; + char *index_line = "{\\\"key1\\\":\\\"123\\\"}"; + + p = strstr(out_js, index_line); + if (!TEST_CHECK(p != NULL)) { + TEST_MSG("Given:%s", out_js); + } + + flb_sds_destroy(out_js); +} + + + +void flb_test_tcp_exit_workers(void) +{ + int ret; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + ctx = flb_create(); + + /* Grace -1 to do a clean shutdown */ + flb_service_set(ctx, "Flush", "1", "Grace", "-1", "Log_Level", "error", NULL); + + /* Lib input mode */ + in_ffd = flb_input(ctx, (char *) "dummy", NULL); + flb_input_set(ctx, in_ffd, + "tag", "test", + "samples", "1", + "dummy", "{\"key1\": 123, \"key2\": {\"s1\": \"fluent\"}}", + NULL); + + + /* TCP output */ + out_ffd = flb_output(ctx, (char *) "tcp", NULL); + flb_output_set(ctx, out_ffd, + "match", "test", + NULL); + + /* Enable test mode */ + ret = flb_output_set_test(ctx, out_ffd, "formatter", + cb_check_basic, + NULL, NULL); + + /* Start */ + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Stop right away */ + flb_stop(ctx); + + sleep(WAIT_STOP); + + flb_destroy(ctx); +} + +void flb_test_tcp_exit_no_workers(void) +{ + int ret; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + ctx = flb_create(); + + /* Grace -1 to do a clean shutdown */ + flb_service_set(ctx, "Flush", "1", "Grace", "-1", "Log_Level", "error", NULL); + + /* Lib input mode */ + in_ffd = flb_input(ctx, (char *) "dummy", NULL); + flb_input_set(ctx, in_ffd, + "tag", "test", + "samples", "1", + "dummy", "{\"key1\": 123, \"key2\": {\"s1\": \"fluent\"}}", + NULL); + + + /* TCP output */ + out_ffd = flb_output(ctx, (char *) "tcp", NULL); + flb_output_set(ctx, out_ffd, + "match", "test", + "workers", "0", + NULL); + + /* Enable test mode */ + ret = flb_output_set_test(ctx, out_ffd, "formatter", + cb_check_basic, + NULL, NULL); + + /* Start */ + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Stop right away */ + flb_stop(ctx); + + sleep(WAIT_STOP); + + flb_destroy(ctx); +} + +/* Test list */ +TEST_LIST = { + {"tcp_with_tls", flb_test_tcp_with_tls}, + {"format_msgpack" , flb_test_format_msgpack}, + {"format_json" , flb_test_format_json}, + {"format_json_stream" , flb_test_format_json_stream}, + {"format_json_lines" , flb_test_format_json_lines}, + {"set_json_date_key" , flb_test_set_json_date_key}, + {"disable_json_date_key" , flb_test_disable_json_date_key}, + {"json_date_format_epoch" , flb_test_json_date_format_epoch}, + {"json_date_format_iso8601" , flb_test_json_date_format_iso8601}, + {"json_date_format_java_sql_timestamp" , flb_test_json_date_format_java_sql_timestamp}, + {"tcp_exit_workers", flb_test_tcp_exit_workers}, + {"tcp_exit_no_workers", flb_test_tcp_exit_no_workers}, + {NULL, NULL} +}; diff --git a/src/fluent-bit/tests/runtime/out_td.c b/src/fluent-bit/tests/runtime/out_td.c new file mode 100644 index 000000000..d54916052 --- /dev/null +++ b/src/fluent-bit/tests/runtime/out_td.c @@ -0,0 +1,45 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ +#include <fluent-bit.h> +#include "flb_tests_runtime.h" + +/* Test data */ +#include "data/td/json_td.h" /* JSON_TD */ + +/* Test functions */ +void flb_test_td_json_long(void); + +/* Test list */ +TEST_LIST = { + {"json_long", flb_test_td_json_long }, + {NULL, NULL} +}; + + +/* It writes a big JSON message (> 3.5MB) */ +void flb_test_td_json_long(void) +{ + int ret; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + ctx = flb_create(); + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx,in_ffd, "tag", "test", NULL); + + out_ffd = flb_output(ctx, (char *) "td", NULL); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd,"match", "test", NULL); + + ret = flb_lib_config_file(ctx, (char *) "/tmp/td.conf"); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + flb_lib_push(ctx, in_ffd, (char *) JSON_TD , (int) sizeof(JSON_TD) - 1); + + flb_stop(ctx); + flb_destroy(ctx); +} diff --git a/src/fluent-bit/tests/runtime/wasm/go/Makefile b/src/fluent-bit/tests/runtime/wasm/go/Makefile new file mode 100644 index 000000000..ddd32f884 --- /dev/null +++ b/src/fluent-bit/tests/runtime/wasm/go/Makefile @@ -0,0 +1,24 @@ +.PHONY: build + +build: append_tag numeric_records say_hello modify_record + +append_tag: + tinygo build -wasm-abi=generic -target=wasi -o $@.wasm $@.go + +modify_record: + tinygo build -wasm-abi=generic -target=wasi -o $@.wasm $@.go + +numeric_records: + tinygo build -wasm-abi=generic -target=wasi -o $@.wasm $@.go + +say_hello: + tinygo build -wasm-abi=generic -target=wasi -o $@.wasm $@.go + +drop_record: + tinygo build -wasm-abi=generic -target=wasi -o $@.wasm $@.go + +testdata: + cp *.wasm ../../data/wasm + +clean: + rm -f *.wasm diff --git a/src/fluent-bit/tests/runtime/wasm/go/append_tag.go b/src/fluent-bit/tests/runtime/wasm/go/append_tag.go new file mode 100644 index 000000000..813afc51b --- /dev/null +++ b/src/fluent-bit/tests/runtime/wasm/go/append_tag.go @@ -0,0 +1,39 @@ +package main + +import ( + "fmt" + "time" + "unsafe" + + "github.com/valyala/fastjson" +) + +//export filter_append_tag +func filter_append_tag(tag *uint8, tag_len uint, time_sec uint, time_nsec uint, record *uint8, record_len uint) *uint8 { + btag := unsafe.Slice(tag, tag_len) // Note, requires Go 1.17+ (tinygo 0.20+) + brecord := unsafe.Slice(record, record_len) + _ = time.Unix(int64(time_sec), int64(time_nsec)) + + br := string(brecord) + var p fastjson.Parser + value, err := p.Parse(br) + if err != nil { + fmt.Println(err) + return nil + } + obj, err := value.Object() + if err != nil { + fmt.Println(err) + return nil + } + + var arena fastjson.Arena + obj.Set("tag", arena.NewString(string(btag))) + s := obj.String() + s += string(rune(0)) // Note: explicit null terminator. + rv := []byte(s) + + return &rv[0] +} + +func main() {} diff --git a/src/fluent-bit/tests/runtime/wasm/go/drop_record.go b/src/fluent-bit/tests/runtime/wasm/go/drop_record.go new file mode 100644 index 000000000..30be9711b --- /dev/null +++ b/src/fluent-bit/tests/runtime/wasm/go/drop_record.go @@ -0,0 +1,8 @@ +package main + +//export filter_drop_record +func filter_drop_record(tag *uint8, tag_len uint, time_sec uint, time_nsec uint, record *uint8, record_len uint) *uint8 { + return nil +} + +func main() {} diff --git a/src/fluent-bit/tests/runtime/wasm/go/go.mod b/src/fluent-bit/tests/runtime/wasm/go/go.mod new file mode 100644 index 000000000..8250a72d2 --- /dev/null +++ b/src/fluent-bit/tests/runtime/wasm/go/go.mod @@ -0,0 +1,9 @@ +module filter_wasm_go + +go 1.18 + +require ( + github.com/valyala/fastjson v1.6.3 // indirect + github.com/vmihailenco/msgpack/v5 v5.3.5 // indirect + github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect +) diff --git a/src/fluent-bit/tests/runtime/wasm/go/go.sum b/src/fluent-bit/tests/runtime/wasm/go/go.sum new file mode 100644 index 000000000..8e55f9a78 --- /dev/null +++ b/src/fluent-bit/tests/runtime/wasm/go/go.sum @@ -0,0 +1,10 @@ +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/valyala/fastjson v1.6.3 h1:tAKFnnwmeMGPbwJ7IwxcTPCNr3uIzoIj3/Fh90ra4xc= +github.com/valyala/fastjson v1.6.3/go.mod h1:CLCAqky6SMuOcxStkYQvblddUtoRxhYMGLrsQns1aXY= +github.com/vmihailenco/msgpack/v5 v5.3.5/go.mod h1:7xyJ9e+0+9SaZT0Wt1RGleJXzli6Q/V5KbhBonMG9jc= +github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/src/fluent-bit/tests/runtime/wasm/go/modify_record.go b/src/fluent-bit/tests/runtime/wasm/go/modify_record.go new file mode 100644 index 000000000..8a53a87fd --- /dev/null +++ b/src/fluent-bit/tests/runtime/wasm/go/modify_record.go @@ -0,0 +1,39 @@ +package main + +import ( + "fmt" + "time" + "unsafe" + + "github.com/valyala/fastjson" +) + +//export filter_modify_record +func filter_modify_record(tag *uint8, tag_len uint, time_sec uint, time_nsec uint, record *uint8, record_len uint) *uint8 { + _ = unsafe.Slice(tag, tag_len) // Note, requires Go 1.17+ (tinygo 0.20+) + brecord := unsafe.Slice(record, record_len) + _ = time.Unix(int64(time_sec), int64(time_nsec)) + + br := string(brecord) + var p fastjson.Parser + value, err := p.Parse(br) + if err != nil { + fmt.Println(err) + return nil + } + obj, err := value.Object() + if err != nil { + fmt.Println(err) + return nil + } + + var arena fastjson.Arena + obj.Set("modify", arena.NewString(string("yes"))) + s := obj.String() + s += string(rune(0)) // Note: explicit null terminator. + rv := []byte(s) + + return &rv[0] +} + +func main() {} diff --git a/src/fluent-bit/tests/runtime/wasm/go/numeric_records.go b/src/fluent-bit/tests/runtime/wasm/go/numeric_records.go new file mode 100644 index 000000000..4604f454e --- /dev/null +++ b/src/fluent-bit/tests/runtime/wasm/go/numeric_records.go @@ -0,0 +1,44 @@ +package main + +import ( + "fmt" + "time" + "unsafe" + + "github.com/valyala/fastjson" +) + +//export filter_numeric_records +func filter_numeric_records(tag *uint8, tag_len uint, time_sec uint, time_nsec uint, record *uint8, record_len uint) *uint8 { + _ = unsafe.Slice(tag, tag_len) // Note, requires Go 1.17+ (tinygo 0.20+) + brecord := unsafe.Slice(record, record_len) + _ = time.Unix(int64(time_sec), int64(time_nsec)) + + br := string(brecord) + var p fastjson.Parser + value, err := p.Parse(br) + if err != nil { + fmt.Println(err) + return nil + } + obj, err := value.Object() + if err != nil { + fmt.Println(err) + return nil + } + + var arena fastjson.Arena + obj.Set("wasm_int1", arena.NewNumberInt(10)) + obj.Set("wasm_int2", arena.NewNumberInt(100)) + obj.Set("wasm_float1", arena.NewNumberFloat64(10.5)) + obj.Set("wasm_float2", arena.NewNumberFloat64(100.5)) + obj.Set("wasm_truncate_float", arena.NewNumberFloat64(120.0)) + obj.Set("wasm_exp_float", arena.NewNumberFloat64(3.54789e-3)) + s := obj.String() + s += string(rune(0)) // Note: explicit null terminator. + rv := []byte(s) + + return &rv[0] +} + +func main() {} diff --git a/src/fluent-bit/tests/runtime/wasm/go/say_hello.go b/src/fluent-bit/tests/runtime/wasm/go/say_hello.go new file mode 100644 index 000000000..e5efa3f97 --- /dev/null +++ b/src/fluent-bit/tests/runtime/wasm/go/say_hello.go @@ -0,0 +1,14 @@ +package main + +import ( + "fmt" +) + +//export filter_say_hello +func filter_say_hello(tag *uint8, tag_len uint, time_sec uint, time_nsec uint, record *uint8, record_len uint) *uint8 { + fmt.Println("Hello from WASM!") + + return record +} + +func main() {} |