diff options
Diffstat (limited to 'fluent-bit/src')
188 files changed, 0 insertions, 78401 deletions
diff --git a/fluent-bit/src/CMakeLists.txt b/fluent-bit/src/CMakeLists.txt deleted file mode 100644 index b6233d9f7..000000000 --- a/fluent-bit/src/CMakeLists.txt +++ /dev/null @@ -1,575 +0,0 @@ -add_definitions(-DFLB_CORE=1) - -# Core Source -set(src - ${src} - flb_mp.c - flb_kv.c - flb_api.c - flb_csv.c - flb_lib.c - flb_log.c - flb_env.c - flb_file.c - flb_uri.c - flb_hash_table.c - flb_help.c - flb_pack.c - flb_pack_gelf.c - flb_sds.c - flb_sds_list.c - flb_pipe.c - flb_meta.c - flb_kernel.c - flb_custom.c - flb_input.c - flb_input_chunk.c - flb_input_log.c - flb_input_metric.c - flb_input_trace.c - flb_input_thread.c - flb_filter.c - flb_output.c - flb_output_thread.c - flb_config.c - flb_config_map.c - flb_socket.c - flb_network.c - flb_utils.c - flb_slist.c - flb_engine.c - flb_engine_dispatch.c - flb_task.c - flb_unescape.c - flb_scheduler.c - flb_io.c - flb_storage.c - flb_connection.c - flb_downstream.c - flb_upstream.c - flb_upstream_ha.c - flb_upstream_node.c - flb_router.c - flb_worker.c - flb_coro.c - flb_time.c - flb_sosreport.c - flb_hmac.c - flb_hash.c - flb_crypto.c - flb_random.c - flb_plugin.c - flb_gzip.c - flb_snappy.c - flb_compression.c - flb_http_client.c - flb_callback.c - flb_strptime.c - flb_fstore.c - flb_thread_pool.c - flb_routes_mask.c - flb_typecast.c - flb_event.c - flb_base64.c - flb_ring_buffer.c - flb_log_event_decoder.c - flb_log_event_encoder.c - flb_log_event_encoder_primitives.c - flb_log_event_encoder_dynamic_field.c - flb_processor.c - flb_reload.c - ) - -# Config format -set(src - ${src} - config_format/flb_config_format.c - config_format/flb_cf_fluentbit.c -) -if(FLB_HAVE_LIBYAML) - set(src - ${src} - config_format/flb_cf_yaml.c - ) -endif() - -# Multiline subsystem -add_subdirectory(multiline) -set(src - ${src} - ${src_multiline} - ) - -if(FLB_SYSTEM_WINDOWS) - set(src - ${src} - flb_dlfcn_win32.c - ) - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /W2") -endif() - -if(FLB_PARSER) - set(src - ${src} - flb_parser.c - flb_parser_regex.c - flb_parser_json.c - flb_parser_decoder.c - flb_parser_ltsv.c - flb_parser_logfmt.c - ) -endif() - -if(FLB_AVRO_ENCODER) - set(src - ${src} - flb_avro.c - ) -endif() - -# Fluent Bit have TLS support -if(FLB_TLS) - # Register the TLS interface and functions - set(src - ${src} - "tls/flb_tls.c" - "flb_oauth2.c" - ) - - # Make sure our output targets links to the TLS library - set(extra_libs - ${extra_libs} - ) -endif() - -if(FLB_PROXY_GO) - set(src - ${src} - "flb_plugin_proxy.c" - ) -endif() - -if(FLB_METRICS) - set(src - ${src} - "flb_metrics.c" - "flb_metrics_exporter.c" - ) -endif() - -if(FLB_SIGNV4 AND FLB_TLS) - set(src - ${src} - "flb_signv4.c" - ) -endif() - -if(FLB_HTTP_CLIENT_DEBUG) - set(src - ${src} - "flb_http_client_debug.c" - ) -endif() - -if (FLB_AWS_ERROR_REPORTER) - set(src - ${src} - "aws/flb_aws_error_reporter.c" - ) -endif() - -if(FLB_LUAJIT) - set(src - ${src} - "flb_lua.c" - "flb_luajit.c" - ) -endif() - -if(FLB_IN_KAFKA OR FLB_OUT_KAFKA) - set(src - ${src} - "flb_kafka.c" - ) -endif() - -# Link to libco -set(extra_libs - ${extra_libs} - "co") - -set(extra_libs - ${extra_libs} - "rbtree") - -if(FLB_JEMALLOC) - set(extra_libs - ${extra_libs} - "libjemalloc") -endif() - -if(FLB_REGEX) - set(FLB_DEPS - ${FLB_DEPSS} - onigmo-static) - set(src - ${src} - "flb_regex.c" - ) -endif() - -if(FLB_LUAJIT) - set(extra_libs - ${extra_libs} - "libluajit") -endif() - -if(FLB_SQLDB) - set(src - ${src} - "flb_sqldb.c" - ) - set(extra_libs - ${extra_libs} - "sqlite3") -endif() - -if(FLB_STATIC_CONF) - set(src - ${src} - "flb_config_static.c" - ) -endif() - -if(FLB_CHUNK_TRACE) - set(src - ${src} - "flb_chunk_trace.c" - ) -endif() - -include(CheckSymbolExists) -check_symbol_exists(accept4 "sys/socket.h" HAVE_ACCEPT4) - -# Core dependencies -if(FLB_SYSTEM_WINDOWS) - set(FLB_DEPS - "ws2_32.lib" - "crypt32.lib" - "Bcrypt.lib" - "Shlwapi.lib" - ) -else() - set(FLB_DEPS - ${FLB_DEPS} - ${CMAKE_DL_LIBS} - m - ) -endif() - -# Link timer library -if(CMAKE_SYSTEM_NAME MATCHES "Linux") - set(FLB_DEPS - ${FLB_DEPS} - rt - ) -endif() - - -# Record Accessor -# --------------- -# Make sure it dependency is enabled -if(FLB_RECORD_ACCESSOR AND NOT FLB_REGEX) - message(FATAL_ERROR - "FLB_RECORD_ACCESSOR depends on FLB_REGEX, " - "enable it with: -DFLB_REGEX=ON") -endif() - -# Build record accessor files -if(FLB_RECORD_ACCESSOR) - set(src - ${src} - "flb_record_accessor.c" - "flb_ra_key.c" - ) - add_subdirectory(record_accessor) -endif() - -# Stream Processor -if(FLB_STREAM_PROCESSOR) - add_subdirectory(stream_processor) -endif() - -# AWS specific -if(FLB_AWS) - add_subdirectory(aws) -endif() - -# HTTP Server -if(FLB_HTTP_SERVER) - add_subdirectory(http_server) -endif() - -# Proxy interfaces -add_subdirectory(proxy) - -set(FLB_PROXY_PLUGINS "") -if(FLB_PROXY_GO) - set(FLB_PROXY_PLUGINS ${FLB_PROXY_PLUGINS} flb-plugin-proxy-go) -endif() - -# WASM runtime -if(FLB_WASM) - add_subdirectory(wasm) -endif() - -# WAMRC compiler -if(FLB_WAMRC) - add_subdirectory(wamrc) -endif() - -# HTTP Server -if(FLB_HTTP_SERVER) - set(FLB_DEPS - ${FLB_DEPS} - flb-http-server) -endif() - -# AVRO Encoding -if(FLB_AVRO_ENCODER) -set(FLB_DEPS - ${FLB_DEPS} - avro-static - jansson - ) -endif() - -# WASM runtime -if(FLB_WASM) - set(FLB_DEPS - ${FLB_DEPS} - vmlib-static - flb-wasm-static) -endif() - -# Set static dependencies -set(FLB_DEPS - ${FLB_DEPS} - cfl-static - fluent-otel-proto - cmetrics-static - ctraces-static - mk_core - jsmn - msgpack-c-static - mpack-static - chunkio-static - miniz - - ${FLB_PLUGINS} - ${FLB_PROXY_PLUGINS} - ${extra_libs} - c-ares - snappy-c - lwrb - ) - -if(OPENSSL_FOUND) - set(FLB_DEPS - ${FLB_DEPS} - OpenSSL::SSL - ) -endif() - -# libyaml -if(FLB_HAVE_LIBYAML) -set(FLB_DEPS - ${FLB_DEPS} - yaml - ) -endif() - -# UTF8 Encoding -if(FLB_UTF8_ENCODER) -set(FLB_DEPS - ${FLB_DEPS} - tutf8e - ) -endif() - -# AWS specific -if(FLB_AWS) - set(FLB_DEPS - ${FLB_DEPS} - flb-aws - ) -endif() - -# Record Accessor -if(FLB_RECORD_ACCESSOR) - set(FLB_DEPS - ${FLB_DEPS} - flb-ra-parser - ) -endif() - -# Stream Processor -if(FLB_STREAM_PROCESSOR) - set(FLB_DEPS - ${FLB_DEPS} - flb-sp - ) -endif() - -if (MSVC) -set(flb_rc_files - ${CMAKE_CURRENT_BINARY_DIR}/version.rc - ) -endif() - -# Shared Library -if(FLB_SHARED_LIB) - add_library(fluent-bit-shared SHARED ${src}) - add_sanitizers(fluent-bit-shared) - set_target_properties(fluent-bit-shared - PROPERTIES OUTPUT_NAME fluent-bit) - - # Windows doesn't provide pthread (see winpthreads.c in mk_core). - if(CMAKE_SYSTEM_NAME MATCHES "Windows") - target_link_libraries(fluent-bit-shared ${FLB_DEPS}) - else() - target_link_libraries(fluent-bit-shared ${FLB_DEPS} -lpthread) - endif() - - if (MSVC) - set_target_properties(fluent-bit-shared - PROPERTIES PDB_NAME fluent-bit.dll) - target_link_options(fluent-bit-shared - PUBLIC /pdb:$<TARGET_PDB_FILE:fluent-bit-shared>) - endif() - - # Library install routines - install(TARGETS fluent-bit-shared - LIBRARY DESTINATION ${FLB_INSTALL_LIBDIR} - COMPONENT library - RUNTIME DESTINATION ${FLB_INSTALL_BINDIR}) -endif() - -# Static Library -add_library(fluent-bit-static STATIC ${src}) -add_sanitizers(fluent-bit-static) -target_link_libraries(fluent-bit-static ${FLB_DEPS}) - -if(MSVC) - # Rename the output for Windows environment to avoid naming issues - set_target_properties(fluent-bit-static PROPERTIES OUTPUT_NAME libfluent-bit) -else() - set_target_properties(fluent-bit-static PROPERTIES OUTPUT_NAME fluent-bit) -endif(MSVC) - -if(FLB_JEMALLOC) - target_link_libraries(fluent-bit-static libjemalloc) -endif() - -# Binary / Executable -if(FLB_BINARY) - find_package (Threads) - if (FLB_SYSTEM_WINDOWS) - add_executable(fluent-bit-bin fluent-bit.c flb_dump.c win32/winsvc.c ${flb_rc_files}) - else() - add_executable(fluent-bit-bin fluent-bit.c flb_dump.c) - endif() - add_sanitizers(fluent-bit-bin) - - - if(FLB_STATIC_CONF) - add_dependencies(fluent-bit-bin flb-static-conf) - endif() - - if(FLB_REGEX) - target_link_libraries(fluent-bit-bin onigmo-static) - endif() - - if(FLB_JEMALLOC) - target_link_libraries(fluent-bit-bin libjemalloc) - endif() - - if(FLB_BACKTRACE) - add_definitions(-DFLB_DUMP_STACKTRACE=1) - target_link_libraries(fluent-bit-bin libbacktrace) - endif() - - target_link_libraries(fluent-bit-bin fluent-bit-static ${CMAKE_THREAD_LIBS_INIT}) - - set_target_properties(fluent-bit-bin - PROPERTIES - OUTPUT_NAME ${FLB_OUT_NAME} - ENABLE_EXPORTS ON) - install(TARGETS fluent-bit-bin RUNTIME DESTINATION ${FLB_INSTALL_BINDIR} COMPONENT binary) - - # Include PDB file (if available) - if (MSVC) - target_link_options(fluent-bit-bin - PUBLIC /pdb:$<TARGET_PDB_FILE:fluent-bit-bin>) - install(FILES $<TARGET_PDB_FILE:fluent-bit-bin> - DESTINATION "${FLB_INSTALL_BINDIR}") - endif() - - # Detect init system, install upstart, systemd or init.d script - - # Handle issues with detection on some systems during build - if(NOT SYSTEMD_UNITDIR AND IS_DIRECTORY /lib/systemd/system) - set(SYSTEMD_UNITDIR /lib/systemd/system) - endif() - - if(SYSTEMD_UNITDIR) - set(FLB_SYSTEMD_SCRIPT "${PROJECT_SOURCE_DIR}/init/${FLB_OUT_NAME}.service") - configure_file( - "${PROJECT_SOURCE_DIR}/init/systemd.in" - ${FLB_SYSTEMD_SCRIPT} - ) - install(FILES ${FLB_SYSTEMD_SCRIPT} COMPONENT binary DESTINATION ${SYSTEMD_UNITDIR}) - install(DIRECTORY DESTINATION ${FLB_INSTALL_CONFDIR} COMPONENT binary) - elseif(IS_DIRECTORY /usr/share/upstart) - set(FLB_UPSTART_SCRIPT "${PROJECT_SOURCE_DIR}/init/${FLB_OUT_NAME}.conf") - configure_file( - "${PROJECT_SOURCE_DIR}/init/upstart.in" - ${FLB_UPSTART_SCRIPT} - ) - install(FILES ${FLB_UPSTART_SCRIPT} COMPONENT binary DESTINATION /etc/init) - install(DIRECTORY DESTINATION COMPONENT binary ${FLB_INSTALL_CONFDIR}) - else() - # FIXME: should we support Sysv init script ? - endif() - - if(FLB_SYSTEM_WINDOWS) - install(FILES - "${PROJECT_SOURCE_DIR}/conf/fluent-bit-win32.conf" - DESTINATION ${FLB_INSTALL_CONFDIR} - COMPONENT binary - RENAME "${FLB_OUT_NAME}.conf") - elseif(FLB_SYSTEM_MACOS) - install(FILES - "${PROJECT_SOURCE_DIR}/conf/fluent-bit-macos.conf" - DESTINATION ${FLB_INSTALL_CONFDIR} - COMPONENT binary - RENAME "${FLB_OUT_NAME}.conf") - else() - install(FILES - "${PROJECT_SOURCE_DIR}/conf/fluent-bit.conf" - DESTINATION ${FLB_INSTALL_CONFDIR} - COMPONENT binary - RENAME "${FLB_OUT_NAME}.conf") - endif() - - install(FILES - "${PROJECT_SOURCE_DIR}/conf/parsers.conf" - COMPONENT binary - DESTINATION ${FLB_INSTALL_CONFDIR}) - - install(FILES - "${PROJECT_SOURCE_DIR}/conf/plugins.conf" - COMPONENT binary - DESTINATION ${FLB_INSTALL_CONFDIR}) - -endif() diff --git a/fluent-bit/src/aws/CMakeLists.txt b/fluent-bit/src/aws/CMakeLists.txt deleted file mode 100644 index a6580e7a5..000000000 --- a/fluent-bit/src/aws/CMakeLists.txt +++ /dev/null @@ -1,32 +0,0 @@ -add_subdirectory(compression) - -include_directories( - ${CMAKE_CURRENT_SOURCE_DIR} - ${CMAKE_CURRENT_BINARY_DIR} - ) - -set(src - "flb_aws_credentials_log.h" - "flb_aws_compress.c" - "flb_aws_util.c" - "flb_aws_credentials.c" - "flb_aws_credentials_sts.c" - "flb_aws_credentials_ec2.c" - "flb_aws_imds.c" - "flb_aws_credentials_http.c" - "flb_aws_credentials_profile.c" - ) - -if(FLB_HAVE_AWS_CREDENTIAL_PROCESS) - set(src - ${src} - "flb_aws_credentials_process.c" - ) -endif() - -add_library(flb-aws STATIC ${src}) -target_link_libraries(flb-aws flb-aws-compress) - -if(FLB_JEMALLOC) - target_link_libraries(flb-aws libjemalloc) -endif() diff --git a/fluent-bit/src/aws/compression/CMakeLists.txt b/fluent-bit/src/aws/compression/CMakeLists.txt deleted file mode 100644 index 02a1ba3a6..000000000 --- a/fluent-bit/src/aws/compression/CMakeLists.txt +++ /dev/null @@ -1,6 +0,0 @@ -add_library(flb-aws-compress INTERFACE) - -if(FLB_ARROW) - add_subdirectory(arrow EXCLUDE_FROM_ALL) - target_link_libraries(flb-aws-compress INTERFACE flb-aws-arrow) -endif() diff --git a/fluent-bit/src/aws/compression/arrow/CMakeLists.txt b/fluent-bit/src/aws/compression/arrow/CMakeLists.txt deleted file mode 100644 index 846f65441..000000000 --- a/fluent-bit/src/aws/compression/arrow/CMakeLists.txt +++ /dev/null @@ -1,7 +0,0 @@ -set(src - compress.c) - -add_library(flb-aws-arrow STATIC ${src}) - -target_include_directories(flb-aws-arrow PRIVATE ${ARROW_GLIB_INCLUDE_DIRS}) -target_link_libraries(flb-aws-arrow ${ARROW_GLIB_LDFLAGS}) diff --git a/fluent-bit/src/aws/compression/arrow/compress.c b/fluent-bit/src/aws/compression/arrow/compress.c deleted file mode 100644 index a48b34f80..000000000 --- a/fluent-bit/src/aws/compression/arrow/compress.c +++ /dev/null @@ -1,147 +0,0 @@ -/* - * This converts S3 plugin's request buffer into Apache Arrow format. - * - * We use GLib binding to call Arrow functions (which is implemented - * in C++) from Fluent Bit. - * - * https://github.com/apache/arrow/tree/master/c_glib - */ - -#include <arrow-glib/arrow-glib.h> -#include <inttypes.h> - -/* - * GArrowTable is the central structure that represents "table" (a.k.a. - * data frame). - */ -static GArrowTable* parse_json(uint8_t *json, int size) -{ - GArrowJSONReader *reader; - GArrowBuffer *buffer; - GArrowBufferInputStream *input; - GArrowJSONReadOptions *options; - GArrowTable *table; - GError *error = NULL; - - buffer = garrow_buffer_new(json, size); - if (buffer == NULL) { - return NULL; - } - - input = garrow_buffer_input_stream_new(buffer); - if (input == NULL) { - g_object_unref(buffer); - return NULL; - } - - options = garrow_json_read_options_new(); - if (options == NULL) { - g_object_unref(buffer); - g_object_unref(input); - return NULL; - } - - reader = garrow_json_reader_new(GARROW_INPUT_STREAM(input), options, &error); - if (reader == NULL) { - g_error_free(error); - g_object_unref(buffer); - g_object_unref(input); - g_object_unref(options); - return NULL; - } - - table = garrow_json_reader_read(reader, &error); - if (table == NULL) { - g_error_free(error); - g_object_unref(buffer); - g_object_unref(input); - g_object_unref(options); - g_object_unref(reader); - return NULL; - } - g_object_unref(buffer); - g_object_unref(input); - g_object_unref(options); - g_object_unref(reader); - return table; -} - -static GArrowResizableBuffer* table_to_buffer(GArrowTable *table) -{ - GArrowResizableBuffer *buffer; - GArrowBufferOutputStream *sink; - GError *error = NULL; - gboolean success; - - buffer = garrow_resizable_buffer_new(0, &error); - if (buffer == NULL) { - g_error_free(error); - return NULL; - } - - sink = garrow_buffer_output_stream_new(buffer); - if (sink == NULL) { - g_object_unref(buffer); - return NULL; - } - - success = garrow_table_write_as_feather( - table, GARROW_OUTPUT_STREAM(sink), - NULL, &error); - if (!success) { - g_error_free(error); - g_object_unref(buffer); - g_object_unref(sink); - return NULL; - } - g_object_unref(sink); - return buffer; -} - -int out_s3_compress_arrow(void *json, size_t size, void **out_buf, size_t *out_size) -{ - GArrowTable *table; - GArrowResizableBuffer *buffer; - GBytes *bytes; - gconstpointer ptr; - gsize len; - uint8_t *buf; - - table = parse_json((uint8_t *) json, size); - if (table == NULL) { - return -1; - } - - buffer = table_to_buffer(table); - g_object_unref(table); - if (buffer == NULL) { - return -1; - } - - bytes = garrow_buffer_get_data(GARROW_BUFFER(buffer)); - if (bytes == NULL) { - g_object_unref(buffer); - return -1; - } - - ptr = g_bytes_get_data(bytes, &len); - if (ptr == NULL) { - g_object_unref(buffer); - g_bytes_unref(bytes); - return -1; - } - - buf = malloc(len); - if (buf == NULL) { - g_object_unref(buffer); - g_bytes_unref(bytes); - return -1; - } - memcpy(buf, ptr, len); - *out_buf = (void *) buf; - *out_size = len; - - g_object_unref(buffer); - g_bytes_unref(bytes); - return 0; -} diff --git a/fluent-bit/src/aws/compression/arrow/compress.h b/fluent-bit/src/aws/compression/arrow/compress.h deleted file mode 100644 index 82e94f43c..000000000 --- a/fluent-bit/src/aws/compression/arrow/compress.h +++ /dev/null @@ -1,13 +0,0 @@ -/* - * This function converts out_s3 buffer into Apache Arrow format. - * - * `json` is a string that contain (concatenated) JSON objects. - * - * `size` is the length of the json data (excluding the trailing - * null-terminator character). - * - * Return 0 on success (with `out_buf` and `out_size` updated), - * and -1 on failure - */ - -int out_s3_compress_arrow(void *json, size_t size, void **out_buf, size_t *out_size); diff --git a/fluent-bit/src/aws/flb_aws_compress.c b/fluent-bit/src/aws/flb_aws_compress.c deleted file mode 100644 index e98ce8318..000000000 --- a/fluent-bit/src/aws/flb_aws_compress.c +++ /dev/null @@ -1,245 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2019-2021 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <fluent-bit/flb_mem.h> -#include <fluent-bit/flb_log.h> -#include <fluent-bit/flb_base64.h> - -#include <fluent-bit/aws/flb_aws_compress.h> -#include <fluent-bit/flb_gzip.h> - -#include <stdint.h> - -#ifdef FLB_HAVE_ARROW -#include "compression/arrow/compress.h" -#endif - -struct compression_option { - int compression_type; - char *compression_keyword; - int(*compress)(void *in_data, size_t in_len, void **out_data, size_t *out_len); -}; - -/* - * Library of compression options - * AWS plugins that support compression will have these options. - * Referenced function should return -1 on error and 0 on success. - */ -static const struct compression_option compression_options[] = { - /* FLB_AWS_COMPRESS_NONE which is 0 is reserved for array footer */ - { - FLB_AWS_COMPRESS_GZIP, - "gzip", - &flb_gzip_compress - }, -#ifdef FLB_HAVE_ARROW - { - FLB_AWS_COMPRESS_ARROW, - "arrow", - &out_s3_compress_arrow - }, -#endif - { 0 } -}; - -int flb_aws_compression_get_type(const char *compression_keyword) -{ - int ret; - const struct compression_option *o; - - o = compression_options; - - while (o->compression_type != 0) { - ret = strcmp(o->compression_keyword, compression_keyword); - if (ret == 0) { - return o->compression_type; - } - ++o; - } - - flb_error("[aws_compress] unknown compression type: %s", compression_keyword); - return -1; -} - -int flb_aws_compression_compress(int compression_type, void *in_data, size_t in_len, - void **out_data, size_t *out_len) -{ - const struct compression_option *o; - - o = compression_options; - - while (o->compression_type != 0) { - if (o->compression_type == compression_type) { - return o->compress(in_data, in_len, out_data, out_len); - } - ++o; - } - - flb_error("[aws_compress] invalid compression type: %i", compression_type); - flb_errno(); - return -1; -} - -int flb_aws_compression_b64_truncate_compress(int compression_type, size_t max_out_len, - void *in_data, size_t in_len, - void **out_data, size_t *out_len) -{ - static const void *truncation_suffix = "[Truncated...]"; - static const size_t truncation_suffix_len = 14; - static const double truncation_reduction_percent = 90; /* % out of 100 */ - static const int truncation_compression_max_attempts = 10; - - int ret; - int is_truncated; - int compression_attempts; - size_t truncated_in_len_prev; - size_t truncated_in_len; - void *truncated_in_buf; - void *compressed_buf; - size_t compressed_len; - size_t original_b64_compressed_len; - - unsigned char *b64_compressed_buf; - size_t b64_compressed_len; - size_t b64_actual_len; - - /* Iterative approach to truncation */ - truncated_in_len = in_len; - truncated_in_buf = in_data; - is_truncated = FLB_FALSE; - b64_compressed_len = SIZE_MAX; - compression_attempts = 0; - while (max_out_len < b64_compressed_len - 1) { - - /* Limit compression truncation attempts, just to be safe */ - if (compression_attempts >= truncation_compression_max_attempts) { - if (is_truncated) { - flb_free(truncated_in_buf); - } - flb_error("[aws_compress] truncation failed, too many compression attempts"); - return -1; - } - - ret = flb_aws_compression_compress(compression_type, truncated_in_buf, - truncated_in_len, &compressed_buf, - &compressed_len); - ++compression_attempts; - if (ret != 0) { - if (is_truncated) { - flb_free(truncated_in_buf); - } - return -1; - } - - /* Determine encoded base64 buffer size */ - b64_compressed_len = compressed_len / 3; /* Compute number of 4 sextet groups */ - b64_compressed_len += (compressed_len % 3 != 0); /* Add padding partial group */ - b64_compressed_len *= 4; /* Compute number of sextets */ - b64_compressed_len += 1; /* Add room for null character 0x00 */ - - /* Truncation needed */ - if (max_out_len < b64_compressed_len - 1) { - flb_debug("[aws_compress] iterative truncation round"); - - /* This compressed_buf is the wrong size. Free */ - flb_free(compressed_buf); - - /* Base case: input compressed empty string, output still too large */ - if (truncated_in_len == 0) { - if (is_truncated) { - flb_free(truncated_in_buf); - } - flb_error("[aws_compress] truncation failed, compressed empty input too " - "large"); - return -1; - } - - /* Calculate corrected input size */ - truncated_in_len_prev = truncated_in_len; - truncated_in_len = (max_out_len * truncated_in_len) / b64_compressed_len; - truncated_in_len = (truncated_in_len * truncation_reduction_percent) / 100; - - /* Ensure working down */ - if (truncated_in_len >= truncated_in_len_prev) { - truncated_in_len = truncated_in_len_prev - 1; - } - - /* Allocate truncation buffer */ - if (!is_truncated) { - is_truncated = FLB_TRUE; - original_b64_compressed_len = b64_compressed_len; - truncated_in_buf = flb_malloc(in_len); - if (!truncated_in_buf) { - flb_errno(); - return -1; - } - memcpy(truncated_in_buf, in_data, in_len); - } - - /* Slap on truncation suffix */ - if (truncated_in_len < truncation_suffix_len) { - /* No room for the truncation suffix. Terminal error */ - flb_error("[aws_compress] truncation failed, no room for suffix"); - flb_free(truncated_in_buf); - return -1; - } - memcpy((char *) truncated_in_buf + truncated_in_len - truncation_suffix_len, - truncation_suffix, truncation_suffix_len); - } - } - - /* Truncate buffer free and compression buffer allocation */ - if (is_truncated) { - flb_free(truncated_in_buf); - flb_warn("[aws_compress][size=%zu] Truncating input for compressed output " - "larger than %zu bytes, output from %zu to %zu bytes", - in_len, max_out_len, original_b64_compressed_len - 1, - b64_compressed_len - 1); - } - b64_compressed_buf = flb_malloc(b64_compressed_len); - if (!b64_compressed_buf) { - flb_errno(); - return -1; - } - - /* Base64 encode compressed out bytes */ - ret = flb_base64_encode(b64_compressed_buf, b64_compressed_len, &b64_actual_len, - compressed_buf, compressed_len); - flb_free(compressed_buf); - - if (ret == FLB_BASE64_ERR_BUFFER_TOO_SMALL) { - flb_error("[aws_compress] compressed log base64 buffer too small"); - return -1; /* not handle truncation at this point */ - } - if (ret != 0) { - flb_free(b64_compressed_buf); - return -1; - } - - /* Double check b64 buf len */ - if (b64_compressed_len - 1 != b64_actual_len) { - flb_error("[aws_compress] buffer len should be 1 greater than actual len"); - flb_free(b64_compressed_buf); - return -1; - } - - *out_data = b64_compressed_buf; - *out_len = b64_compressed_len - 1; /* disregard added null character */ - return 0; -} diff --git a/fluent-bit/src/aws/flb_aws_credentials.c b/fluent-bit/src/aws/flb_aws_credentials.c deleted file mode 100644 index 850142e24..000000000 --- a/fluent-bit/src/aws/flb_aws_credentials.c +++ /dev/null @@ -1,862 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <fluent-bit/flb_info.h> -#include <fluent-bit/flb_sds.h> -#include <fluent-bit/flb_http_client.h> -#include <fluent-bit/flb_aws_credentials.h> -#include <fluent-bit/flb_aws_util.h> -#include <fluent-bit/flb_jsmn.h> -#include <fluent-bit/flb_output_plugin.h> - -#include <stdlib.h> -#include <time.h> - -#define FIVE_MINUTES 300 -#define TWELVE_HOURS 43200 - -/* Credentials Environment Variables */ -#define AWS_ACCESS_KEY_ID "AWS_ACCESS_KEY_ID" -#define AWS_SECRET_ACCESS_KEY "AWS_SECRET_ACCESS_KEY" -#define AWS_SESSION_TOKEN "AWS_SESSION_TOKEN" - -#define EKS_POD_EXECUTION_ROLE "EKS_POD_EXECUTION_ROLE" - -/* declarations */ -static struct flb_aws_provider *standard_chain_create(struct flb_config - *config, - struct flb_tls *tls, - char *region, - char *sts_endpoint, - char *proxy, - struct - flb_aws_client_generator - *generator, - int eks_irsa, - char *profile); - - -/* - * The standard credential provider chain: - * 1. Environment variables - * 2. Shared credentials file (AWS Profile) - * 3. EKS OIDC - * 4. EC2 IMDS - * 5. ECS HTTP credentials endpoint - * - * This provider will evaluate each provider in order, returning the result - * from the first provider that returns valid credentials. - * - * Note: Client code should use this provider by default. - */ -struct flb_aws_provider_chain { - struct mk_list sub_providers; - - /* - * The standard chain provider picks the first successful provider and - * then uses it until a call to refresh is made. - */ - struct flb_aws_provider *sub_provider; -}; - -/* - * Iterates through the chain and returns credentials from the first provider - * that successfully returns creds. Caches this provider on the implementation. - */ -struct flb_aws_credentials *get_from_chain(struct flb_aws_provider_chain - *implementation) -{ - struct flb_aws_provider *sub_provider = NULL; - struct mk_list *tmp; - struct mk_list *head; - struct flb_aws_credentials *creds = NULL; - - /* find the first provider that produces a valid set of creds */ - mk_list_foreach_safe(head, tmp, &implementation->sub_providers) { - sub_provider = mk_list_entry(head, - struct flb_aws_provider, - _head); - creds = sub_provider->provider_vtable->get_credentials(sub_provider); - if (creds) { - implementation->sub_provider = sub_provider; - return creds; - } - } - - return NULL; -} - -struct flb_aws_credentials *get_credentials_fn_standard_chain(struct - flb_aws_provider - *provider) -{ - struct flb_aws_credentials *creds = NULL; - struct flb_aws_provider_chain *implementation = provider->implementation; - struct flb_aws_provider *sub_provider = implementation->sub_provider; - - if (sub_provider) { - return sub_provider->provider_vtable->get_credentials(sub_provider); - } - - if (try_lock_provider(provider)) { - creds = get_from_chain(implementation); - unlock_provider(provider); - return creds; - } - - /* - * We failed to lock the provider and sub_provider is unset. This means that - * another co-routine is selecting a provider from the chain. - */ - flb_warn("[aws_credentials] No cached credentials are available and " - "a credential refresh is already in progress. The current " - "co-routine will retry."); - return NULL; -} - -int init_fn_standard_chain(struct flb_aws_provider *provider) -{ - struct flb_aws_provider_chain *implementation = provider->implementation; - struct flb_aws_provider *sub_provider = NULL; - struct mk_list *tmp; - struct mk_list *head; - int ret = -1; - - if (try_lock_provider(provider)) { - /* find the first provider that indicates successful init */ - mk_list_foreach_safe(head, tmp, &implementation->sub_providers) { - sub_provider = mk_list_entry(head, - struct flb_aws_provider, - _head); - ret = sub_provider->provider_vtable->init(sub_provider); - if (ret >= 0) { - implementation->sub_provider = sub_provider; - break; - } - } - unlock_provider(provider); - } - - return ret; -} - -/* - * Client code should only call refresh if there has been an - * error from the AWS APIs indicating creds are expired/invalid. - * Refresh may change the current sub_provider. - */ -int refresh_fn_standard_chain(struct flb_aws_provider *provider) -{ - struct flb_aws_provider_chain *implementation = provider->implementation; - struct flb_aws_provider *sub_provider = NULL; - struct mk_list *tmp; - struct mk_list *head; - int ret = -1; - - if (try_lock_provider(provider)) { - /* find the first provider that indicates successful refresh */ - mk_list_foreach_safe(head, tmp, &implementation->sub_providers) { - sub_provider = mk_list_entry(head, - struct flb_aws_provider, - _head); - ret = sub_provider->provider_vtable->refresh(sub_provider); - if (ret >= 0) { - implementation->sub_provider = sub_provider; - break; - } - } - unlock_provider(provider); - } - - return ret; -} - -void sync_fn_standard_chain(struct flb_aws_provider *provider) -{ - struct flb_aws_provider_chain *implementation = provider->implementation; - struct flb_aws_provider *sub_provider = NULL; - struct mk_list *tmp; - struct mk_list *head; - - /* set all providers to sync mode */ - mk_list_foreach_safe(head, tmp, &implementation->sub_providers) { - sub_provider = mk_list_entry(head, - struct flb_aws_provider, - _head); - sub_provider->provider_vtable->sync(sub_provider); - } -} - -void async_fn_standard_chain(struct flb_aws_provider *provider) -{ - struct flb_aws_provider_chain *implementation = provider->implementation; - struct flb_aws_provider *sub_provider = NULL; - struct mk_list *tmp; - struct mk_list *head; - - /* set all providers to async mode */ - mk_list_foreach_safe(head, tmp, &implementation->sub_providers) { - sub_provider = mk_list_entry(head, - struct flb_aws_provider, - _head); - sub_provider->provider_vtable->async(sub_provider); - } -} - -void upstream_set_fn_standard_chain(struct flb_aws_provider *provider, - struct flb_output_instance *ins) -{ - struct flb_aws_provider_chain *implementation = provider->implementation; - struct flb_aws_provider *sub_provider = NULL; - struct mk_list *tmp; - struct mk_list *head; - - /* set all providers to async mode */ - mk_list_foreach_safe(head, tmp, &implementation->sub_providers) { - sub_provider = mk_list_entry(head, - struct flb_aws_provider, - _head); - sub_provider->provider_vtable->upstream_set(sub_provider, ins); - } -} - -void destroy_fn_standard_chain(struct flb_aws_provider *provider) { - struct flb_aws_provider *sub_provider; - struct flb_aws_provider_chain *implementation; - struct mk_list *tmp; - struct mk_list *head; - - implementation = provider->implementation; - - if (implementation) { - mk_list_foreach_safe(head, tmp, &implementation->sub_providers) { - sub_provider = mk_list_entry(head, struct flb_aws_provider, - _head); - mk_list_del(&sub_provider->_head); - flb_aws_provider_destroy(sub_provider); - } - - flb_free(implementation); - } -} - -static struct flb_aws_provider_vtable standard_chain_provider_vtable = { - .get_credentials = get_credentials_fn_standard_chain, - .init = init_fn_standard_chain, - .refresh = refresh_fn_standard_chain, - .destroy = destroy_fn_standard_chain, - .sync = sync_fn_standard_chain, - .async = async_fn_standard_chain, - .upstream_set = upstream_set_fn_standard_chain, -}; - -struct flb_aws_provider *flb_standard_chain_provider_create(struct flb_config - *config, - struct flb_tls *tls, - char *region, - char *sts_endpoint, - char *proxy, - struct - flb_aws_client_generator - *generator, - char *profile) -{ - struct flb_aws_provider *provider; - struct flb_aws_provider *tmp_provider; - char *eks_pod_role = NULL; - char *session_name; - - eks_pod_role = getenv(EKS_POD_EXECUTION_ROLE); - if (eks_pod_role && strlen(eks_pod_role) > 0) { - /* - * eks fargate - * standard chain will be base provider used to - * assume the EKS_POD_EXECUTION_ROLE - */ - flb_debug("[aws_credentials] Using EKS_POD_EXECUTION_ROLE=%s", eks_pod_role); - tmp_provider = standard_chain_create(config, tls, region, sts_endpoint, - proxy, generator, FLB_FALSE, profile); - - if (!tmp_provider) { - return NULL; - } - - session_name = flb_sts_session_name(); - if (!session_name) { - flb_error("Failed to generate random STS session name"); - flb_aws_provider_destroy(tmp_provider); - return NULL; - } - - provider = flb_sts_provider_create(config, tls, tmp_provider, NULL, - eks_pod_role, session_name, - region, sts_endpoint, - NULL, generator); - if (!provider) { - flb_error("Failed to create EKS Fargate Credential Provider"); - flb_aws_provider_destroy(tmp_provider); - return NULL; - } - /* session name can freed after provider is created */ - flb_free(session_name); - session_name = NULL; - - return provider; - } - - /* standard case- not in EKS Fargate */ - provider = standard_chain_create(config, tls, region, sts_endpoint, - proxy, generator, FLB_TRUE, profile); - return provider; -} - -struct flb_aws_provider *flb_managed_chain_provider_create(struct flb_output_instance - *ins, - struct flb_config - *config, - char *config_key_prefix, - char *proxy, - struct - flb_aws_client_generator - *generator) -{ - flb_sds_t config_key_region; - flb_sds_t config_key_sts_endpoint; - flb_sds_t config_key_role_arn; - flb_sds_t config_key_external_id; - flb_sds_t config_key_profile; - const char *region = NULL; - const char *sts_endpoint = NULL; - const char *role_arn = NULL; - const char *external_id = NULL; - const char *profile = NULL; - char *session_name = NULL; - int key_prefix_len; - int key_max_len; - - /* Provider managed dependencies */ - struct flb_aws_provider *aws_provider = NULL; - struct flb_aws_provider *base_aws_provider = NULL; - struct flb_tls *cred_tls = NULL; - struct flb_tls *sts_tls = NULL; - - /* Config keys */ - key_prefix_len = strlen(config_key_prefix); - key_max_len = key_prefix_len + 12; /* max length of - "region", "sts_endpoint", "role_arn", - "external_id" */ - - /* Evaluate full config keys */ - config_key_region = flb_sds_create_len(config_key_prefix, key_max_len); - strcpy(config_key_region + key_prefix_len, "region"); - config_key_sts_endpoint = flb_sds_create_len(config_key_prefix, key_max_len); - strcpy(config_key_sts_endpoint + key_prefix_len, "sts_endpoint"); - config_key_role_arn = flb_sds_create_len(config_key_prefix, key_max_len); - strcpy(config_key_role_arn + key_prefix_len, "role_arn"); - config_key_external_id = flb_sds_create_len(config_key_prefix, key_max_len); - strcpy(config_key_external_id + key_prefix_len, "external_id"); - config_key_profile = flb_sds_create_len(config_key_prefix, key_max_len); - strcpy(config_key_profile + key_prefix_len, "profile"); - - /* AWS provider needs a separate TLS instance */ - cred_tls = flb_tls_create(FLB_TLS_CLIENT_MODE, - FLB_TRUE, - ins->tls_debug, - ins->tls_vhost, - ins->tls_ca_path, - ins->tls_ca_file, - ins->tls_crt_file, - ins->tls_key_file, - ins->tls_key_passwd); - if (!cred_tls) { - flb_plg_error(ins, "Failed to create TLS instance for AWS Provider"); - flb_errno(); - goto error; - } - - region = flb_output_get_property(config_key_region, ins); - if (!region) { - flb_plg_error(ins, "aws_auth enabled but %s not set", config_key_region); - goto error; - } - - /* Use null sts_endpoint if none provided */ - sts_endpoint = flb_output_get_property(config_key_sts_endpoint, ins); - /* Get the profile from configuration */ - profile = flb_output_get_property(config_key_profile, ins); - aws_provider = flb_standard_chain_provider_create(config, - cred_tls, - (char *) region, - (char *) sts_endpoint, - NULL, - flb_aws_client_generator(), - profile); - if (!aws_provider) { - flb_plg_error(ins, "Failed to create AWS Credential Provider"); - goto error; - } - - role_arn = flb_output_get_property(config_key_role_arn, ins); - if (role_arn) { - /* Use the STS Provider */ - base_aws_provider = aws_provider; - external_id = flb_output_get_property(config_key_external_id, ins); - - session_name = flb_sts_session_name(); - if (!session_name) { - flb_plg_error(ins, "Failed to generate aws iam role " - "session name"); - goto error; - } - - /* STS provider needs yet another separate TLS instance */ - sts_tls = flb_tls_create(FLB_TLS_CLIENT_MODE, - FLB_TRUE, - ins->tls_debug, - ins->tls_vhost, - ins->tls_ca_path, - ins->tls_ca_file, - ins->tls_crt_file, - ins->tls_key_file, - ins->tls_key_passwd); - if (!sts_tls) { - flb_plg_error(ins, "Failed to create TLS instance for AWS STS Credential " - "Provider"); - flb_errno(); - goto error; - } - - aws_provider = flb_sts_provider_create(config, - sts_tls, - base_aws_provider, - (char *) external_id, - (char *) role_arn, - session_name, - (char *) region, - (char *) sts_endpoint, - NULL, - flb_aws_client_generator()); - if (!aws_provider) { - flb_plg_error(ins, "Failed to create AWS STS Credential " - "Provider"); - goto error; - } - } - - /* initialize credentials in sync mode */ - aws_provider->provider_vtable->sync(aws_provider); - aws_provider->provider_vtable->init(aws_provider); - - /* set back to async */ - aws_provider->provider_vtable->async(aws_provider); - - /* store dependencies in aws_provider for managed cleanup */ - aws_provider->base_aws_provider = base_aws_provider; - aws_provider->cred_tls = cred_tls; - aws_provider->sts_tls = sts_tls; - - goto cleanup; - -error: - if (aws_provider) { - /* disconnect dependencies */ - aws_provider->base_aws_provider = NULL; - aws_provider->cred_tls = NULL; - aws_provider->sts_tls = NULL; - /* destroy */ - flb_aws_provider_destroy(aws_provider); - } - /* free dependencies */ - if (base_aws_provider) { - flb_aws_provider_destroy(base_aws_provider); - } - if (cred_tls) { - flb_tls_destroy(cred_tls); - } - if (sts_tls) { - flb_tls_destroy(sts_tls); - } - aws_provider = NULL; - -cleanup: - if (config_key_region) { - flb_sds_destroy(config_key_region); - } - if (config_key_sts_endpoint) { - flb_sds_destroy(config_key_sts_endpoint); - } - if (config_key_role_arn) { - flb_sds_destroy(config_key_role_arn); - } - if (config_key_external_id) { - flb_sds_destroy(config_key_external_id); - } - if (session_name) { - flb_free(session_name); - } - - return aws_provider; -} - -static struct flb_aws_provider *standard_chain_create(struct flb_config - *config, - struct flb_tls *tls, - char *region, - char *sts_endpoint, - char *proxy, - struct - flb_aws_client_generator - *generator, - int eks_irsa, - char *profile) -{ - struct flb_aws_provider *sub_provider; - struct flb_aws_provider *provider; - struct flb_aws_provider_chain *implementation; - - provider = flb_calloc(1, sizeof(struct flb_aws_provider)); - - if (!provider) { - flb_errno(); - return NULL; - } - - pthread_mutex_init(&provider->lock, NULL); - - implementation = flb_calloc(1, sizeof(struct flb_aws_provider_chain)); - - if (!implementation) { - flb_errno(); - flb_free(provider); - return NULL; - } - - provider->provider_vtable = &standard_chain_provider_vtable; - provider->implementation = implementation; - - /* Create chain of providers */ - mk_list_init(&implementation->sub_providers); - - sub_provider = flb_aws_env_provider_create(); - if (!sub_provider) { - /* Env provider will only fail creation if a memory alloc failed */ - flb_aws_provider_destroy(provider); - return NULL; - } - flb_debug("[aws_credentials] Initialized Env Provider in standard chain"); - - mk_list_add(&sub_provider->_head, &implementation->sub_providers); - - flb_debug("[aws_credentials] creating profile %s provider", profile); - sub_provider = flb_profile_provider_create(profile); - if (sub_provider) { - /* Profile provider can fail if HOME env var is not set */; - mk_list_add(&sub_provider->_head, &implementation->sub_providers); - flb_debug("[aws_credentials] Initialized AWS Profile Provider in " - "standard chain"); - } - - if (eks_irsa == FLB_TRUE) { - sub_provider = flb_eks_provider_create(config, tls, region, sts_endpoint, proxy, generator); - if (sub_provider) { - /* EKS provider can fail if we are not running in k8s */; - mk_list_add(&sub_provider->_head, &implementation->sub_providers); - flb_debug("[aws_credentials] Initialized EKS Provider in standard chain"); - } - } - - sub_provider = flb_ecs_provider_create(config, generator); - if (sub_provider) { - /* ECS Provider will fail creation if we are not running in ECS */ - mk_list_add(&sub_provider->_head, &implementation->sub_providers); - flb_debug("[aws_credentials] Initialized ECS Provider in standard chain"); - } - - sub_provider = flb_ec2_provider_create(config, generator); - if (!sub_provider) { - /* EC2 provider will only fail creation if a memory alloc failed */ - flb_aws_provider_destroy(provider); - return NULL; - } - mk_list_add(&sub_provider->_head, &implementation->sub_providers); - flb_debug("[aws_credentials] Initialized EC2 Provider in standard chain"); - - return provider; -} - -/* Environment Provider */ -struct flb_aws_credentials *get_credentials_fn_environment(struct - flb_aws_provider - *provider) -{ - char *access_key = NULL; - char *secret_key = NULL; - char *session_token = NULL; - struct flb_aws_credentials *creds = NULL; - - flb_debug("[aws_credentials] Requesting credentials from the " - "env provider.."); - - access_key = getenv(AWS_ACCESS_KEY_ID); - if (!access_key || strlen(access_key) <= 0) { - return NULL; - } - - secret_key = getenv(AWS_SECRET_ACCESS_KEY); - if (!secret_key || strlen(secret_key) <= 0) { - return NULL; - } - - creds = flb_calloc(1, sizeof(struct flb_aws_credentials)); - if (!creds) { - flb_errno(); - return NULL; - } - - creds->access_key_id = flb_sds_create(access_key); - if (!creds->access_key_id) { - flb_aws_credentials_destroy(creds); - flb_errno(); - return NULL; - } - - creds->secret_access_key = flb_sds_create(secret_key); - if (!creds->secret_access_key) { - flb_aws_credentials_destroy(creds); - flb_errno(); - return NULL; - } - - session_token = getenv(AWS_SESSION_TOKEN); - if (session_token && strlen(session_token) > 0) { - creds->session_token = flb_sds_create(session_token); - if (!creds->session_token) { - flb_aws_credentials_destroy(creds); - flb_errno(); - return NULL; - } - } else { - creds->session_token = NULL; - } - - return creds; - -} - -int refresh_env(struct flb_aws_provider *provider) -{ - char *access_key = NULL; - char *secret_key = NULL; - - access_key = getenv(AWS_ACCESS_KEY_ID); - if (!access_key || strlen(access_key) <= 0) { - return -1; - } - - secret_key = getenv(AWS_SECRET_ACCESS_KEY); - if (!secret_key || strlen(secret_key) <= 0) { - return -1; - } - - return 0; -} - -/* - * For the env provider, refresh simply checks if the environment - * variables are available. - */ -int refresh_fn_environment(struct flb_aws_provider *provider) -{ - flb_debug("[aws_credentials] Refresh called on the env provider"); - - return refresh_env(provider); -} - -int init_fn_environment(struct flb_aws_provider *provider) -{ - flb_debug("[aws_credentials] Init called on the env provider"); - - return refresh_env(provider); -} - -/* - * sync and async are no-ops for the env provider because it does not make - * network IO calls - */ -void sync_fn_environment(struct flb_aws_provider *provider) -{ - return; -} - -void async_fn_environment(struct flb_aws_provider *provider) -{ - return; -} - -void upstream_set_fn_environment(struct flb_aws_provider *provider, - struct flb_output_instance *ins) -{ - return; -} - -/* Destroy is a no-op for the env provider */ -void destroy_fn_environment(struct flb_aws_provider *provider) { - return; -} - -static struct flb_aws_provider_vtable environment_provider_vtable = { - .get_credentials = get_credentials_fn_environment, - .init = init_fn_environment, - .refresh = refresh_fn_environment, - .destroy = destroy_fn_environment, - .sync = sync_fn_environment, - .async = async_fn_environment, - .upstream_set = upstream_set_fn_environment, -}; - -struct flb_aws_provider *flb_aws_env_provider_create() { - struct flb_aws_provider *provider = flb_calloc(1, sizeof( - struct flb_aws_provider)); - - if (!provider) { - flb_errno(); - return NULL; - } - - provider->provider_vtable = &environment_provider_vtable; - provider->implementation = NULL; - - return provider; -} - - -void flb_aws_credentials_destroy(struct flb_aws_credentials *creds) -{ - if (creds) { - if (creds->access_key_id) { - flb_sds_destroy(creds->access_key_id); - } - if (creds->secret_access_key) { - flb_sds_destroy(creds->secret_access_key); - } - if (creds->session_token) { - flb_sds_destroy(creds->session_token); - } - - flb_free(creds); - } -} - -void flb_aws_provider_destroy(struct flb_aws_provider *provider) -{ - if (provider) { - if (provider->implementation) { - provider->provider_vtable->destroy(provider); - } - - pthread_mutex_destroy(&provider->lock); - - /* free managed dependencies */ - if (provider->base_aws_provider) { - flb_aws_provider_destroy(provider->base_aws_provider); - } - if (provider->cred_tls) { - flb_tls_destroy(provider->cred_tls); - } - if (provider->sts_tls) { - flb_tls_destroy(provider->sts_tls); - } - - flb_free(provider); - } -} - -time_t timestamp_to_epoch(const char *timestamp) -{ - struct tm tm = {0}; - time_t seconds; - int r; - - r = sscanf(timestamp, "%d-%d-%dT%d:%d:%dZ", &tm.tm_year, &tm.tm_mon, - &tm.tm_mday, &tm.tm_hour, &tm.tm_min, &tm.tm_sec); - if (r != 6) { - return -1; - } - - tm.tm_year -= 1900; - tm.tm_mon -= 1; - tm.tm_isdst = -1; - seconds = timegm(&tm); - if (seconds < 0) { - return -1; - } - - return seconds; -} - -time_t flb_aws_cred_expiration(const char *timestamp) -{ - time_t now; - time_t expiration = timestamp_to_epoch(timestamp); - if (expiration < 0) { - flb_warn("[aws_credentials] Could not parse expiration: %s", timestamp); - return -1; - } - /* - * Sanity check - expiration should be ~10 minutes to 12 hours in the future - * (> 12 hours is impossible with the current APIs and would likely indicate - * a bug in how this code processes timestamps.) - */ - now = time(NULL); - if (expiration < (now + FIVE_MINUTES)) { - flb_warn("[aws_credentials] Credential expiration '%s' is less than " - "5 minutes in the future.", - timestamp); - } - if (expiration > (now + TWELVE_HOURS)) { - flb_warn("[aws_credentials] Credential expiration '%s' is greater than " - "12 hours in the future. This should not be possible.", - timestamp); - } - return expiration; -} - -/* - * Fluent Bit is now multi-threaded and asynchonous with coros. - * The trylock prevents deadlock, and protects the provider - * when a cred refresh happens. The refresh frees and - * sets the shared cred cache, a double free could occur - * if two threads do it at the same exact time. - */ - -/* Like a traditional try lock- it does not block if the lock is not obtained */ -int try_lock_provider(struct flb_aws_provider *provider) -{ - int ret = 0; - ret = pthread_mutex_trylock(&provider->lock); - if (ret != 0) { - return FLB_FALSE; - } - return FLB_TRUE; -} - -void unlock_provider(struct flb_aws_provider *provider) -{ - pthread_mutex_unlock(&provider->lock); -} diff --git a/fluent-bit/src/aws/flb_aws_credentials_ec2.c b/fluent-bit/src/aws/flb_aws_credentials_ec2.c deleted file mode 100644 index 5d2d8515c..000000000 --- a/fluent-bit/src/aws/flb_aws_credentials_ec2.c +++ /dev/null @@ -1,371 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <fluent-bit/flb_info.h> -#include <fluent-bit/flb_sds.h> -#include <fluent-bit/flb_http_client.h> -#include <fluent-bit/flb_aws_credentials.h> -#include <fluent-bit/flb_aws_util.h> -#include <fluent-bit/flb_jsmn.h> -#include <fluent-bit/aws/flb_aws_imds.h> - -#include <stdlib.h> -#include <time.h> -#include <sys/types.h> -#include <sys/stat.h> - -#define AWS_IMDS_ROLE_PATH "/latest/meta-data/iam/security-credentials/" -#define AWS_IMDS_ROLE_PATH_LEN 43 - -struct flb_aws_provider_ec2; -static int get_creds_ec2(struct flb_aws_provider_ec2 *implementation); -static int ec2_credentials_request(struct flb_aws_provider_ec2 - *implementation, char *cred_path); - -/* EC2 IMDS Provider */ - -/* - * A provider that obtains credentials from EC2 IMDS. - */ -struct flb_aws_provider_ec2 { - struct flb_aws_credentials *creds; - time_t next_refresh; - - /* upstream connection to IMDS */ - struct flb_aws_client *client; - - /* IMDS interface */ - struct flb_aws_imds *imds_interface; -}; - -struct flb_aws_credentials *get_credentials_fn_ec2(struct flb_aws_provider - *provider) -{ - struct flb_aws_credentials *creds; - int refresh = FLB_FALSE; - struct flb_aws_provider_ec2 *implementation = provider->implementation; - - flb_debug("[aws_credentials] Requesting credentials from the " - "EC2 provider.."); - - /* a negative next_refresh means that auto-refresh is disabled */ - if (implementation->next_refresh > 0 - && time(NULL) > implementation->next_refresh) { - refresh = FLB_TRUE; - } - if (!implementation->creds || refresh == FLB_TRUE) { - if (try_lock_provider(provider)) { - get_creds_ec2(implementation); - unlock_provider(provider); - } - } - - if (!implementation->creds) { - /* - * We failed to lock the provider and creds are unset. This means that - * another co-routine is performing the refresh. - */ - flb_warn("[aws_credentials] No cached credentials are available and " - "a credential refresh is already in progress. The current " - "co-routine will retry."); - - return NULL; - } - - creds = flb_calloc(1, sizeof(struct flb_aws_credentials)); - if (!creds) { - flb_errno(); - return NULL; - } - - creds->access_key_id = flb_sds_create(implementation->creds->access_key_id); - if (!creds->access_key_id) { - flb_errno(); - flb_aws_credentials_destroy(creds); - return NULL; - } - - creds->secret_access_key = flb_sds_create(implementation->creds-> - secret_access_key); - if (!creds->secret_access_key) { - flb_errno(); - flb_aws_credentials_destroy(creds); - return NULL; - } - - if (implementation->creds->session_token) { - creds->session_token = flb_sds_create(implementation->creds-> - session_token); - if (!creds->session_token) { - flb_errno(); - flb_aws_credentials_destroy(creds); - return NULL; - } - - } else { - creds->session_token = NULL; - } - - return creds; -} - -int refresh_fn_ec2(struct flb_aws_provider *provider) { - struct flb_aws_provider_ec2 *implementation = provider->implementation; - int ret = -1; - - flb_debug("[aws_credentials] Refresh called on the EC2 IMDS provider"); - if (try_lock_provider(provider)) { - ret = get_creds_ec2(implementation); - unlock_provider(provider); - } - return ret; -} - -int init_fn_ec2(struct flb_aws_provider *provider) { - struct flb_aws_provider_ec2 *implementation = provider->implementation; - int ret = -1; - - implementation->client->debug_only = FLB_TRUE; - - flb_debug("[aws_credentials] Init called on the EC2 IMDS provider"); - if (try_lock_provider(provider)) { - ret = get_creds_ec2(implementation); - unlock_provider(provider); - } - - implementation->client->debug_only = FLB_FALSE; - return ret; -} - -void sync_fn_ec2(struct flb_aws_provider *provider) { - struct flb_aws_provider_ec2 *implementation = provider->implementation; - - flb_debug("[aws_credentials] Sync called on the EC2 provider"); - /* remove async flag */ - flb_stream_disable_async_mode(&implementation->client->upstream->base); -} - -void async_fn_ec2(struct flb_aws_provider *provider) { - struct flb_aws_provider_ec2 *implementation = provider->implementation; - - flb_debug("[aws_credentials] Async called on the EC2 provider"); - /* add async flag */ - flb_stream_enable_async_mode(&implementation->client->upstream->base); -} - -void upstream_set_fn_ec2(struct flb_aws_provider *provider, - struct flb_output_instance *ins) { - struct flb_aws_provider_ec2 *implementation = provider->implementation; - - flb_debug("[aws_credentials] upstream_set called on the EC2 provider"); - /* Make sure TLS is set to false before setting upstream, then reset it */ - ins->use_tls = FLB_FALSE; - flb_output_upstream_set(implementation->client->upstream, ins); - ins->use_tls = FLB_TRUE; -} - -void destroy_fn_ec2(struct flb_aws_provider *provider) { - struct flb_aws_provider_ec2 *implementation = provider->implementation; - - if (implementation) { - if (implementation->creds) { - flb_aws_credentials_destroy(implementation->creds); - } - - if (implementation->imds_interface) { - flb_aws_imds_destroy(implementation->imds_interface); - } - - if (implementation->client) { - flb_aws_client_destroy(implementation->client); - } - - flb_free(implementation); - provider->implementation = NULL; - } - - return; -} - -static struct flb_aws_provider_vtable ec2_provider_vtable = { - .get_credentials = get_credentials_fn_ec2, - .init = init_fn_ec2, - .refresh = refresh_fn_ec2, - .destroy = destroy_fn_ec2, - .sync = sync_fn_ec2, - .async = async_fn_ec2, - .upstream_set = upstream_set_fn_ec2, -}; - -struct flb_aws_provider *flb_ec2_provider_create(struct flb_config *config, - struct - flb_aws_client_generator - *generator) -{ - struct flb_aws_provider_ec2 *implementation; - struct flb_aws_provider *provider; - struct flb_upstream *upstream; - - provider = flb_calloc(1, sizeof(struct flb_aws_provider)); - - if (!provider) { - flb_errno(); - return NULL; - } - - pthread_mutex_init(&provider->lock, NULL); - - implementation = flb_calloc(1, sizeof(struct flb_aws_provider_ec2)); - - if (!implementation) { - flb_free(provider); - flb_errno(); - return NULL; - } - - provider->provider_vtable = &ec2_provider_vtable; - provider->implementation = implementation; - - upstream = flb_upstream_create(config, FLB_AWS_IMDS_HOST, FLB_AWS_IMDS_PORT, - FLB_IO_TCP, NULL); - if (!upstream) { - flb_aws_provider_destroy(provider); - flb_debug("[aws_credentials] unable to connect to EC2 IMDS."); - return NULL; - } - - /* IMDSv2 token request will timeout if hops = 1 and running within container */ - upstream->base.net.connect_timeout = FLB_AWS_IMDS_TIMEOUT; - upstream->base.net.io_timeout = FLB_AWS_IMDS_TIMEOUT; - upstream->base.net.keepalive = FLB_FALSE; /* On timeout, the connection is broken */ - - implementation->client = generator->create(); - if (!implementation->client) { - flb_aws_provider_destroy(provider); - flb_upstream_destroy(upstream); - flb_error("[aws_credentials] EC2 IMDS: client creation error"); - return NULL; - } - implementation->client->name = "ec2_imds_provider_client"; - implementation->client->has_auth = FLB_FALSE; - implementation->client->provider = NULL; - implementation->client->region = NULL; - implementation->client->service = NULL; - implementation->client->port = 80; - implementation->client->flags = 0; - implementation->client->proxy = NULL; - implementation->client->upstream = upstream; - - /* Use default imds configuration */ - implementation->imds_interface = flb_aws_imds_create(&flb_aws_imds_config_default, - implementation->client); - if (!implementation->imds_interface) { - flb_aws_provider_destroy(provider); - flb_error("[aws_credentials] EC2 IMDS configuration error"); - return NULL; - } - - return provider; -} - -/* Requests creds from IMDSv1 and sets them on the provider */ -static int get_creds_ec2(struct flb_aws_provider_ec2 *implementation) -{ - int ret; - flb_sds_t instance_role; - size_t instance_role_len; - char *cred_path; - size_t cred_path_size; - - flb_debug("[aws_credentials] requesting credentials from EC2 IMDS"); - - /* Get the name of the instance role */ - ret = flb_aws_imds_request(implementation->imds_interface, AWS_IMDS_ROLE_PATH, - &instance_role, &instance_role_len); - - if (ret < 0) { - return -1; - } - - flb_debug("[aws_credentials] Requesting credentials for instance role %s", - instance_role); - - /* Construct path where we will find the credentials */ - cred_path_size = sizeof(char) * (AWS_IMDS_ROLE_PATH_LEN + - instance_role_len) + 1; - cred_path = flb_malloc(cred_path_size); - if (!cred_path) { - flb_sds_destroy(instance_role); - flb_errno(); - return -1; - } - - ret = snprintf(cred_path, cred_path_size, "%s%s", AWS_IMDS_ROLE_PATH, - instance_role); - if (ret < 0) { - flb_sds_destroy(instance_role); - flb_free(cred_path); - flb_errno(); - return -1; - } - - /* request creds */ - ret = ec2_credentials_request(implementation, cred_path); - - flb_sds_destroy(instance_role); - flb_free(cred_path); - return ret; - -} - -static int ec2_credentials_request(struct flb_aws_provider_ec2 - *implementation, char *cred_path) -{ - int ret; - flb_sds_t credentials_response; - size_t credentials_response_len; - struct flb_aws_credentials *creds; - time_t expiration; - - ret = flb_aws_imds_request(implementation->imds_interface, cred_path, - &credentials_response, &credentials_response_len); - - if (ret < 0) { - return -1; - } - - creds = flb_parse_http_credentials(credentials_response, - credentials_response_len, - &expiration); - - if (creds == NULL) { - flb_sds_destroy(credentials_response); - return -1; - } - - /* destroy existing credentials first */ - flb_aws_credentials_destroy(implementation->creds); - implementation->creds = NULL; - /* set new creds */ - implementation->creds = creds; - implementation->next_refresh = expiration - FLB_AWS_REFRESH_WINDOW; - - flb_sds_destroy(credentials_response); - return 0; -} diff --git a/fluent-bit/src/aws/flb_aws_credentials_http.c b/fluent-bit/src/aws/flb_aws_credentials_http.c deleted file mode 100644 index c08b6b559..000000000 --- a/fluent-bit/src/aws/flb_aws_credentials_http.c +++ /dev/null @@ -1,566 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <fluent-bit/flb_info.h> -#include <fluent-bit/flb_sds.h> -#include <fluent-bit/flb_http_client.h> -#include <fluent-bit/flb_aws_credentials.h> -#include <fluent-bit/flb_aws_util.h> - -#include <fluent-bit/flb_jsmn.h> -#include <stdlib.h> -#include <time.h> -#include <sys/types.h> -#include <sys/stat.h> - -#define AWS_CREDENTIAL_RESPONSE_ACCESS_KEY "AccessKeyId" -#define AWS_CREDENTIAL_RESPONSE_SECRET_KEY "SecretAccessKey" -#define AWS_HTTP_RESPONSE_TOKEN "Token" -#define AWS_CREDENTIAL_RESPONSE_EXPIRATION "Expiration" - -#define ECS_CREDENTIALS_HOST "169.254.170.2" -#define ECS_CREDENTIALS_HOST_LEN 13 -#define ECS_CREDENTIALS_PATH_ENV_VAR "AWS_CONTAINER_CREDENTIALS_RELATIVE_URI" - - -/* Declarations */ -struct flb_aws_provider_http; -static int http_credentials_request(struct flb_aws_provider_http - *implementation); - - -/* - * HTTP Credentials Provider - retrieve credentials from a local http server - * Used to implement the ECS Credentials provider. - * Equivalent to: - * https://github.com/aws/aws-sdk-go/tree/master/aws/credentials/endpointcreds - */ - -struct flb_aws_provider_http { - struct flb_aws_credentials *creds; - time_t next_refresh; - - struct flb_aws_client *client; - - /* Host and Path to request credentials */ - flb_sds_t host; - flb_sds_t path; -}; - - -struct flb_aws_credentials *get_credentials_fn_http(struct flb_aws_provider - *provider) -{ - struct flb_aws_credentials *creds = NULL; - int refresh = FLB_FALSE; - struct flb_aws_provider_http *implementation = provider->implementation; - - flb_debug("[aws_credentials] Retrieving credentials from the " - "HTTP provider.."); - - /* a negative next_refresh means that auto-refresh is disabled */ - if (implementation->next_refresh > 0 - && time(NULL) > implementation->next_refresh) { - refresh = FLB_TRUE; - } - if (!implementation->creds || refresh == FLB_TRUE) { - if (try_lock_provider(provider)) { - http_credentials_request(implementation); - unlock_provider(provider); - } - } - - if (!implementation->creds) { - /* - * We failed to lock the provider and creds are unset. This means that - * another co-routine is performing the refresh. - */ - flb_warn("[aws_credentials] No cached credentials are available and " - "a credential refresh is already in progress. The current " - "co-routine will retry."); - - return NULL; - } - - creds = flb_calloc(1, sizeof(struct flb_aws_credentials)); - if (!creds) { - flb_errno(); - goto error; - } - - creds->access_key_id = flb_sds_create(implementation->creds->access_key_id); - if (!creds->access_key_id) { - flb_errno(); - goto error; - } - - creds->secret_access_key = flb_sds_create(implementation->creds-> - secret_access_key); - if (!creds->secret_access_key) { - flb_errno(); - goto error; - } - - if (implementation->creds->session_token) { - creds->session_token = flb_sds_create(implementation->creds-> - session_token); - if (!creds->session_token) { - flb_errno(); - goto error; - } - - } else { - creds->session_token = NULL; - } - - return creds; - -error: - flb_aws_credentials_destroy(creds); - return NULL; -} - -int refresh_fn_http(struct flb_aws_provider *provider) { - struct flb_aws_provider_http *implementation = provider->implementation; - int ret = -1; - flb_debug("[aws_credentials] Refresh called on the http provider"); - - if (try_lock_provider(provider)) { - ret = http_credentials_request(implementation); - unlock_provider(provider); - } - return ret; -} - -int init_fn_http(struct flb_aws_provider *provider) { - struct flb_aws_provider_http *implementation = provider->implementation; - int ret = -1; - flb_debug("[aws_credentials] Init called on the http provider"); - - implementation->client->debug_only = FLB_TRUE; - - if (try_lock_provider(provider)) { - ret = http_credentials_request(implementation); - unlock_provider(provider); - } - - implementation->client->debug_only = FLB_FALSE; - - return ret; -} - -void sync_fn_http(struct flb_aws_provider *provider) { - struct flb_aws_provider_http *implementation = provider->implementation; - - flb_debug("[aws_credentials] Sync called on the http provider"); - /* remove async flag */ - flb_stream_disable_async_mode(&implementation->client->upstream->base); -} - -void async_fn_http(struct flb_aws_provider *provider) { - struct flb_aws_provider_http *implementation = provider->implementation; - - flb_debug("[aws_credentials] Async called on the http provider"); - /* add async flag */ - flb_stream_enable_async_mode(&implementation->client->upstream->base); -} - -void upstream_set_fn_http(struct flb_aws_provider *provider, - struct flb_output_instance *ins) { - struct flb_aws_provider_http *implementation = provider->implementation; - - flb_debug("[aws_credentials] upstream_set called on the http provider"); - /* Make sure TLS is set to false before setting upstream, then reset it */ - ins->use_tls = FLB_FALSE; - flb_output_upstream_set(implementation->client->upstream, ins); - ins->use_tls = FLB_TRUE; -} - -void destroy_fn_http(struct flb_aws_provider *provider) { - struct flb_aws_provider_http *implementation = provider->implementation; - - if (implementation) { - if (implementation->creds) { - flb_aws_credentials_destroy(implementation->creds); - } - - if (implementation->client) { - flb_aws_client_destroy(implementation->client); - } - - if (implementation->host) { - flb_sds_destroy(implementation->host); - } - - if (implementation->path) { - flb_sds_destroy(implementation->path); - } - - flb_free(implementation); - provider->implementation = NULL; - } - - return; -} - -static struct flb_aws_provider_vtable http_provider_vtable = { - .get_credentials = get_credentials_fn_http, - .init = init_fn_http, - .refresh = refresh_fn_http, - .destroy = destroy_fn_http, - .sync = sync_fn_http, - .async = async_fn_http, - .upstream_set = upstream_set_fn_http, -}; - -struct flb_aws_provider *flb_http_provider_create(struct flb_config *config, - flb_sds_t host, - flb_sds_t path, - struct - flb_aws_client_generator - *generator) -{ - struct flb_aws_provider_http *implementation = NULL; - struct flb_aws_provider *provider = NULL; - struct flb_upstream *upstream = NULL; - - flb_debug("[aws_credentials] Configuring HTTP provider with %s:80%s", - host, path); - - provider = flb_calloc(1, sizeof(struct flb_aws_provider)); - - if (!provider) { - flb_errno(); - return NULL; - } - - pthread_mutex_init(&provider->lock, NULL); - - implementation = flb_calloc(1, sizeof(struct flb_aws_provider_http)); - - if (!implementation) { - flb_free(provider); - flb_errno(); - return NULL; - } - - provider->provider_vtable = &http_provider_vtable; - provider->implementation = implementation; - - implementation->host = host; - implementation->path = path; - - upstream = flb_upstream_create(config, host, 80, FLB_IO_TCP, NULL); - - if (!upstream) { - flb_aws_provider_destroy(provider); - flb_error("[aws_credentials] HTTP Provider: connection initialization " - "error"); - return NULL; - } - - upstream->base.net.connect_timeout = FLB_AWS_CREDENTIAL_NET_TIMEOUT; - - implementation->client = generator->create(); - if (!implementation->client) { - flb_aws_provider_destroy(provider); - flb_upstream_destroy(upstream); - flb_error("[aws_credentials] HTTP Provider: client creation error"); - return NULL; - } - implementation->client->name = "http_provider_client"; - implementation->client->has_auth = FLB_FALSE; - implementation->client->provider = NULL; - implementation->client->region = NULL; - implementation->client->service = NULL; - implementation->client->port = 80; - implementation->client->flags = 0; - implementation->client->proxy = NULL; - implementation->client->upstream = upstream; - - return provider; -} - -/* - * ECS Provider - * The ECS Provider is just a wrapper around the HTTP Provider - * with the ECS credentials endpoint. - */ - - struct flb_aws_provider *flb_ecs_provider_create(struct flb_config *config, - struct - flb_aws_client_generator - *generator) -{ - flb_sds_t host = NULL; - flb_sds_t path = NULL; - char *path_var = NULL; - - host = flb_sds_create_len(ECS_CREDENTIALS_HOST, ECS_CREDENTIALS_HOST_LEN); - if (!host) { - flb_errno(); - return NULL; - } - - path_var = getenv(ECS_CREDENTIALS_PATH_ENV_VAR); - if (path_var && strlen(path_var) > 0) { - path = flb_sds_create(path_var); - if (!path) { - flb_errno(); - flb_free(host); - return NULL; - } - - return flb_http_provider_create(config, host, path, generator); - } else { - flb_debug("[aws_credentials] Not initializing ECS Provider because" - " %s is not set", ECS_CREDENTIALS_PATH_ENV_VAR); - flb_sds_destroy(host); - return NULL; - } - -} - -static int http_credentials_request(struct flb_aws_provider_http - *implementation) -{ - char *response = NULL; - size_t response_len; - time_t expiration; - struct flb_aws_credentials *creds = NULL; - struct flb_aws_client *client = implementation->client; - struct flb_http_client *c = NULL; - - c = client->client_vtable->request(client, FLB_HTTP_GET, - implementation->path, NULL, 0, - NULL, 0); - - if (!c || c->resp.status != 200) { - flb_debug("[aws_credentials] http credentials request failed"); - if (c) { - flb_http_client_destroy(c); - } - return -1; - } - - response = c->resp.payload; - response_len = c->resp.payload_size; - - creds = flb_parse_http_credentials(response, response_len, &expiration); - if (!creds) { - flb_http_client_destroy(c); - return -1; - } - - /* destroy existing credentials */ - flb_aws_credentials_destroy(implementation->creds); - implementation->creds = NULL; - - implementation->creds = creds; - implementation->next_refresh = expiration - FLB_AWS_REFRESH_WINDOW; - flb_http_client_destroy(c); - return 0; -} - -/* - * All HTTP credentials endpoints (IMDS, ECS, custom) follow the same spec: - * { - * "AccessKeyId": "ACCESS_KEY_ID", - * "Expiration": "2019-12-18T21:27:58Z", - * "SecretAccessKey": "SECRET_ACCESS_KEY", - * "Token": "SECURITY_TOKEN_STRING" - * } - * (some implementations (IMDS) have additional fields) - * Returns NULL if any part of parsing was unsuccessful. - */ -struct flb_aws_credentials *flb_parse_http_credentials(char *response, - size_t response_len, - time_t *expiration) -{ - return flb_parse_json_credentials(response, response_len, AWS_HTTP_RESPONSE_TOKEN, - expiration); -} - -struct flb_aws_credentials *flb_parse_json_credentials(char *response, - size_t response_len, - char* session_token_field, - time_t *expiration) -{ - jsmntok_t *tokens = NULL; - const jsmntok_t *t = NULL; - char *current_token = NULL; - jsmn_parser parser; - int tokens_size = 50; - size_t size; - int ret; - struct flb_aws_credentials *creds = NULL; - int i = 0; - int len; - flb_sds_t tmp; - - /* - * Remove/reset existing value of expiration. - * Expiration should be in the response, but it is not - * strictly speaking needed. Fluent Bit logs a warning if it is missing. - */ - *expiration = -1; - - jsmn_init(&parser); - - size = sizeof(jsmntok_t) * tokens_size; - tokens = flb_calloc(1, size); - if (!tokens) { - goto error; - } - - ret = jsmn_parse(&parser, response, response_len, - tokens, tokens_size); - - if (ret == JSMN_ERROR_INVAL || ret == JSMN_ERROR_PART) { - flb_error("[aws_credentials] Could not parse credentials response" - " - invalid JSON."); - goto error; - } - - /* Shouldn't happen, but just in case, check for too many tokens error */ - if (ret == JSMN_ERROR_NOMEM) { - flb_error("[aws_credentials] Could not parse credentials response" - " - response contained more tokens than expected."); - goto error; - } - - /* return value is number of tokens parsed */ - tokens_size = ret; - - creds = flb_calloc(1, sizeof(struct flb_aws_credentials)); - if (!creds) { - flb_errno(); - goto error; - } - - /* - * jsmn will create an array of tokens like: - * key, value, key, value - */ - while (i < (tokens_size - 1)) { - t = &tokens[i]; - - if (t->start == -1 || t->end == -1 || (t->start == 0 && t->end == 0)) { - break; - } - - if (t->type == JSMN_STRING) { - current_token = &response[t->start]; - len = t->end - t->start; - - if (strncmp(current_token, AWS_CREDENTIAL_RESPONSE_ACCESS_KEY, len) == 0) - { - i++; - t = &tokens[i]; - current_token = &response[t->start]; - len = t->end - t->start; - if (creds->access_key_id != NULL) { - flb_error("Trying to double allocate access_key_id"); - goto error; - } - creds->access_key_id = flb_sds_create_len(current_token, len); - if (!creds->access_key_id) { - flb_errno(); - goto error; - } - continue; - } - if (strncmp(current_token, AWS_CREDENTIAL_RESPONSE_SECRET_KEY, len) == 0) - { - i++; - t = &tokens[i]; - current_token = &response[t->start]; - len = t->end - t->start; - if (creds->secret_access_key != NULL) { - flb_error("Trying to double allocate secret_access_key"); - goto error; - } - creds->secret_access_key = flb_sds_create_len(current_token, - len); - if (!creds->secret_access_key) { - flb_errno(); - goto error; - } - continue; - } - if (strncmp(current_token, session_token_field, len) == 0) { - i++; - t = &tokens[i]; - current_token = &response[t->start]; - len = t->end - t->start; - if (creds->session_token != NULL) { - flb_error("Trying to double allocate session_token"); - goto error; - } - creds->session_token = flb_sds_create_len(current_token, len); - if (!creds->session_token) { - flb_errno(); - goto error; - } - continue; - } - if (strncmp(current_token, AWS_CREDENTIAL_RESPONSE_EXPIRATION, len) == 0) - { - i++; - t = &tokens[i]; - current_token = &response[t->start]; - len = t->end - t->start; - tmp = flb_sds_create_len(current_token, len); - if (!tmp) { - flb_errno(); - goto error; - } - *expiration = flb_aws_cred_expiration(tmp); - flb_sds_destroy(tmp); - if (*expiration < 0) { - flb_warn("[aws_credentials] '%s' was invalid or " - "could not be parsed. Disabling auto-refresh of " - "credentials.", AWS_CREDENTIAL_RESPONSE_EXPIRATION); - } - } - } - - i++; - } - - if (creds->access_key_id == NULL) { - flb_error("[aws_credentials] Missing %s field in" - "credentials response", AWS_CREDENTIAL_RESPONSE_ACCESS_KEY); - goto error; - } - - if (creds->secret_access_key == NULL) { - flb_error("[aws_credentials] Missing %s field in" - "credentials response", AWS_CREDENTIAL_RESPONSE_SECRET_KEY); - goto error; - } - - flb_free(tokens); - return creds; - -error: - flb_aws_credentials_destroy(creds); - flb_free(tokens); - return NULL; -} diff --git a/fluent-bit/src/aws/flb_aws_credentials_log.h b/fluent-bit/src/aws/flb_aws_credentials_log.h deleted file mode 100644 index 6e9f0806d..000000000 --- a/fluent-bit/src/aws/flb_aws_credentials_log.h +++ /dev/null @@ -1,39 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2021 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_AWS_CREDENTIALS_LOG_H - -#define FLB_AWS_CREDENTIALS_LOG_H - -#include <fluent-bit/flb_log.h> - -#define AWS_CREDS_ERROR(format, ...) flb_error("[aws_credentials] " format, ##__VA_ARGS__) -#define AWS_CREDS_WARN(format, ...) flb_warn("[aws_credentials] " format, ##__VA_ARGS__) -#define AWS_CREDS_DEBUG(format, ...) flb_debug("[aws_credentials] " format, ##__VA_ARGS__) - -#define AWS_CREDS_ERROR_OR_DEBUG(debug_only, format, ...) do {\ - if (debug_only == FLB_TRUE) {\ - AWS_CREDS_DEBUG(format, ##__VA_ARGS__);\ - }\ - else {\ - AWS_CREDS_ERROR(format, ##__VA_ARGS__);\ - }\ -} while (0) - -#endif diff --git a/fluent-bit/src/aws/flb_aws_credentials_process.c b/fluent-bit/src/aws/flb_aws_credentials_process.c deleted file mode 100644 index 44c024ca7..000000000 --- a/fluent-bit/src/aws/flb_aws_credentials_process.c +++ /dev/null @@ -1,783 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2021 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <fluent-bit/flb_aws_credentials.h> - -#include "flb_aws_credentials_log.h" - -#include <fluent-bit/flb_compat.h> -#include <fluent-bit/flb_info.h> -#include <fluent-bit/flb_pipe.h> -#include <fluent-bit/flb_time.h> - -#include <fcntl.h> -#include <poll.h> -#include <stdlib.h> -#include <sys/wait.h> - -#define DEV_NULL "/dev/null" - -#define MS_PER_SEC 1000 -#define MICROS_PER_MS 1000 -#define NS_PER_MS 1000000 - -#define CREDENTIAL_PROCESS_TIMEOUT_MS 60000 -#define CREDENTIAL_PROCESS_BUFFER_SIZE 8 * 1024 - -#define WAITPID_POLL_FREQUENCY_MS 20 -#define WAITPID_TIMEOUT_MS 10 * WAITPID_POLL_FREQUENCY_MS - -#define CREDENTIAL_PROCESS_RESPONSE_SESSION_TOKEN "SessionToken" - -/* Declarations */ -struct token_array; -static int new_token_array(struct token_array *arr, int cap); -static int append_token(struct token_array *arr, char* elem); - -struct readbuf; -static int new_readbuf(struct readbuf* buf, int cap); - -static int get_monotonic_time(struct flb_time* tm); - -static char* ltrim(char* input); -static int scan_credential_process_token_quoted(char *input); -static int scan_credential_process_token_unquoted(char *input); -static int credential_process_token_count(char* process); -static int parse_credential_process_token(char **input, char** out_token); - -static int read_until_block(char* name, flb_pipefd_t fd, struct readbuf* buf); -static int waitpid_timeout(char* name, pid_t pid, int* wstatus); - -struct process; -static int new_process(struct process* p, char** args); -static void exec_process_child(struct process* p); -static int exec_process(struct process* p); -static int read_from_process(struct process* p, struct readbuf* buf); -static int wait_process(struct process* p); -static void destroy_process(struct process* p); -/* End Declarations */ - -struct token_array { - char** tokens; - int len; - int cap; -}; - -/* - * Initializes a new token array with the given capacity. - * Returns 0 on success and < 0 on failure. - * The caller is responsible for calling `flb_free(arr->tokens)`. - */ -static int new_token_array(struct token_array *arr, int cap) -{ - *arr = (struct token_array) { .len = 0, .cap = cap }; - arr->tokens = flb_malloc(cap * sizeof(char*)); - if (!arr->tokens) { - flb_errno(); - return -1; - } - return 0; -} - -/* - * Appends the given token to the array, if there is capacity. - * Returns 0 on success and < 0 on failure. - */ -static int append_token(struct token_array *arr, char* token) -{ - if (arr->len >= arr->cap) { - /* This means there is a bug in credential_process_token_count. */ - AWS_CREDS_ERROR("append_token called on full token_array"); - return -1; - } - - (arr->tokens)[arr->len] = token; - arr->len++; - return 0; -} - -struct readbuf { - char* buf; - int len; - int cap; -}; - -/* - * Initializes a new buffer with the given capacity. - * Returns 0 on success and < 0 on failure. - * The caller is responsible for calling `flb_free(buf->buf)`. - */ -static int new_readbuf(struct readbuf* buf, int cap) -{ - *buf = (struct readbuf) { .len = 0, .cap = cap }; - buf->buf = flb_malloc(cap * sizeof(char)); - if (!buf->buf) { - flb_errno(); - return -1; - } - return 0; -} - -/* - * Fetches the current time from the monotonic clock. - * Returns 0 on success and < 0 on failure. - * This is useful for calculating deadlines that are not sensitive to changes - * in the system clock. - */ -static int get_monotonic_time(struct flb_time* tm) -{ - struct timespec ts; - if (clock_gettime(CLOCK_MONOTONIC, &ts) < 0) { - flb_errno(); - return -1; - } - flb_time_set(tm, ts.tv_sec, ts.tv_nsec); - return 0; -} - -/* - * Skips over any leading spaces in the input string, returning the remainder. - * If the entire string is consumed, returns the empty string (not NULL). - */ -static char* ltrim(char* input) -{ - while (*input == ' ') { - input++; - } - return input; -} - -/* - * Scans the unquoted token string at the start of the input string. - * The input must be the start of an unquoted token. - * Returns the token length on success, and < 0 on failure. - * This function does not add a null terminator to the token. - * The token length is the index where the null terminator must be placed. - * If the entire input is consumed, returns the length of the input string - * (excluding the null terminator). - */ -static int scan_credential_process_token_unquoted(char *input) -{ - int i; - - for (i = 0; input[i] != ' '; i++) { - if (input[i] == '\0') { - break; - } - if (input[i] == '"') { - AWS_CREDS_ERROR("unexpected quote in credential_process"); - return -1; - } - } - - return i; -} - -/* - * Scans the quoted token at the start of the input string. - * The input must be the string after the opening quote. - * Returns the token length on success, and < 0 on failure. - * This function does not add a null terminator to the token. - * The token length is the index where the null terminator must be placed. - */ -static int scan_credential_process_token_quoted(char *input) -{ - int i; - - for (i = 0; input[i] != '"'; i++) { - if (input[i] == '\0') { - AWS_CREDS_ERROR("unterminated quote in credential_process"); - return -1; - } - } - - if (input[i+1] != '\0' && input[i+1] != ' ') { - AWS_CREDS_ERROR("unexpected character %c after closing quote in " - "credential_process", input[i+1]); - return -1; - } - - return i; -} - -/* - * Counts the number of tokens in the input string, which is assumed to be the - * credential_process from the config file. - * Returns < 0 on failure. - */ -static int credential_process_token_count(char* process) -{ - int count = 0; - int i; - - while (1) { - process = ltrim(process); - if (*process == '\0') { - break; - } - - count++; - - if (*process == '"') { - process++; - i = scan_credential_process_token_quoted(process); - } - else { - i = scan_credential_process_token_unquoted(process); - } - - if (i < 0) { - return -1; - } - - process += i; - if (*process != '\0') { - process++; - } - } - - return count; -} - -/* - * Parses the input string, which is assumed to be the credential_process - * from the config file. The next token will be put in *out_token, and the - * remaining unprocessed input will be put in *input. - * Returns 0 on success and < 0 on failure. - * If there is an error, the value of *input and *out_token is not defined. - * If it succeeds and *out_token is NULL, then there are no more tokens, - * and this function should not be called again. - * *out_token will be some substring of the original *input, so it should not - * be freed. - */ -static int parse_credential_process_token(char** input, char** out_token) -{ - *out_token = NULL; - int i; - - if (!*input) { - AWS_CREDS_ERROR("parse_credential_process_token called after yielding last token"); - return -1; - } - - *input = ltrim(*input); - - if (**input == '\0') { - *input = NULL; - *out_token = NULL; - return 0; - } - - if (**input == '"') { - (*input)++; - i = scan_credential_process_token_quoted(*input); - } - else { - i = scan_credential_process_token_unquoted(*input); - } - - if (i < 0) { - return -1; - } - - *out_token = *input; - *input += i; - - if (**input != '\0') { - **input = '\0'; - (*input)++; - } - - return 0; -} - -/* See <fluent-bit/flb_aws_credentials.h>. */ -char** parse_credential_process(char* input) -{ - char* next_token = NULL; - struct token_array arr = { 0 }; - int token_count = credential_process_token_count(input); - - if (token_count < 0) { - goto error; - } - - /* Add one extra capacity for the NULL terminator. */ - if (new_token_array(&arr, token_count + 1) < 0) { - goto error; - } - - while (1) { - if (parse_credential_process_token(&input, &next_token) < 0) { - goto error; - } - - if (!next_token) { - break; - } - - if (append_token(&arr, next_token) < 0) { - goto error; - } - } - - if (append_token(&arr, NULL) < 0) { - goto error; - } - - return arr.tokens; - -error: - flb_free(arr.tokens); - return NULL; -} - -/* - * Reads from the pipe into the buffer until no more input is available. - * If the input is exhausted (EOF), returns 0. - * If reading would block (EWOULDBLOCK/EAGAIN), returns > 0. - * If an error occurs or the buffer is full, returns < 0. - */ -static int read_until_block(char* name, flb_pipefd_t fd, struct readbuf* buf) -{ - int result = -1; - - while (1) { - if (buf->len >= buf->cap) { - AWS_CREDS_ERROR("credential_process %s exceeded max buffer size", name); - return -1; - } - - result = flb_pipe_r(fd, buf->buf + buf->len, buf->cap - buf->len); - if (result < 0) { - if (FLB_PIPE_WOULDBLOCK()) { - return 1; - } - flb_errno(); - return -1; - } - else if (result == 0) { /* EOF */ - return 0; - } - else { - buf->len += result; - } - } -} - -/* - * Polls waitpid until the given process exits, or the timeout is reached. - * Returns 0 on success and < 0 on failure. - */ -static int waitpid_timeout(char* name, pid_t pid, int* wstatus) -{ - int result = -1; - int retries = WAITPID_TIMEOUT_MS / WAITPID_POLL_FREQUENCY_MS; - - while (1) { - result = waitpid(pid, wstatus, WNOHANG); - if (result < 0) { - flb_errno(); - return -1; - } - - if (result > 0) { - return 0; - } - - if (retries <= 0) { - AWS_CREDS_ERROR("timed out waiting for credential_process %s to exit", name); - return -1; - } - retries--; - - usleep(WAITPID_POLL_FREQUENCY_MS * MICROS_PER_MS); - } -} - -struct process { - int initialized; - char** args; - int stdin_stream; - flb_pipefd_t stdout_stream[2]; - int stderr_stream; - pid_t pid; -}; - -/* - * Initializes a new process with the given args. - * args is assumed to be a NULL terminated array, for use with execvp. - * It must have a least one element, and the first element is assumed to be the - * name/path of the executable. - * Returns 0 on success and < 0 on failure. - * The caller is responsible for calling `destroy_process(p)`. - */ -static int new_process(struct process* p, char** args) -{ - *p = (struct process) { - .initialized = FLB_TRUE, - .args = args, - .stdin_stream = -1, - .stdout_stream = {-1, -1}, - .stderr_stream = -1, - .pid = -1, - }; - - while ((p->stdin_stream = open(DEV_NULL, O_RDONLY|O_CLOEXEC)) < 0) { - if (errno != EINTR) { - flb_errno(); - return -1; - } - } - - if (flb_pipe_create(p->stdout_stream) < 0) {; - flb_errno(); - return -1; - } - - if (fcntl(p->stdout_stream[0], F_SETFL, O_CLOEXEC) < 0) { - flb_errno(); - return -1; - } - - if (fcntl(p->stdout_stream[1], F_SETFL, O_CLOEXEC) < 0) { - flb_errno(); - return -1; - } - - while ((p->stderr_stream = open(DEV_NULL, O_WRONLY|O_CLOEXEC)) < 0) { - if (errno != EINTR) { - flb_errno(); - return -1; - } - } - - return 0; -} - -/* - * Sets up the credential_process's stdin, stdout, and stderr, and exec's - * the actual process. - * For this function to return at all is an error. - * This function should not be called more than once. - */ -static void exec_process_child(struct process* p) -{ - while ((dup2(p->stdin_stream, STDIN_FILENO) < 0)) { - if (errno != EINTR) { - return; - } - } - while ((dup2(p->stdout_stream[1], STDOUT_FILENO) < 0)) { - if (errno != EINTR) { - return; - } - } - while ((dup2(p->stderr_stream, STDERR_FILENO) < 0)) { - if (errno != EINTR) { - return; - } - } - - close(p->stdin_stream); - flb_pipe_close(p->stdout_stream[0]); - flb_pipe_close(p->stdout_stream[1]); - close(p->stderr_stream); - - execvp(p->args[0], p->args); -} - -/* - * Forks the credential_process, but does not wait for it to finish. - * Returns 0 on success and < 0 on failure. - * This function should not be called more than once. - */ -static int exec_process(struct process* p) -{ - AWS_CREDS_DEBUG("executing credential_process %s", p->args[0]); - - p->pid = fork(); - if (p->pid < 0) { - flb_errno(); - return -1; - } - - if (p->pid == 0) { - exec_process_child(p); - - /* It should not be possible to reach this under normal circumstances. */ - exit(EXIT_FAILURE); - } - - close(p->stdin_stream); - p->stdin_stream = -1; - - flb_pipe_close(p->stdout_stream[1]); - p->stdout_stream[1] = -1; - - close(p->stderr_stream); - p->stderr_stream = -1; - - return 0; -} - -/* - * Reads from the credential_process's stdout into the given buffer. - * Returns 0 on success, and < 0 on failure or timeout. - * This function should not be called more than once. - */ -static int read_from_process(struct process* p, struct readbuf* buf) -{ - int result = -1; - struct pollfd pfd; - struct flb_time start, timeout, deadline, now, remaining; - int remaining_ms; - - if (fcntl(p->stdout_stream[0], F_SETFL, O_NONBLOCK) < 0) { - flb_errno(); - return -1; - } - - if (get_monotonic_time(&start) < 0) { - return -1; - } - - flb_time_set(&timeout, - (time_t) (CREDENTIAL_PROCESS_TIMEOUT_MS / MS_PER_SEC), - ((long) (CREDENTIAL_PROCESS_TIMEOUT_MS % MS_PER_SEC)) * NS_PER_MS); - - /* deadline = start + timeout */ - flb_time_add(&start, &timeout, &deadline); - - while (1) { - pfd = (struct pollfd) { - .fd = p->stdout_stream[0], - .events = POLLIN, - }; - - if (get_monotonic_time(&now) < 0) { - return -1; - } - - /* remaining = deadline - now */ - if (flb_time_diff(&deadline, &now, &remaining) < 0) { - AWS_CREDS_ERROR("credential_process %s timed out", p->args[0]); - return -1; - } - - /* - * poll uses millisecond resolution for the timeout. - * If there is less than a millisecond left, then for simplicity we'll just - * declare that it timed out. - */ - remaining_ms = (int) (flb_time_to_nanosec(&remaining) / NS_PER_MS); - if (remaining_ms <= 0) { - AWS_CREDS_ERROR("credential_process %s timed out", p->args[0]); - return -1; - } - - result = poll(&pfd, 1, remaining_ms); - if (result < 0) { - if (errno != EINTR) { - flb_errno(); - return -1; - } - continue; - } - - if (result == 0) { - AWS_CREDS_ERROR("credential_process %s timed out", p->args[0]); - return -1; - } - - if ((pfd.revents & POLLNVAL) == POLLNVAL) { - AWS_CREDS_ERROR("credential_process %s POLLNVAL", p->args[0]); - return -1; - } - - if ((pfd.revents & POLLERR) == POLLERR) { - AWS_CREDS_ERROR("credential_process %s POLLERR", p->args[0]); - return -1; - } - - if ((pfd.revents & POLLIN) == POLLIN || (pfd.revents & POLLHUP) == POLLHUP) { - result = read_until_block(p->args[0], p->stdout_stream[0], buf); - if (result <= 0) { - return result; - } - } - } -} - -/* - * Waits for the process to exit, up to a timeout. - * Returns 0 on success and < 0 on failure. - * This function should not be called more than once. - */ -static int wait_process(struct process* p) -{ - int wstatus; - - if (waitpid_timeout(p->args[0], p->pid, &wstatus) < 0) { - return -1; - } - p->pid = -1; - - if (!WIFEXITED(wstatus)) { - AWS_CREDS_ERROR("credential_process %s did not terminate normally", p->args[0]); - return -1; - } - - if (WEXITSTATUS(wstatus) != EXIT_SUCCESS) { - AWS_CREDS_ERROR("credential_process %s exited with status %d", p->args[0], - WEXITSTATUS(wstatus)); - return -1; - } - - AWS_CREDS_DEBUG("credential_process %s exited successfully", p->args[0]); - return 0; -} - -/* - * Release all resources associated with this process. - * Calling this function multiple times is a no-op. - * Since the process does not own p->args, it does not free it. - * Note that p->args will be set to NULL, so the caller must hold onto - * it separately in order to free it. - */ -static void destroy_process(struct process* p) -{ - if (p->initialized) { - if (p->stdin_stream >= 0) { - close(p->stdin_stream); - p->stdin_stream = -1; - } - if (p->stdout_stream[0] >= 0) { - close(p->stdout_stream[0]); - p->stdout_stream[0] = -1; - } - if (p->stdout_stream[1] >= 0) { - close(p->stdout_stream[1]); - p->stdout_stream[1] = -1; - } - if (p->stderr_stream >= 0) { - close(p->stderr_stream); - p->stderr_stream = -1; - } - - if (p->pid > 0) { - if (kill(p->pid, SIGKILL) < 0) { - flb_errno(); - AWS_CREDS_ERROR("could not kill credential_process %s (pid=%d) " - "during cleanup", p->args[0], p->pid); - } - else { - while (waitpid(p->pid, NULL, 0) < 0) { - if (errno != EINTR) { - flb_errno(); - break; - } - } - } - p->pid = -1; - } - - p->args = NULL; - - p->initialized = FLB_FALSE; - } -} - -/* See <fluent-bit/flb_aws_credentials.h>. */ -int exec_credential_process(char* process, struct flb_aws_credentials** creds, - time_t* expiration) -{ - char** args = NULL; - int result = -1; - struct process p = { 0 }; - struct readbuf buf = { 0 }; - *creds = NULL; - *expiration = 0; - - args = parse_credential_process(process); - if (!args) { - result = -1; - goto end; - } - - if (!args[0]) { - AWS_CREDS_ERROR("invalid credential_process"); - result = -1; - goto end; - } - - if (new_process(&p, args) < 0) { - result = -1; - goto end; - } - - if (new_readbuf(&buf, CREDENTIAL_PROCESS_BUFFER_SIZE) < 0) { - result = -1; - goto end; - } - - if (exec_process(&p) < 0) { - result = -1; - goto end; - } - - if (read_from_process(&p, &buf) < 0) { - result = -1; - goto end; - } - - if (wait_process(&p) < 0) { - result = -1; - goto end; - } - - *creds = flb_parse_json_credentials(buf.buf, buf.len, - CREDENTIAL_PROCESS_RESPONSE_SESSION_TOKEN, - expiration); - if (!*creds) { - AWS_CREDS_ERROR("could not parse credentials from credential_process %s", args[0]); - result = -1; - goto end; - } - - AWS_CREDS_DEBUG("successfully parsed credentials from credential_process %s", args[0]); - - result = 0; - -end: - destroy_process(&p); - - flb_free(buf.buf); - buf.buf = NULL; - - flb_free(args); - args = NULL; - - if (result < 0) { - flb_aws_credentials_destroy(*creds); - *creds = NULL; - } - - return result; -} diff --git a/fluent-bit/src/aws/flb_aws_credentials_profile.c b/fluent-bit/src/aws/flb_aws_credentials_profile.c deleted file mode 100644 index 6b3ab5dbd..000000000 --- a/fluent-bit/src/aws/flb_aws_credentials_profile.c +++ /dev/null @@ -1,753 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "flb_aws_credentials_log.h" - -#include <fluent-bit/flb_aws_credentials.h> -#include <fluent-bit/flb_aws_util.h> -#include <fluent-bit/flb_http_client.h> -#include <fluent-bit/flb_info.h> -#include <fluent-bit/flb_sds.h> - -#include <stdlib.h> -#include <string.h> -#include <time.h> -#include <sys/types.h> -#include <sys/stat.h> -#include <ctype.h> - -#define ACCESS_KEY_PROPERTY_NAME "aws_access_key_id" -#define SECRET_KEY_PROPERTY_NAME "aws_secret_access_key" -#define SESSION_TOKEN_PROPERTY_NAME "aws_session_token" -#define CREDENTIAL_PROCESS_PROPERTY_NAME "credential_process" - -#define AWS_PROFILE "AWS_PROFILE" -#define AWS_DEFAULT_PROFILE "AWS_DEFAULT_PROFILE" - -#define AWS_CONFIG_FILE "AWS_CONFIG_FILE" -#define AWS_SHARED_CREDENTIALS_FILE "AWS_SHARED_CREDENTIALS_FILE" - -#define DEFAULT_PROFILE "default" -#define CONFIG_PROFILE_PREFIX "profile " -#define CONFIG_PROFILE_PREFIX_LEN (sizeof(CONFIG_PROFILE_PREFIX)-1) - -/* Declarations */ -struct flb_aws_provider_profile; -static int refresh_credentials(struct flb_aws_provider_profile *implementation, - int debug_only); - -static int get_aws_shared_file_path(flb_sds_t* field, char* env_var, char* home_aws_path); - -static int parse_config_file(char *buf, char* profile, struct flb_aws_credentials** creds, - time_t* expiration, int debug_only); -static int parse_credentials_file(char *buf, char *profile, - struct flb_aws_credentials *creds, int debug_only); - -static int get_shared_config_credentials(char* config_path, - char*profile, - struct flb_aws_credentials** creds, - time_t* expiration, - int debug_only); -static int get_shared_credentials(char* credentials_path, - char* profile, - struct flb_aws_credentials** creds, - int debug_only); - -static flb_sds_t parse_property_value(char *s, int debug_only); -static char *parse_property_line(char *line); -static int has_profile(char *line, char* profile, int debug_only); -static int is_profile_line(char *line); -static int config_file_profile_matches(char *line, char *profile); - -/* - * A provider that reads from the shared credentials file. - */ -struct flb_aws_provider_profile { - struct flb_aws_credentials *creds; - time_t next_refresh; - - flb_sds_t profile; - flb_sds_t config_path; - flb_sds_t credentials_path; -}; - -struct flb_aws_credentials *get_credentials_fn_profile(struct flb_aws_provider - *provider) -{ - struct flb_aws_credentials *creds; - int ret; - struct flb_aws_provider_profile *implementation = provider->implementation; - - /* - * If next_refresh <= 0, it means we don't know how long the credentials - * are valid for. So we won't refresh them unless explicitly asked - * via refresh_fn_profile. - */ - if (!implementation->creds || (implementation->next_refresh > 0 && - time(NULL) >= implementation->next_refresh)) { - AWS_CREDS_DEBUG("Retrieving credentials for AWS Profile %s", - implementation->profile); - if (try_lock_provider(provider) == FLB_TRUE) { - ret = refresh_credentials(implementation, FLB_FALSE); - unlock_provider(provider); - if (ret < 0) { - AWS_CREDS_ERROR("Failed to retrieve credentials for AWS Profile %s", - implementation->profile); - return NULL; - } - } else { - AWS_CREDS_WARN("Another thread is refreshing credentials, will retry"); - return NULL; - } - } - - creds = flb_calloc(1, sizeof(struct flb_aws_credentials)); - if (!creds) { - flb_errno(); - goto error; - } - - creds->access_key_id = flb_sds_create(implementation->creds->access_key_id); - if (!creds->access_key_id) { - flb_errno(); - goto error; - } - - creds->secret_access_key = flb_sds_create(implementation-> - creds->secret_access_key); - if (!creds->secret_access_key) { - flb_errno(); - goto error; - } - - if (implementation->creds->session_token) { - creds->session_token = flb_sds_create(implementation-> - creds->session_token); - if (!creds->session_token) { - flb_errno(); - goto error; - } - - } else { - creds->session_token = NULL; - } - - return creds; - -error: - flb_aws_credentials_destroy(creds); - return NULL; -} - -int refresh_fn_profile(struct flb_aws_provider *provider) -{ - struct flb_aws_provider_profile *implementation = provider->implementation; - int ret = -1; - AWS_CREDS_DEBUG("Refresh called on the profile provider"); - if (try_lock_provider(provider) == FLB_TRUE) { - ret = refresh_credentials(implementation, FLB_FALSE); - unlock_provider(provider); - return ret; - } - return ret; -} - -int init_fn_profile(struct flb_aws_provider *provider) -{ - struct flb_aws_provider_profile *implementation = provider->implementation; - int ret = -1; - AWS_CREDS_DEBUG("Init called on the profile provider"); - if (try_lock_provider(provider) == FLB_TRUE) { - ret = refresh_credentials(implementation, FLB_TRUE); - unlock_provider(provider); - return ret; - } - return ret; -} - -/* - * Sync and Async are no-ops for the profile provider because it does not - * make network IO calls - */ -void sync_fn_profile(struct flb_aws_provider *provider) -{ - return; -} - -void async_fn_profile(struct flb_aws_provider *provider) -{ - return; -} - -void upstream_set_fn_profile(struct flb_aws_provider *provider, - struct flb_output_instance *ins) -{ - return; -} - -void destroy_fn_profile(struct flb_aws_provider *provider) -{ - struct flb_aws_provider_profile *implementation = provider->implementation; - - if (implementation) { - if (implementation->creds) { - flb_aws_credentials_destroy(implementation->creds); - } - - if (implementation->profile) { - flb_sds_destroy(implementation->profile); - } - - if (implementation->config_path) { - flb_sds_destroy(implementation->config_path); - } - - if (implementation->credentials_path) { - flb_sds_destroy(implementation->credentials_path); - } - - flb_free(implementation); - provider->implementation = NULL; - } - - return; -} - -static struct flb_aws_provider_vtable profile_provider_vtable = { - .get_credentials = get_credentials_fn_profile, - .init = init_fn_profile, - .refresh = refresh_fn_profile, - .destroy = destroy_fn_profile, - .sync = sync_fn_profile, - .async = async_fn_profile, - .upstream_set = upstream_set_fn_profile, -}; - -struct flb_aws_provider *flb_profile_provider_create(char* profile) -{ - struct flb_aws_provider *provider = NULL; - struct flb_aws_provider_profile *implementation = NULL; - int result = -1; - - provider = flb_calloc(1, sizeof(struct flb_aws_provider)); - - if (!provider) { - flb_errno(); - goto error; - } - - pthread_mutex_init(&provider->lock, NULL); - - implementation = flb_calloc(1, - sizeof( - struct flb_aws_provider_profile)); - - if (!implementation) { - flb_errno(); - goto error; - } - - provider->provider_vtable = &profile_provider_vtable; - provider->implementation = implementation; - - result = get_aws_shared_file_path(&implementation->config_path, AWS_CONFIG_FILE, - "/.aws/config"); - if (result < 0) { - goto error; - } - - result = get_aws_shared_file_path(&implementation->credentials_path, - AWS_SHARED_CREDENTIALS_FILE, "/.aws/credentials"); - if (result < 0) { - goto error; - } - - if (!implementation->config_path && !implementation->credentials_path) { - AWS_CREDS_WARN("Failed to initialize profile provider: " - "HOME, %s, and %s not set.", - AWS_CONFIG_FILE, AWS_SHARED_CREDENTIALS_FILE); - goto error; - } - - /* AWS profile name. */ - if (profile == NULL) { - profile = getenv(AWS_PROFILE); - } - if (profile && strlen(profile) > 0) { - goto set_profile; - } - - profile = getenv(AWS_DEFAULT_PROFILE); - if (profile && strlen(profile) > 0) { - goto set_profile; - } - - profile = DEFAULT_PROFILE; - -set_profile: - implementation->profile = flb_sds_create(profile); - if (!implementation->profile) { - flb_errno(); - goto error; - } - - return provider; - -error: - flb_aws_provider_destroy(provider); - return NULL; -} - - -/* - * Fetches the path of either the shared config file or the shared credentials file. - * Returns 0 on success and < 0 on failure. - * On success, the result will be stored in *field. - * - * If the given environment variable is set, then its value will be used verbatim. - * Else if $HOME is set, then it will be concatenated with home_aws_path. - * If neither is set, then *field will be set to NULL. This is not considered a failure. - * - * In practice, env_var will be "AWS_CONFIG_FILE" or "AWS_SHARED_CREDENTIALS_FILE", - * and home_aws_path will be "/.aws/config" or "/.aws/credentials". - */ -static int get_aws_shared_file_path(flb_sds_t* field, char* env_var, char* home_aws_path) -{ - char* path = NULL; - int result = -1; - flb_sds_t value = NULL; - - path = getenv(env_var); - if (path && *path) { - value = flb_sds_create(path); - if (!value) { - flb_errno(); - goto error; - } - } else { - path = getenv("HOME"); - if (path && *path) { - value = flb_sds_create(path); - if (!value) { - flb_errno(); - goto error; - } - - if (path[strlen(path) - 1] == '/') { - home_aws_path++; - } - result = flb_sds_cat_safe(&value, home_aws_path, strlen(home_aws_path)); - if (result < 0) { - flb_errno(); - goto error; - } - } - } - - *field = value; - return 0; - -error: - flb_sds_destroy(value); - return -1; -} - -static int is_profile_line(char *line) { - if (line[0] == '[') { - return FLB_TRUE; - } - return FLB_FALSE; -} - -/* Called on lines that have is_profile_line == True */ -static int has_profile(char *line, char* profile, int debug_only) { - char *end_bracket = strchr(line, ']'); - if (!end_bracket) { - if (debug_only) { - AWS_CREDS_DEBUG("Profile header has no ending bracket:\n %s", line); - } - else { - AWS_CREDS_WARN("Profile header has no ending bracket:\n %s", line); - } - return FLB_FALSE; - } - *end_bracket = '\0'; - - if (strcmp(&line[1], profile) == 0) { - return FLB_TRUE; - } - - return FLB_FALSE; -} - -/* - * Sets a null byte such that line becomes the property name - * Returns a pointer to the rest of the line (the value), if successful. - */ -static char *parse_property_line(char *line) { - int len = strlen(line); - int found_delimeter = FLB_FALSE; - int i = 0; - - if (isspace(line[0])) { - /* property line can not start with whitespace */ - return NULL; - } - - /* - * Go through the line char by char, once we find whitespace/= we are - * passed the property name. Return the first char of the property value. - * There should be a single "=" separating name and value. - */ - for (i=0; i < (len - 1); i++) { - if (isspace(line[i])) { - line[i] = '\0'; - } else if (found_delimeter == FLB_FALSE && line[i] == '=') { - found_delimeter = FLB_TRUE; - line[i] = '\0'; - } else if (found_delimeter == FLB_TRUE) { - return &line[i]; - } - } - - return NULL; -} - -/* called on the rest of a line after parse_property_line is called */ -static flb_sds_t parse_property_value(char *s, int debug_only) { - int len = strlen(s); - int i = 0; - char *val = NULL; - flb_sds_t prop; - - for (i=0; i < len; i++) { - if (isspace(s[i])) { - s[i] = '\0'; - continue; - } else if (!val) { - val = &s[i]; - } - } - - if (!val) { - AWS_CREDS_ERROR_OR_DEBUG(debug_only, "Could not parse credential value from %s", s); - } - - prop = flb_sds_create(val); - if (!prop) { - flb_errno(); - return NULL; - } - - return prop; -} - -static int config_file_profile_matches(char *line, char *profile) { - char *current_profile = line + 1; - char* current_profile_end = strchr(current_profile, ']'); - - if (!current_profile_end) { - return FLB_FALSE; - } - *current_profile_end = '\0'; - - /* - * Non-default profiles look like `[profile <name>]`. - * The default profile can look like `[profile default]` or just `[default]`. - * This is different than the credentials file, where everything is `[<name>]`. - */ - if (strncmp(current_profile, CONFIG_PROFILE_PREFIX, CONFIG_PROFILE_PREFIX_LEN) != 0) { - if (strcmp(current_profile, DEFAULT_PROFILE) != 0) { - /* This is not a valid profile line. */ - return FLB_FALSE; - } - } else { - current_profile += CONFIG_PROFILE_PREFIX_LEN; - } - - if (strcmp(current_profile, profile) == 0) { - return FLB_TRUE; - } - return FLB_FALSE; -} - -static int parse_config_file(char *buf, char* profile, struct flb_aws_credentials** creds, - time_t* expiration, int debug_only) -{ - char *line = NULL; - char *line_end = NULL; - char *prop_val = NULL; - char *credential_process = NULL; - int found_profile = FLB_FALSE; - - for (line = buf; line[0] != '\0'; line = buf) { - /* - * Find the next newline and replace it with a null terminator. - * That way we can easily manipulate the current line as a string. - */ - line_end = strchr(line, '\n'); - if (line_end) { - *line_end = '\0'; - buf = line_end + 1; - } else { - buf = ""; - } - - if (found_profile != FLB_TRUE) { - if (is_profile_line(line) != FLB_TRUE) { - continue; - } - if (config_file_profile_matches(line, profile) != FLB_TRUE) { - continue; - } - found_profile = FLB_TRUE; - } else { - if (is_profile_line(line) == FLB_TRUE) { - break; - } - prop_val = parse_property_line(line); - if (strcmp(line, CREDENTIAL_PROCESS_PROPERTY_NAME) == 0) { - credential_process = prop_val; - } - } - } - - if (credential_process) { -#ifdef FLB_HAVE_AWS_CREDENTIAL_PROCESS - if (exec_credential_process(credential_process, creds, expiration) < 0) { - return -1; - } -#else - AWS_CREDS_WARN("credential_process not supported for this platform"); - return -1; -#endif - } - - return 0; -} - -/* - * Parses a shared credentials file. - * Expects the contents of 'creds' to be initialized to NULL (i.e use calloc). - */ -static int parse_credentials_file(char *buf, char *profile, - struct flb_aws_credentials *creds, int debug_only) -{ - char *line; - char *line_end; - char *prop_val = NULL; - int found_profile = FLB_FALSE; - - line = buf; - - while (line[0] != '\0') { - /* turn the line into a C string */ - line_end = strchr(line, '\n'); - if (line_end) { - *line_end = '\0'; - } - - if (is_profile_line(line) == FLB_TRUE) { - if (found_profile == FLB_TRUE) { - break; - } - if (has_profile(line, profile, debug_only)) { - found_profile = FLB_TRUE; - } - } else { - prop_val = parse_property_line(line); - if (prop_val && found_profile == FLB_TRUE) { - if (strcmp(line, ACCESS_KEY_PROPERTY_NAME) == 0) { - creds->access_key_id = parse_property_value(prop_val, - debug_only); - } - if (strcmp(line, SECRET_KEY_PROPERTY_NAME) == 0) { - creds->secret_access_key = parse_property_value(prop_val, - debug_only); - } - if (strcmp(line, SESSION_TOKEN_PROPERTY_NAME) == 0) { - creds->session_token = parse_property_value(prop_val, - debug_only); - } - } - } - - /* advance to next line */ - if (line_end) { - line = line_end + 1; - } else { - break; - } - } - - if (creds->access_key_id && creds->secret_access_key) { - return 0; - } - AWS_CREDS_ERROR_OR_DEBUG(debug_only, "%s and %s keys not parsed in shared " - "credentials file for profile %s.", ACCESS_KEY_PROPERTY_NAME, - SECRET_KEY_PROPERTY_NAME, profile); - return -1; -} - -static int get_shared_config_credentials(char* config_path, - char*profile, - struct flb_aws_credentials** creds, - time_t* expiration, - int debug_only) { - int result = -1; - char* buf = NULL; - size_t size; - *creds = NULL; - *expiration = 0; - - AWS_CREDS_DEBUG("Reading shared config file."); - - if (flb_read_file(config_path, &buf, &size) < 0) { - if (errno == ENOENT) { - AWS_CREDS_DEBUG("Shared config file %s does not exist", config_path); - result = 0; - goto end; - } - flb_errno(); - AWS_CREDS_ERROR_OR_DEBUG(debug_only, "Could not read shared config file %s", - config_path); - result = -1; - goto end; - } - - if (parse_config_file(buf, profile, creds, expiration, debug_only) < 0) { - result = -1; - goto end; - } - - result = 0; - -end: - flb_free(buf); - return result; -} - -static int get_shared_credentials(char* credentials_path, - char* profile, - struct flb_aws_credentials** creds, - int debug_only) { - int result = -1; - char* buf = NULL; - size_t size; - *creds = NULL; - - *creds = flb_calloc(1, sizeof(struct flb_aws_credentials)); - if (!*creds) { - flb_errno(); - result = -1; - goto end; - } - - AWS_CREDS_DEBUG("Reading shared credentials file."); - - if (flb_read_file(credentials_path, &buf, &size) < 0) { - if (errno == ENOENT) { - AWS_CREDS_ERROR_OR_DEBUG(debug_only, "Shared credentials file %s does not exist", - credentials_path); - } else { - flb_errno(); - AWS_CREDS_ERROR_OR_DEBUG(debug_only, "Could not read shared credentials file %s", - credentials_path); - } - result = -1; - goto end; - } - - if (parse_credentials_file(buf, profile, *creds, debug_only) < 0) { - AWS_CREDS_ERROR_OR_DEBUG(debug_only, "Could not parse shared credentials file: " - "valid profile with name '%s' not found", profile); - result = -1; - goto end; - } - - result = 0; - -end: - flb_free(buf); - - if (result < 0) { - flb_aws_credentials_destroy(*creds); - *creds = NULL; - } - - return result; -} - -static int refresh_credentials(struct flb_aws_provider_profile *implementation, - int debug_only) -{ - struct flb_aws_credentials *creds = NULL; - time_t expiration = 0; - int ret; - - if (implementation->config_path) { - ret = get_shared_config_credentials(implementation->config_path, - implementation->profile, - &creds, - &expiration, - debug_only); - if (ret < 0) { - goto error; - } - } - - /* - * If we did not find a credential_process in the shared config file, fall back to - * the shared credentials file. - */ - if (!creds) { - if (!implementation->credentials_path) { - AWS_CREDS_ERROR("shared config file contains no credential_process and " - "no shared credentials file was configured"); - goto error; - } - - ret = get_shared_credentials(implementation->credentials_path, - implementation->profile, - &creds, - debug_only); - if (ret < 0) { - goto error; - } - - /* The shared credentials file does not record when the credentials expire. */ - expiration = 0; - } - - /* unset and free existing credentials */ - flb_aws_credentials_destroy(implementation->creds); - implementation->creds = creds; - - if (expiration > 0) { - implementation->next_refresh = expiration - FLB_AWS_REFRESH_WINDOW; - } else { - implementation->next_refresh = 0; - } - - return 0; - -error: - flb_aws_credentials_destroy(creds); - return -1; -} diff --git a/fluent-bit/src/aws/flb_aws_credentials_sts.c b/fluent-bit/src/aws/flb_aws_credentials_sts.c deleted file mode 100644 index d992485cc..000000000 --- a/fluent-bit/src/aws/flb_aws_credentials_sts.c +++ /dev/null @@ -1,958 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <fluent-bit/flb_info.h> -#include <fluent-bit/flb_sds.h> -#include <fluent-bit/flb_http_client.h> -#include <fluent-bit/flb_aws_credentials.h> -#include <fluent-bit/flb_aws_util.h> -#include <fluent-bit/flb_random.h> -#include <fluent-bit/flb_jsmn.h> - -#include <stdlib.h> -#include <time.h> -#include <string.h> - -#define STS_ASSUME_ROLE_URI_FORMAT "/?Version=2011-06-15&Action=%s\ -&RoleSessionName=%s&RoleArn=%s" -#define STS_ASSUME_ROLE_URI_BASE_LEN 54 - -/* - * The STS APIs return an XML document with credentials. - * The part of the document we care about looks like this: - * <Credentials> - * <AccessKeyId>akid</AccessKeyId> - * <SecretAccessKey>skid</SecretAccessKey> - * <SessionToken>token</SessionToken> - * <Expiration>2019-11-09T13:34:41Z</Expiration> - * </Credentials> - */ -#define CREDENTIALS_NODE "<Credentials>" -#define CREDENTIALS_NODE_LEN 13 -#define ACCESS_KEY_NODE "<AccessKeyId>" -#define ACCESS_KEY_NODE_LEN 13 -#define ACCESS_KEY_NODE_END "</AccessKeyId>" -#define SECRET_KEY_NODE "<SecretAccessKey>" -#define SECRET_KEY_NODE_LEN 17 -#define SECRET_KEY_NODE_END "</SecretAccessKey>" -#define SESSION_TOKEN_NODE "<SessionToken>" -#define SESSION_TOKEN_NODE_LEN 14 -#define SESSION_TOKEN_NODE_END "</SessionToken>" -#define EXPIRATION_NODE "<Expiration>" -#define EXPIRATION_NODE_LEN 12 -#define EXPIRATION_NODE_END "</Expiration>" - -#define TOKEN_FILE_ENV_VAR "AWS_WEB_IDENTITY_TOKEN_FILE" -#define ROLE_ARN_ENV_VAR "AWS_ROLE_ARN" -#define SESSION_NAME_ENV_VAR "AWS_ROLE_SESSION_NAME" - -#define SESSION_NAME_RANDOM_BYTE_LEN 32 - -struct flb_aws_provider_eks; -void bytes_to_string(unsigned char *data, char *buf, size_t len); -static int assume_with_web_identity(struct flb_aws_provider_eks - *implementation); -static int sts_assume_role_request(struct flb_aws_client *sts_client, - struct flb_aws_credentials **creds, - char *uri, - time_t *next_refresh); -static flb_sds_t get_node(char *cred_node, char* node_name, int node_name_len, char* node_end); - - -/* - * A provider that uses credentials from the base provider to call STS - * and assume an IAM Role. - */ -struct flb_aws_provider_sts { - int custom_endpoint; - struct flb_aws_provider *base_provider; - - struct flb_aws_credentials *creds; - time_t next_refresh; - - struct flb_aws_client *sts_client; - - /* Fluent Bit uses regional STS endpoints; this is a best practice. */ - char *endpoint; - - flb_sds_t uri; -}; - -struct flb_aws_credentials *get_credentials_fn_sts(struct flb_aws_provider - *provider) -{ - struct flb_aws_credentials *creds; - int refresh = FLB_FALSE; - struct flb_aws_provider_sts *implementation = provider->implementation; - - flb_debug("[aws_credentials] Requesting credentials from the " - "STS provider.."); - - /* a negative next_refresh means that auto-refresh is disabled */ - if (implementation->next_refresh > 0 - && time(NULL) > implementation->next_refresh) { - refresh = FLB_TRUE; - } - if (!implementation->creds || refresh == FLB_TRUE) { - /* credentials need to be refreshed/obtained */ - if (try_lock_provider(provider)) { - flb_debug("[aws_credentials] STS Provider: Refreshing credential " - "cache."); - sts_assume_role_request(implementation->sts_client, - &implementation->creds, - implementation->uri, - &implementation->next_refresh); - unlock_provider(provider); - } - } - - if (!implementation->creds) { - /* - * We failed to lock the provider and creds are unset. This means that - * another co-routine is performing the refresh. - */ - flb_warn("[aws_credentials] No cached credentials are available and " - "a credential refresh is already in progress. The current " - "co-routine will retry."); - - return NULL; - } - - /* return a copy of the existing cached credentials */ - creds = flb_calloc(1, sizeof(struct flb_aws_credentials)); - if (!creds) { - goto error; - } - - creds->access_key_id = flb_sds_create(implementation->creds->access_key_id); - if (!creds->access_key_id) { - goto error; - } - - creds->secret_access_key = flb_sds_create(implementation->creds-> - secret_access_key); - if (!creds->secret_access_key) { - goto error; - } - - if (implementation->creds->session_token) { - creds->session_token = flb_sds_create(implementation->creds-> - session_token); - if (!creds->session_token) { - goto error; - } - - } else { - creds->session_token = NULL; - } - - return creds; - -error: - flb_errno(); - flb_aws_credentials_destroy(creds); - return NULL; -} - -int refresh_fn_sts(struct flb_aws_provider *provider) { - int ret = -1; - struct flb_aws_provider_sts *implementation = provider->implementation; - - flb_debug("[aws_credentials] Refresh called on the STS provider"); - - if (try_lock_provider(provider)) { - ret = sts_assume_role_request(implementation->sts_client, - &implementation->creds, implementation->uri, - &implementation->next_refresh); - unlock_provider(provider); - } - return ret; -} - -int init_fn_sts(struct flb_aws_provider *provider) { - int ret = -1; - struct flb_aws_provider_sts *implementation = provider->implementation; - - flb_debug("[aws_credentials] Init called on the STS provider"); - - /* Call Init on the base provider first */ - implementation->base_provider->provider_vtable-> - init(implementation->base_provider); - - implementation->sts_client->debug_only = FLB_TRUE; - - if (try_lock_provider(provider)) { - ret = sts_assume_role_request(implementation->sts_client, - &implementation->creds, implementation->uri, - &implementation->next_refresh); - unlock_provider(provider); - } - - implementation->sts_client->debug_only = FLB_FALSE; - return ret; -} - -void sync_fn_sts(struct flb_aws_provider *provider) { - struct flb_aws_provider_sts *implementation = provider->implementation; - struct flb_aws_provider *base_provider = implementation->base_provider; - - flb_debug("[aws_credentials] Sync called on the STS provider"); - /* Remove async flag */ - flb_stream_disable_async_mode(&implementation->sts_client->upstream->base); - - /* we also need to call sync on the base_provider */ - base_provider->provider_vtable->sync(base_provider); -} - -void async_fn_sts(struct flb_aws_provider *provider) { - struct flb_aws_provider_sts *implementation = provider->implementation; - struct flb_aws_provider *base_provider = implementation->base_provider; - - flb_debug("[aws_credentials] Async called on the STS provider"); - /* Add async flag */ - flb_stream_enable_async_mode(&implementation->sts_client->upstream->base); - - /* we also need to call async on the base_provider */ - base_provider->provider_vtable->async(base_provider); -} - -void upstream_set_fn_sts(struct flb_aws_provider *provider, - struct flb_output_instance *ins) { - struct flb_aws_provider_sts *implementation = provider->implementation; - struct flb_aws_provider *base_provider = implementation->base_provider; - - flb_debug("[aws_credentials] upstream_set called on the STS provider"); - /* associate output and upstream */ - flb_output_upstream_set(implementation->sts_client->upstream, ins); - - /* we also need to call upstream_set on the base_provider */ - base_provider->provider_vtable->upstream_set(base_provider, ins); -} - -void destroy_fn_sts(struct flb_aws_provider *provider) { - struct flb_aws_provider_sts *implementation; - - implementation = provider->implementation; - - if (implementation) { - if (implementation->creds) { - flb_aws_credentials_destroy(implementation->creds); - } - - if (implementation->sts_client) { - flb_aws_client_destroy(implementation->sts_client); - } - - if (implementation->uri) { - flb_sds_destroy(implementation->uri); - } - - if (implementation->custom_endpoint == FLB_FALSE) { - flb_free(implementation->endpoint); - } - - flb_free(implementation); - provider->implementation = NULL; - } - - return; -} - -static struct flb_aws_provider_vtable sts_provider_vtable = { - .get_credentials = get_credentials_fn_sts, - .init = init_fn_sts, - .refresh = refresh_fn_sts, - .destroy = destroy_fn_sts, - .sync = sync_fn_sts, - .async = async_fn_sts, - .upstream_set = upstream_set_fn_sts, -}; - -struct flb_aws_provider *flb_sts_provider_create(struct flb_config *config, - struct flb_tls *tls, - struct flb_aws_provider - *base_provider, - char *external_id, - char *role_arn, - char *session_name, - char *region, - char *sts_endpoint, - char *proxy, - struct - flb_aws_client_generator - *generator) -{ - struct flb_aws_provider_sts *implementation = NULL; - struct flb_aws_provider *provider = NULL; - struct flb_upstream *upstream = NULL; - - provider = flb_calloc(1, sizeof(struct flb_aws_provider)); - if (!provider) { - flb_errno(); - return NULL; - } - - pthread_mutex_init(&provider->lock, NULL); - - implementation = flb_calloc(1, sizeof(struct flb_aws_provider_sts)); - if (!implementation) { - goto error; - } - - provider->provider_vtable = &sts_provider_vtable; - provider->implementation = implementation; - - implementation->uri = flb_sts_uri("AssumeRole", role_arn, session_name, - external_id, NULL); - if (!implementation->uri) { - goto error; - } - - if (sts_endpoint) { - implementation->endpoint = removeProtocol(sts_endpoint, "https://"); - implementation->custom_endpoint = FLB_TRUE; - } - else { - implementation->endpoint = flb_aws_endpoint("sts", region); - implementation->custom_endpoint = FLB_FALSE; - } - - if(!implementation->endpoint) { - goto error; - } - - implementation->base_provider = base_provider; - implementation->sts_client = generator->create(); - if (!implementation->sts_client) { - goto error; - } - implementation->sts_client->name = "sts_client_assume_role_provider"; - implementation->sts_client->has_auth = FLB_TRUE; - implementation->sts_client->provider = base_provider; - implementation->sts_client->region = region; - implementation->sts_client->service = "sts"; - implementation->sts_client->port = 443; - implementation->sts_client->flags = 0; - implementation->sts_client->proxy = proxy; - - upstream = flb_upstream_create(config, implementation->endpoint, 443, - FLB_IO_TLS, tls); - if (!upstream) { - flb_error("[aws_credentials] Connection initialization error"); - goto error; - } - - upstream->base.net.connect_timeout = FLB_AWS_CREDENTIAL_NET_TIMEOUT; - - implementation->sts_client->upstream = upstream; - implementation->sts_client->host = implementation->endpoint; - - return provider; - -error: - flb_errno(); - flb_aws_provider_destroy(provider); - return NULL; -} - -/* - * A provider that uses OIDC tokens provided by kubernetes to obtain - * AWS credentials. - * - * The AWS SDKs have defined a spec for an OIDC provider that obtains tokens - * from environment variables or the shared config file. - * This provider only contains the functionality needed for EKS- obtaining the - * location of the OIDC token from an environment variable. - */ -struct flb_aws_provider_eks { - int custom_endpoint; - struct flb_aws_credentials *creds; - /* - * Time to auto-refresh creds before they expire. A negative value disables - * auto-refresh. Client code can always force a refresh. - */ - time_t next_refresh; - - struct flb_aws_client *sts_client; - - /* Fluent Bit uses regional STS endpoints; this is a best practice. */ - char *endpoint; - - char *session_name; - /* session name can come from env or be generated by the provider */ - int free_session_name; - char *role_arn; - - char *token_file; -}; - - -struct flb_aws_credentials *get_credentials_fn_eks(struct flb_aws_provider - *provider) -{ - struct flb_aws_credentials *creds = NULL; - int refresh = FLB_FALSE; - struct flb_aws_provider_eks *implementation = provider->implementation; - - flb_debug("[aws_credentials] Requesting credentials from the " - "EKS provider.."); - - /* a negative next_refresh means that auto-refresh is disabled */ - if (implementation->next_refresh > 0 - && time(NULL) > implementation->next_refresh) { - refresh = FLB_TRUE; - } - if (!implementation->creds || refresh == FLB_TRUE) { - if (try_lock_provider(provider)) { - flb_debug("[aws_credentials] EKS Provider: Refreshing credential " - "cache."); - assume_with_web_identity(implementation); - unlock_provider(provider); - } - } - - if (!implementation->creds) { - /* - * We failed to lock the provider and creds are unset. This means that - * another co-routine is performing the refresh. - */ - flb_warn("[aws_credentials] No cached credentials are available and " - "a credential refresh is already in progress. The current " - "co-routine will retry."); - - return NULL; - } - - creds = flb_calloc(1, sizeof(struct flb_aws_credentials)); - if (!creds) { - goto error; - } - - creds->access_key_id = flb_sds_create(implementation->creds->access_key_id); - if (!creds->access_key_id) { - goto error; - } - - creds->secret_access_key = flb_sds_create(implementation->creds-> - secret_access_key); - if (!creds->secret_access_key) { - goto error; - } - - if (implementation->creds->session_token) { - creds->session_token = flb_sds_create(implementation->creds-> - session_token); - if (!creds->session_token) { - goto error; - } - - } - else { - creds->session_token = NULL; - } - - return creds; - -error: - flb_errno(); - flb_aws_credentials_destroy(creds); - return NULL; -} - -int refresh_fn_eks(struct flb_aws_provider *provider) { - int ret = -1; - struct flb_aws_provider_eks *implementation = provider->implementation; - - flb_debug("[aws_credentials] Refresh called on the EKS provider"); - if (try_lock_provider(provider)) { - ret = assume_with_web_identity(implementation); - unlock_provider(provider); - } - return ret; -} - -int init_fn_eks(struct flb_aws_provider *provider) { - int ret = -1; - struct flb_aws_provider_eks *implementation = provider->implementation; - - implementation->sts_client->debug_only = FLB_TRUE; - - flb_debug("[aws_credentials] Init called on the EKS provider"); - if (try_lock_provider(provider)) { - ret = assume_with_web_identity(implementation); - unlock_provider(provider); - } - - implementation->sts_client->debug_only = FLB_FALSE; - return ret; -} - -void sync_fn_eks(struct flb_aws_provider *provider) { - struct flb_aws_provider_eks *implementation = provider->implementation; - flb_debug("[aws_credentials] Sync called on the EKS provider"); - /* remove async flag */ - flb_stream_disable_async_mode(&implementation->sts_client->upstream->base); -} - -void async_fn_eks(struct flb_aws_provider *provider) { - struct flb_aws_provider_eks *implementation = provider->implementation; - flb_debug("[aws_credentials] Async called on the EKS provider"); - /* add async flag */ - flb_stream_enable_async_mode(&implementation->sts_client->upstream->base); -} - -void upstream_set_fn_eks(struct flb_aws_provider *provider, - struct flb_output_instance *ins) { - struct flb_aws_provider_eks *implementation = provider->implementation; - flb_debug("[aws_credentials] upstream_set called on the EKS provider"); - /* set upstream on output */ - flb_output_upstream_set(implementation->sts_client->upstream, ins); -} - -void destroy_fn_eks(struct flb_aws_provider *provider) { - struct flb_aws_provider_eks *implementation = provider-> - implementation; - if (implementation) { - if (implementation->creds) { - flb_aws_credentials_destroy(implementation->creds); - } - - if (implementation->sts_client) { - flb_aws_client_destroy(implementation->sts_client); - } - - if (implementation->custom_endpoint == FLB_FALSE) { - flb_free(implementation->endpoint); - } - if (implementation->free_session_name == FLB_TRUE) { - flb_free(implementation->session_name); - } - - flb_free(implementation); - provider->implementation = NULL; - } - - return; -} - -static struct flb_aws_provider_vtable eks_provider_vtable = { - .get_credentials = get_credentials_fn_eks, - .init = init_fn_eks, - .refresh = refresh_fn_eks, - .destroy = destroy_fn_eks, - .sync = sync_fn_eks, - .async = async_fn_eks, - .upstream_set = upstream_set_fn_eks, -}; - -struct flb_aws_provider *flb_eks_provider_create(struct flb_config *config, - struct flb_tls *tls, - char *region, - char *sts_endpoint, - char *proxy, - struct - flb_aws_client_generator - *generator) -{ - struct flb_aws_provider_eks *implementation = NULL; - struct flb_aws_provider *provider = NULL; - struct flb_upstream *upstream = NULL; - - provider = flb_calloc(1, sizeof(struct flb_aws_provider)); - - if (!provider) { - flb_errno(); - return NULL; - } - - pthread_mutex_init(&provider->lock, NULL); - - implementation = flb_calloc(1, sizeof(struct flb_aws_provider_eks)); - - if (!implementation) { - goto error; - } - - provider->provider_vtable = &eks_provider_vtable; - provider->implementation = implementation; - - /* session name either comes from the env var or is a random uuid */ - implementation->session_name = getenv(SESSION_NAME_ENV_VAR); - implementation->free_session_name = FLB_FALSE; - if (!implementation->session_name || - strlen(implementation->session_name) == 0) { - implementation->session_name = flb_sts_session_name(); - if (!implementation->session_name) { - goto error; - } - implementation->free_session_name = FLB_TRUE; - } - - implementation->role_arn = getenv(ROLE_ARN_ENV_VAR); - if (!implementation->role_arn || strlen(implementation->role_arn) == 0) { - flb_debug("[aws_credentials] Not initializing EKS provider because" - " %s was not set", ROLE_ARN_ENV_VAR); - flb_aws_provider_destroy(provider); - return NULL; - } - - implementation->token_file = getenv(TOKEN_FILE_ENV_VAR); - if (!implementation->token_file || strlen(implementation->token_file) == 0) - { - flb_debug("[aws_credentials] Not initializing EKS provider because" - " %s was not set", TOKEN_FILE_ENV_VAR); - flb_aws_provider_destroy(provider); - return NULL; - } - - if (sts_endpoint) { - implementation->endpoint = removeProtocol(sts_endpoint, "https://"); - implementation->custom_endpoint = FLB_TRUE; - } - else { - implementation->endpoint = flb_aws_endpoint("sts", region); - implementation->custom_endpoint = FLB_FALSE; - } - - if(!implementation->endpoint) { - goto error; - } - - implementation->sts_client = generator->create(); - if (!implementation->sts_client) { - goto error; - } - implementation->sts_client->name = "sts_client_eks_provider"; - /* AssumeRoleWithWebIdentity does not require sigv4 */ - implementation->sts_client->has_auth = FLB_FALSE; - implementation->sts_client->provider = NULL; - implementation->sts_client->region = region; - implementation->sts_client->service = "sts"; - implementation->sts_client->port = 443; - implementation->sts_client->flags = 0; - implementation->sts_client->proxy = proxy; - - upstream = flb_upstream_create(config, implementation->endpoint, 443, - FLB_IO_TLS, tls); - - if (!upstream) { - goto error; - } - - upstream->base.net.connect_timeout = FLB_AWS_CREDENTIAL_NET_TIMEOUT; - - implementation->sts_client->upstream = upstream; - implementation->sts_client->host = implementation->endpoint; - - return provider; - -error: - flb_errno(); - flb_aws_provider_destroy(provider); - return NULL; -} - -/* Generates string which can serve as a unique session name */ -char *flb_sts_session_name() { - unsigned char random_data[SESSION_NAME_RANDOM_BYTE_LEN]; - char *session_name = NULL; - int ret; - - ret = flb_random_bytes(random_data, SESSION_NAME_RANDOM_BYTE_LEN); - - if (ret != 0) { - flb_errno(); - - return NULL; - } - - session_name = flb_calloc(SESSION_NAME_RANDOM_BYTE_LEN + 1, - sizeof(char)); - if (session_name == NULL) { - flb_errno(); - - return NULL; - } - - bytes_to_string(random_data, session_name, SESSION_NAME_RANDOM_BYTE_LEN); - - return session_name; -} - -/* converts random bytes to a string we can safely put in a URL */ -void bytes_to_string(unsigned char *data, char *buf, size_t len) { - int index; - char charset[] = "0123456789" - "abcdefghijklmnopqrstuvwxyz" - "ABCDEFGHIJKLMNOPQRSTUVWXYZ"; - - while (len-- > 0) { - index = (int) data[len]; - index = index % (sizeof(charset) - 1); - buf[len] = charset[index]; - } -} - -static int assume_with_web_identity(struct flb_aws_provider_eks - *implementation) -{ - int ret; - char *web_token = NULL; - size_t web_token_size; - flb_sds_t uri = NULL; - int init_mode = implementation->sts_client->debug_only; - - ret = flb_read_file(implementation->token_file, &web_token, - &web_token_size); - if (ret < 0) { - if (init_mode == FLB_TRUE) { - flb_debug("[aws_credentials] Could not read web identify token file"); - } else { - flb_error("[aws_credentials] Could not read web identify token file"); - } - return -1; - } - - uri = flb_sts_uri("AssumeRoleWithWebIdentity", implementation->role_arn, - implementation->session_name, NULL, web_token); - if (!uri) { - flb_free(web_token); - return -1; - } - - ret = sts_assume_role_request(implementation->sts_client, - &implementation->creds, uri, - &implementation->next_refresh); - flb_free(web_token); - flb_sds_destroy(uri); - return ret; -} - -static int sts_assume_role_request(struct flb_aws_client *sts_client, - struct flb_aws_credentials **creds, - char *uri, - time_t *next_refresh) -{ - time_t expiration; - struct flb_aws_credentials *credentials = NULL; - struct flb_http_client *c = NULL; - flb_sds_t error_type; - int init_mode = sts_client->debug_only; - - flb_debug("[aws_credentials] Calling STS.."); - - c = sts_client->client_vtable->request(sts_client, FLB_HTTP_GET, - uri, NULL, 0, NULL, 0); - - if (c && c->resp.status == 200) { - credentials = flb_parse_sts_resp(c->resp.payload, &expiration); - if (!credentials) { - if (init_mode == FLB_TRUE) { - flb_debug("[aws_credentials] Failed to parse response from STS"); - } - else { - flb_error("[aws_credentials] Failed to parse response from STS"); - } - flb_http_client_destroy(c); - return -1; - } - - /* unset and free existing credentials first */ - flb_aws_credentials_destroy(*creds); - *creds = NULL; - - *next_refresh = expiration - FLB_AWS_REFRESH_WINDOW; - *creds = credentials; - flb_http_client_destroy(c); - return 0; - } - - if (c && c->resp.payload_size > 0) { - error_type = flb_aws_error(c->resp.payload, c->resp.payload_size); - if (error_type) { - if (init_mode == FLB_TRUE) { - flb_debug("[aws_credentials] STS API responded with %s", error_type); - } - else { - flb_error("[aws_credentials] STS API responded with %s", error_type); - } - } else { - flb_debug("[aws_credentials] STS raw response: \n%s", - c->resp.payload); - } - } - - if (c) { - flb_http_client_destroy(c); - } - if (init_mode == FLB_TRUE) { - flb_debug("[aws_credentials] STS assume role request failed"); - } - else { - flb_error("[aws_credentials] STS assume role request failed"); - } - return -1; - -} - -/* - * The STS APIs return an XML document with credentials. - * The part of the document we care about looks like this: - * <Credentials> - * <AccessKeyId>akid</AccessKeyId> - * <SecretAccessKey>skid</SecretAccessKey> - * <SessionToken>token</SessionToken> - * <Expiration>2019-11-09T13:34:41Z</Expiration> - * </Credentials> - */ -struct flb_aws_credentials *flb_parse_sts_resp(char *response, - time_t *expiration) -{ - struct flb_aws_credentials *creds = NULL; - char *cred_node = NULL; - flb_sds_t tmp = NULL; - - cred_node = strstr(response, CREDENTIALS_NODE); - if (!cred_node) { - flb_error("[aws_credentials] Could not find '%s' node in sts response", - CREDENTIALS_NODE); - return NULL; - } - cred_node += CREDENTIALS_NODE_LEN; - - creds = flb_calloc(1, sizeof(struct flb_aws_credentials)); - if (!creds) { - flb_errno(); - return NULL; - } - - creds->access_key_id = get_node(cred_node, ACCESS_KEY_NODE, - ACCESS_KEY_NODE_LEN, ACCESS_KEY_NODE_END); - if (!creds->access_key_id) { - goto error; - } - - creds->secret_access_key = get_node(cred_node, SECRET_KEY_NODE, - SECRET_KEY_NODE_LEN, SECRET_KEY_NODE_END); - if (!creds->secret_access_key) { - goto error; - } - - creds->session_token = get_node(cred_node, SESSION_TOKEN_NODE, - SESSION_TOKEN_NODE_LEN, SESSION_TOKEN_NODE_END); - if (!creds->session_token) { - goto error; - } - - tmp = get_node(cred_node, EXPIRATION_NODE, EXPIRATION_NODE_LEN, EXPIRATION_NODE_END); - if (!tmp) { - goto error; - } - *expiration = flb_aws_cred_expiration(tmp); - - flb_sds_destroy(tmp); - return creds; - -error: - flb_aws_credentials_destroy(creds); - if (tmp) { - flb_sds_destroy(tmp); - } - return NULL; -} - -/* - * Constructs the STS request uri. - * external_id can be NULL. - */ -flb_sds_t flb_sts_uri(char *action, char *role_arn, char *session_name, - char *external_id, char *identity_token) -{ - flb_sds_t tmp; - flb_sds_t uri = NULL; - size_t len = STS_ASSUME_ROLE_URI_BASE_LEN; - - if (external_id) { - len += 12; /* will add "&ExternalId=" */ - len += strlen(external_id); - } - - if (identity_token) { - len += 18; /* will add "&WebIdentityToken=" */ - len += strlen(identity_token); - } - - - len += strlen(session_name); - len += strlen(role_arn); - len += strlen(action); - len++; /* null char */ - - uri = flb_sds_create_size(len); - if (!uri) { - return NULL; - } - - tmp = flb_sds_printf(&uri, STS_ASSUME_ROLE_URI_FORMAT, action, session_name, - role_arn); - if (!tmp) { - flb_sds_destroy(uri); - return NULL; - } - - if (external_id) { - flb_sds_printf(&uri, "&ExternalId=%s", external_id); - } - - if (identity_token) { - flb_sds_printf(&uri, "&WebIdentityToken=%s", identity_token); - } - - return uri; -} - -static flb_sds_t get_node(char *cred_node, char* node_name, int node_name_len, char* node_end) -{ - char *node = NULL; - char *end = NULL; - flb_sds_t val = NULL; - int len; - - node = strstr(cred_node, node_name); - if (!node) { - flb_error("[aws_credentials] Could not find '%s' node in sts response", - node_name); - return NULL; - } - node += node_name_len; - end = strstr(node, node_end); - if (!end) { - flb_error("[aws_credentials] Could not find end of '%s' node in " - "sts response", node_name); - return NULL; - } - len = end - node; - val = flb_sds_create_len(node, len); - if (!val) { - flb_errno(); - return NULL; - } - - return val; -} diff --git a/fluent-bit/src/aws/flb_aws_error_reporter.c b/fluent-bit/src/aws/flb_aws_error_reporter.c deleted file mode 100644 index 72da9666c..000000000 --- a/fluent-bit/src/aws/flb_aws_error_reporter.c +++ /dev/null @@ -1,276 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <stdio.h> -#include <stdlib.h> -#include <time.h> -#include <monkey/mk_core/mk_list.h> - -#include <fluent-bit/flb_mem.h> -#include <fluent-bit/flb_log.h> -#include <fluent-bit/flb_env.h> -#include <fluent-bit/flb_sds.h> -#include <fluent-bit/aws/flb_aws_error_reporter.h> - -/* helper function to get int type environment variable*/ -static int getenv_int(const char *name) { - char *value, *end; - long result; - - value = getenv(name); - if (!value) { - return 0; - } - - result = strtol(value, &end, 10); - if (*end != '\0') { - return 0; - } - return (int) result; -} - -/* create an error reporter*/ -struct flb_aws_error_reporter *flb_aws_error_reporter_create() -{ - char *path_var = NULL; - int ttl_var, status_message_length; - struct flb_aws_error_reporter *error_reporter; - FILE *f; - int ret; - - error_reporter = flb_calloc(1, sizeof(struct flb_aws_error_reporter)); - if (!error_reporter) { - flb_errno(); - return NULL; - } - - /* setup error report file path */ - path_var = getenv(STATUS_MESSAGE_FILE_PATH_ENV); - if (path_var == NULL) { - flb_free(error_reporter); - flb_errno(); - return NULL; - } - - error_reporter->file_path = flb_sds_create(path_var); - if (!error_reporter->file_path) { - flb_free(error_reporter); - flb_errno(); - return NULL; - } - - /* clean up existing file*/ - if ((f = fopen(error_reporter->file_path, "r")) != NULL) { - /* file exist, try delete it*/ - if (remove(error_reporter->file_path)) { - flb_free(error_reporter); - flb_errno(); - return NULL; - } - } - - /* setup error reporter message TTL */ - ttl_var = getenv_int(STATUS_MESSAGE_TTL_ENV); - if (ttl_var <= 0) { - ttl_var = STATUS_MESSAGE_TTL_DEFAULT; - } - error_reporter->ttl = ttl_var; - - /* setup error reporter file size */ - status_message_length = getenv_int(STATUS_MESSAGE_MAX_BYTE_LENGTH_ENV); - if(status_message_length <= 0) { - status_message_length = STATUS_MESSAGE_MAX_BYTE_LENGTH_DEFAULT; - } - error_reporter->max_size = status_message_length; - - /* create the message Linked Lists */ - mk_list_init(&error_reporter->messages); - - return error_reporter; -} - -/* error reporter write the error message into reporting file and memory*/ -int flb_aws_error_reporter_write(struct flb_aws_error_reporter *error_reporter, char *msg) -{ - struct mk_list *tmp; - struct mk_list *head; - struct flb_error_message *message; - struct flb_error_message *tmp_message; - flb_sds_t buf; - flb_sds_t buf_tmp; - int deleted_message_count = 0; - FILE *f; - - if (error_reporter == NULL) { - return -1; - } - - buf = flb_sds_create(msg); - if (!buf) { - flb_errno(); - return -1; - } - /* check if the message is the same with latest one in queue*/ - if (mk_list_is_empty(&error_reporter->messages) != 0) { - tmp_message = mk_list_entry_last(&error_reporter->messages, - struct flb_error_message, _head); - if (tmp_message->len == flb_sds_len(buf) && - flb_sds_cmp(tmp_message->data, buf, tmp_message->len) == 0) { - - tmp_message->timestamp = time(NULL); - flb_sds_destroy(buf); - return 0; - } - } - - message = flb_malloc(sizeof(struct flb_error_message)); - if (!message) { - flb_sds_destroy(buf); - flb_errno(); - return -1; - } - - /* check if new message is too large and truncate*/ - if (flb_sds_len(buf) > error_reporter->max_size) { - // truncate message - buf_tmp = flb_sds_copy(buf, msg, error_reporter->max_size); - if (!buf_tmp) { - flb_sds_destroy(buf); - flb_free(message); - return -1; - } - } - - message->data = flb_sds_create(buf); - if (!message->data) { - flb_sds_destroy(buf); - flb_free(message); - return -1; - } - - message->len = flb_sds_len(buf); - - /* clean up old message to provide enough space for new message*/ - mk_list_foreach_safe(head, tmp, &error_reporter->messages) { - tmp_message = mk_list_entry(head, struct flb_error_message, _head); - if (error_reporter->file_size + flb_sds_len(buf) <= error_reporter->max_size) { - break; - } - else { - error_reporter->file_size -= tmp_message->len; - deleted_message_count++; - mk_list_del(&tmp_message->_head); - flb_sds_destroy(tmp_message->data); - flb_free(tmp_message); - } - } - message->timestamp = time(NULL); - - mk_list_add(&message->_head, &error_reporter->messages); - error_reporter->file_size += message->len; - - if (deleted_message_count == 0) { - f = fopen(error_reporter->file_path, "a"); - fprintf(f, message->data); - } - else { - f = fopen(error_reporter->file_path, "w"); - mk_list_foreach_safe(head, tmp, &error_reporter->messages) { - tmp_message = mk_list_entry(head, struct flb_error_message, _head); - fprintf(f, tmp_message->data); - } - } - fclose(f); - - flb_sds_destroy(buf); - - return 0; - -} - -/* error reporter clean up the expired message based on TTL*/ -void flb_aws_error_reporter_clean(struct flb_aws_error_reporter *error_reporter) -{ - struct mk_list *tmp; - struct mk_list *head; - struct flb_error_message *message; - int expired_message_count = 0; - FILE *f; - - if (error_reporter == NULL) { - return; - } - - /* check the timestamp for every message and clean up expired messages*/ - mk_list_foreach_safe(head, tmp, &error_reporter->messages) { - message = mk_list_entry(head, struct flb_error_message, _head); - if (error_reporter->ttl > time(NULL) - message->timestamp) { - break; - } - error_reporter->file_size -= message->len; - mk_list_del(&message->_head); - flb_sds_destroy(message->data); - flb_free(message); - expired_message_count++; - } - - /* rewrite error report file if any message is cleaned up*/ - if (expired_message_count > 0) { - f = fopen(error_reporter->file_path, "w"); - mk_list_foreach_safe(head, tmp, &error_reporter->messages) { - message = mk_list_entry(head, struct flb_error_message, _head); - fprintf(f, message->data); - } - fclose(f); - } -} - -/* error reporter clean up when fluent bit shutdown*/ -void flb_aws_error_reporter_destroy(struct flb_aws_error_reporter *error_reporter) -{ - struct mk_list *tmp; - struct mk_list *head; - struct flb_error_message *message; - - if (error_reporter == NULL) { - return; - } - - if(error_reporter->file_path) { - flb_sds_destroy(error_reporter->file_path); - } - if (mk_list_is_empty(&error_reporter->messages) != 0) { - - mk_list_foreach_safe(head, tmp, &error_reporter->messages) { - message = mk_list_entry(head, struct flb_error_message, _head); - mk_list_del(&message->_head); - flb_sds_destroy(message->data); - flb_free(message); - } - mk_list_del(&error_reporter->messages); - } - - flb_free(error_reporter); -} - -/*check if system enable error reporting*/ -int is_error_reporting_enabled() -{ - return getenv(STATUS_MESSAGE_FILE_PATH_ENV) != NULL; -}
\ No newline at end of file diff --git a/fluent-bit/src/aws/flb_aws_imds.c b/fluent-bit/src/aws/flb_aws_imds.c deleted file mode 100644 index 0e54db161..000000000 --- a/fluent-bit/src/aws/flb_aws_imds.c +++ /dev/null @@ -1,370 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <fluent-bit/aws/flb_aws_imds.h> -#include <fluent-bit/flb_aws_credentials.h> -#include <fluent-bit/flb_aws_util.h> -#include <fluent-bit/flb_http_client.h> -#include <fluent-bit/flb_info.h> -#include <fluent-bit/flb_jsmn.h> - -#define FLB_AWS_IMDS_ROOT "/" -#define FLB_AWS_IMDS_V2_TOKEN_PATH "/latest/api/token" - -/* Request headers */ -static struct flb_aws_header imds_v2_token_ttl_header = { - .key = "X-aws-ec2-metadata-token-ttl-seconds", - .key_len = 36, - .val = "21600", /* 6 hours (ie maximum ttl) */ - .val_len = 5, -}; - -/* Request header templates */ -const static struct flb_aws_header imds_v2_token_token_header_template = { - .key = "X-aws-ec2-metadata-token", - .key_len = 24, - .val = "", /* Replace with token value */ - .val_len = 0, /* Replace with token length */ -}; - -/* Declarations */ -static int get_imds_version(struct flb_aws_imds *ctx); -static int refresh_imds_v2_token(struct flb_aws_imds *ctx); - -/* Default config values */ -const struct flb_aws_imds_config flb_aws_imds_config_default = { - FLB_AWS_IMDS_VERSION_EVALUATE}; - -/* Create IMDS context */ -struct flb_aws_imds *flb_aws_imds_create(const struct flb_aws_imds_config *imds_config, - struct flb_aws_client *ec2_imds_client) -{ - struct flb_aws_imds *ctx = NULL; - - /* Create context */ - ctx = flb_calloc(1, sizeof(struct flb_aws_imds)); - if (!ctx) { - flb_errno(); - return NULL; - } - - /* - * Set IMDS version to whatever is specified in config - * Version may be evaluated later if set to FLB_AWS_IMDS_VERSION_EVALUATE - */ - ctx->imds_version = imds_config->use_imds_version; - ctx->imds_v2_token = flb_sds_create_len("INVALID_TOKEN", 13); - ctx->imds_v2_token_len = 13; - - /* Detect IMDS support */ - if (!ec2_imds_client->upstream) { - flb_debug( - "[imds] unable to connect to EC2 IMDS. ec2_imds_client upstream is null"); - - flb_aws_imds_destroy(ctx); - return NULL; - } - if (0 != strncmp(ec2_imds_client->upstream->tcp_host, FLB_AWS_IMDS_HOST, - FLB_AWS_IMDS_HOST_LEN)) { - flb_debug("[imds] ec2_imds_client tcp host must be set to %s", FLB_AWS_IMDS_HOST); - flb_aws_imds_destroy(ctx); - return NULL; - } - if (ec2_imds_client->upstream->tcp_port != FLB_AWS_IMDS_PORT) { - flb_debug("[imds] ec2_imds_client tcp port must be set to %i", FLB_AWS_IMDS_PORT); - flb_aws_imds_destroy(ctx); - return NULL; - } - - /* Connect client */ - ctx->ec2_imds_client = ec2_imds_client; - return ctx; -} - -/* Destroy IMDS context */ -void flb_aws_imds_destroy(struct flb_aws_imds *ctx) -{ - if (ctx->imds_v2_token) { - flb_sds_destroy(ctx->imds_v2_token); - } - - flb_free(ctx); -} - -/* Get IMDS metadata */ -int flb_aws_imds_request(struct flb_aws_imds *ctx, const char *metadata_path, - flb_sds_t *metadata, size_t *metadata_len) -{ - return flb_aws_imds_request_by_key(ctx, metadata_path, metadata, metadata_len, NULL); -} - -/* Get IMDS metadata by key */ -int flb_aws_imds_request_by_key(struct flb_aws_imds *ctx, const char *metadata_path, - flb_sds_t *metadata, size_t *metadata_len, char *key) -{ - int ret; - flb_sds_t tmp; - - struct flb_http_client *c = NULL; - - struct flb_aws_client *ec2_imds_client = ctx->ec2_imds_client; - struct flb_aws_header token_header = imds_v2_token_token_header_template; - - /* Get IMDS version */ - int imds_version = get_imds_version(ctx); - - /* Abort on version detection failure */ - if (imds_version == FLB_AWS_IMDS_VERSION_EVALUATE) { - /* Exit gracefully allowing for retrys */ - flb_warn("[imds] unable to evaluate IMDS version"); - return -1; - } - - if (imds_version == FLB_AWS_IMDS_VERSION_2) { - token_header.val = ctx->imds_v2_token; - token_header.val_len = ctx->imds_v2_token_len; - flb_debug("[imds] using IMDSv2"); - } - else { - flb_debug("[imds] using IMDSv1"); - } - - c = ec2_imds_client->client_vtable->request( - ec2_imds_client, FLB_HTTP_GET, metadata_path, NULL, 0, &token_header, - (imds_version == FLB_AWS_IMDS_VERSION_1) ? 0 : 1); - if (!c) { - /* Exit gracefully allowing for retrys */ - flb_warn("[imds] failed to retrieve metadata"); - return -1; - } - - /* Detect invalid token */ - if (imds_version == FLB_AWS_IMDS_VERSION_2 && c->resp.status == 401) { - /* Refresh token and retry request */ - flb_http_client_destroy(c); - ret = refresh_imds_v2_token(ctx); - if (ret < 0) { - flb_debug("[imds] failed to refresh IMDSv2 token"); - return -1; - } - token_header.val = ctx->imds_v2_token; - token_header.val_len = ctx->imds_v2_token_len; - flb_debug("[imds] refreshed IMDSv2 token"); - c = ec2_imds_client->client_vtable->request( - ec2_imds_client, FLB_HTTP_GET, metadata_path, NULL, 0, &token_header, 1); - if (!c) { - /* Exit gracefully allowing for retries */ - flb_warn("[imds] failed to retrieve metadata"); - return -1; - } - } - - if (c->resp.status != 200) { - ret = -1; - if (c->resp.status == 404) { - ret = -2; - } - if (c->resp.payload_size > 0) { - flb_debug("[imds] metadata request failure response\n%s", c->resp.payload); - } - flb_http_client_destroy(c); - return ret; - } - - if (key != NULL) { - /* get the value of the key from payload json string */ - tmp = flb_json_get_val(c->resp.payload, c->resp.payload_size, key); - if (!tmp) { - tmp = flb_sds_create_len("NULL", 4); - flb_error("[imds] %s is undefined in EC2 instance", key); - } - } - else { - tmp = flb_sds_create_len(c->resp.payload, c->resp.payload_size); - } - - if (!tmp) { - flb_errno(); - flb_http_client_destroy(c); - return -1; - } - - *metadata = tmp; - *metadata_len = key == NULL ? c->resp.payload_size : strlen(tmp); - - flb_http_client_destroy(c); - return 0; -} - -/* Get VPC Id */ -flb_sds_t flb_aws_imds_get_vpc_id(struct flb_aws_imds *ctx) -{ - int ret; - flb_sds_t mac_id = NULL; - size_t mac_len = 0; - flb_sds_t vpc_id = NULL; - size_t vpc_id_len = 0; - - /* get EC2 instance Mac id first before getting VPC id */ - ret = flb_aws_imds_request(ctx, FLB_AWS_IMDS_MAC_PATH, &mac_id, &mac_len); - - if (ret < 0) { - flb_sds_destroy(mac_id); - return NULL; - } - - /* - * the VPC full path should be like: - * latest/meta-data/network/interfaces/macs/{mac_id}/vpc-id/" - */ - flb_sds_t vpc_path = flb_sds_create_size(70); - vpc_path = - flb_sds_printf(&vpc_path, "%s/%s/%s/", - "/latest/meta-data/network/interfaces/macs", mac_id, "vpc-id"); - ret = flb_aws_imds_request(ctx, vpc_path, &vpc_id, &vpc_id_len); - - flb_sds_destroy(mac_id); - flb_sds_destroy(vpc_path); - - return vpc_id; -} - -/* Obtain the IMDS version */ -static int get_imds_version(struct flb_aws_imds *ctx) -{ - int ret; - struct flb_aws_client *client = ctx->ec2_imds_client; - struct flb_aws_header invalid_token_header; - struct flb_http_client *c = NULL; - - if (ctx->imds_version != FLB_AWS_IMDS_VERSION_EVALUATE) { - return ctx->imds_version; - } - - /* - * Evaluate version - * To evaluate wether IMDSv2 is available, send an invalid token - * in IMDS request. If response status is 'Unauthorized', then IMDSv2 - * is available. - */ - invalid_token_header = imds_v2_token_token_header_template; - invalid_token_header.val = "INVALID"; - invalid_token_header.val_len = 7; - c = client->client_vtable->request(client, FLB_HTTP_GET, FLB_AWS_IMDS_ROOT, NULL, 0, - &invalid_token_header, 1); - - if (!c) { - flb_debug("[imds] imds endpoint unavailable"); - return FLB_AWS_IMDS_VERSION_EVALUATE; - } - - /* Unauthorized response means that IMDS version 2 is in use */ - if (c->resp.status == 401) { - ctx->imds_version = FLB_AWS_IMDS_VERSION_2; - ret = refresh_imds_v2_token(ctx); - if (ret == -1) { - /* - * Token cannot be refreshed, test IMDSv1 - * If IMDSv1 cannot be used, response will be status 401 - */ - flb_http_client_destroy(c); - ctx->imds_version = FLB_AWS_IMDS_VERSION_EVALUATE; - c = client->client_vtable->request(client, FLB_HTTP_GET, FLB_AWS_IMDS_ROOT, - NULL, 0, NULL, 0); - if (!c) { - flb_debug("[imds] imds v1 attempt, endpoint unavailable"); - return FLB_AWS_IMDS_VERSION_EVALUATE; - } - - if (c->resp.status == 200) { - flb_info("[imds] to use IMDSv2, set --http-put-response-hop-limit to 2"); - } - else { - /* IMDSv1 unavailable. IMDSv2 beyond network hop count */ - flb_warn("[imds] failed to retrieve IMDSv2 token and IMDSv1 unavailable. " - "This is likely due to instance-metadata-options " - "--http-put-response-hop-limit being set to 1 and --http-tokens " - "set to required. " - "To use IMDSv2, please set --http-put-response-hop-limit to 2 as " - "described https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/" - "configuring-instance-metadata-options.html"); - } - } - } - - /* - * Success means that IMDS version 1 is in use - */ - if (c->resp.status == 200) { - flb_warn("[imds] falling back on IMDSv1"); - ctx->imds_version = FLB_AWS_IMDS_VERSION_1; - } - - flb_http_client_destroy(c); - return ctx->imds_version; -} - -/* - * Get an IMDSv2 token - * Token preserved in imds context - */ -static int refresh_imds_v2_token(struct flb_aws_imds *ctx) -{ - struct flb_http_client *c = NULL; - struct flb_aws_client *ec2_imds_client = ctx->ec2_imds_client; - - c = ec2_imds_client->client_vtable->request(ec2_imds_client, FLB_HTTP_PUT, - FLB_AWS_IMDS_V2_TOKEN_PATH, NULL, 0, - &imds_v2_token_ttl_header, 1); - - if (!c) { - return -1; - } - - if (c->resp.status != 200) { - if (c->resp.payload_size > 0) { - flb_error("[imds] IMDSv2 token retrieval failure response\n%s", - c->resp.payload); - } - - flb_http_client_destroy(c); - return -1; - } - - /* Preserve token information in ctx */ - if (c->resp.payload_size > 0) { - if (ctx->imds_v2_token) { - flb_sds_destroy(ctx->imds_v2_token); - } - ctx->imds_v2_token = flb_sds_create_len(c->resp.payload, c->resp.payload_size); - if (!ctx->imds_v2_token) { - flb_errno(); - flb_http_client_destroy(c); - return -1; - } - ctx->imds_v2_token_len = c->resp.payload_size; - - flb_http_client_destroy(c); - return 0; - } - - flb_debug("[imds] IMDS metadata response was empty"); - flb_http_client_destroy(c); - return -1; -} diff --git a/fluent-bit/src/aws/flb_aws_util.c b/fluent-bit/src/aws/flb_aws_util.c deleted file mode 100644 index 533bba7eb..000000000 --- a/fluent-bit/src/aws/flb_aws_util.c +++ /dev/null @@ -1,1047 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <fluent-bit/flb_info.h> -#include <fluent-bit/flb_sds.h> -#include <fluent-bit/flb_http_client.h> -#include <fluent-bit/flb_signv4.h> -#include <fluent-bit/flb_aws_util.h> -#include <fluent-bit/flb_aws_credentials.h> -#include <fluent-bit/flb_output_plugin.h> -#include <fluent-bit/flb_jsmn.h> -#include <fluent-bit/flb_env.h> - -#include <stdlib.h> -#include <sys/types.h> -#include <sys/stat.h> -#include <fcntl.h> - -#define AWS_SERVICE_ENDPOINT_FORMAT "%s.%s.amazonaws.com" -#define AWS_SERVICE_ENDPOINT_BASE_LEN 15 - -#define TAG_PART_DESCRIPTOR "$TAG[%d]" -#define TAG_DESCRIPTOR "$TAG" -#define MAX_TAG_PARTS 10 -#define S3_KEY_SIZE 1024 -#define RANDOM_STRING "$UUID" -#define INDEX_STRING "$INDEX" -#define AWS_USER_AGENT_NONE "none" -#define AWS_USER_AGENT_ECS "ecs" -#define AWS_USER_AGENT_K8S "k8s" -#define AWS_ECS_METADATA_URI "ECS_CONTAINER_METADATA_URI_V4" -#define FLB_MAX_AWS_RESP_BUFFER_SIZE 0 /* 0 means unlimited capacity as per requirement */ - -#ifdef FLB_SYSTEM_WINDOWS -#define FLB_AWS_BASE_USER_AGENT "aws-fluent-bit-plugin-windows" -#define FLB_AWS_BASE_USER_AGENT_FORMAT "aws-fluent-bit-plugin-windows-%s" -#define FLB_AWS_BASE_USER_AGENT_LEN 29 -#else -#define FLB_AWS_BASE_USER_AGENT "aws-fluent-bit-plugin" -#define FLB_AWS_BASE_USER_AGENT_FORMAT "aws-fluent-bit-plugin-%s" -#define FLB_AWS_BASE_USER_AGENT_LEN 21 -#endif - -#define FLB_AWS_MILLISECOND_FORMATTER_LENGTH 3 -#define FLB_AWS_NANOSECOND_FORMATTER_LENGTH 9 -#define FLB_AWS_MILLISECOND_FORMATTER "%3N" -#define FLB_AWS_NANOSECOND_FORMATTER_N "%9N" -#define FLB_AWS_NANOSECOND_FORMATTER_L "%L" - -struct flb_http_client *request_do(struct flb_aws_client *aws_client, - int method, const char *uri, - const char *body, size_t body_len, - struct flb_aws_header *dynamic_headers, - size_t dynamic_headers_len); - -/* - * https://service.region.amazonaws.com(.cn) - */ -char *flb_aws_endpoint(char* service, char* region) -{ - char *endpoint = NULL; - size_t len = AWS_SERVICE_ENDPOINT_BASE_LEN; - int is_cn = FLB_FALSE; - int bytes; - - - /* In the China regions, ".cn" is appended to the URL */ - if (strcmp("cn-north-1", region) == 0) { - len += 3; - is_cn = FLB_TRUE; - } - if (strcmp("cn-northwest-1", region) == 0) { - len += 3; - is_cn = FLB_TRUE; - } - - len += strlen(service); - len += strlen(region); - len++; /* null byte */ - - endpoint = flb_calloc(len, sizeof(char)); - if (!endpoint) { - flb_errno(); - return NULL; - } - - bytes = snprintf(endpoint, len, AWS_SERVICE_ENDPOINT_FORMAT, service, region); - if (bytes < 0) { - flb_errno(); - flb_free(endpoint); - return NULL; - } - - if (is_cn) { - memcpy(endpoint + bytes, ".cn", 3); - endpoint[bytes + 3] = '\0'; - } - - return endpoint; - -} - -int flb_read_file(const char *path, char **out_buf, size_t *out_size) -{ - int ret; - long bytes; - char *buf = NULL; - struct stat st; - int fd; - - fd = open(path, O_RDONLY); - if (fd < 0) { - return -1; - } - - ret = fstat(fd, &st); - if (ret == -1) { - flb_errno(); - close(fd); - return -1; - } - - buf = flb_calloc(st.st_size + 1, sizeof(char)); - if (!buf) { - flb_errno(); - close(fd); - return -1; - } - - bytes = read(fd, buf, st.st_size); - if (bytes < 0) { - flb_errno(); - flb_free(buf); - close(fd); - return -1; - } - - /* fread does not add null byte */ - buf[st.st_size] = '\0'; - - close(fd); - *out_buf = buf; - *out_size = st.st_size; - - return 0; -} - - -char *removeProtocol (char *endpoint, char *protocol) { - if (strncmp(protocol, endpoint, strlen(protocol)) == 0){ - endpoint = endpoint + strlen(protocol); - } - return endpoint; -} - -struct flb_http_client *flb_aws_client_request(struct flb_aws_client *aws_client, - int method, const char *uri, - const char *body, size_t body_len, - struct flb_aws_header - *dynamic_headers, - size_t dynamic_headers_len) -{ - struct flb_http_client *c = NULL; - - c = request_do(aws_client, method, uri, body, body_len, - dynamic_headers, dynamic_headers_len); - - // Auto retry if request fails - if (c == NULL && aws_client->retry_requests) { - flb_debug("[aws_client] auto-retrying"); - c = request_do(aws_client, method, uri, body, body_len, - dynamic_headers, dynamic_headers_len); - } - - /* - * 400 or 403 could indicate an issue with credentials- so we check for auth - * specific error messages and then force a refresh on the provider. - * For safety a refresh can be performed only once - * per FLB_AWS_CREDENTIAL_REFRESH_LIMIT. - * - */ - if (c && (c->resp.status >= 400 && c->resp.status < 500)) { - if (aws_client->has_auth && time(NULL) > aws_client->refresh_limit) { - if (flb_aws_is_auth_error(c->resp.payload, c->resp.payload_size) - == FLB_TRUE) { - flb_info("[aws_client] auth error, refreshing creds"); - aws_client->refresh_limit = time(NULL) - + FLB_AWS_CREDENTIAL_REFRESH_LIMIT; - aws_client->provider->provider_vtable-> - refresh(aws_client->provider); - } - } - } - - return c; -} - -static struct flb_aws_client_vtable client_vtable = { - .request = flb_aws_client_request, -}; - -struct flb_aws_client *flb_aws_client_create() -{ - struct flb_aws_client *client = flb_calloc(1, sizeof(struct flb_aws_client)); - if (!client) { - flb_errno(); - return NULL; - } - client->client_vtable = &client_vtable; - client->retry_requests = FLB_FALSE; - client->debug_only = FLB_FALSE; - return client; -} - -/* Generator that returns clients with the default vtable */ - -static struct flb_aws_client_generator default_generator = { - .create = flb_aws_client_create, -}; - -struct flb_aws_client_generator *flb_aws_client_generator() -{ - return &default_generator; -} - -void flb_aws_client_destroy(struct flb_aws_client *aws_client) -{ - if (aws_client) { - if (aws_client->upstream) { - flb_upstream_destroy(aws_client->upstream); - } - if (aws_client->extra_user_agent) { - flb_sds_destroy(aws_client->extra_user_agent); - } - flb_free(aws_client); - } -} - -int flb_aws_is_auth_error(char *payload, size_t payload_size) -{ - flb_sds_t error = NULL; - - if (payload_size == 0) { - return FLB_FALSE; - } - - /* Fluent Bit calls the STS API which returns XML */ - if (strcasestr(payload, "InvalidClientTokenId") != NULL) { - return FLB_TRUE; - } - - if (strcasestr(payload, "AccessDenied") != NULL) { - return FLB_TRUE; - } - - if (strcasestr(payload, "Expired") != NULL) { - return FLB_TRUE; - } - - /* Most APIs we use return JSON */ - error = flb_aws_error(payload, payload_size); - if (error != NULL) { - if (strcmp(error, "ExpiredToken") == 0 || - strcmp(error, "ExpiredTokenException") == 0 || - strcmp(error, "AccessDeniedException") == 0 || - strcmp(error, "AccessDenied") == 0 || - strcmp(error, "IncompleteSignature") == 0 || - strcmp(error, "SignatureDoesNotMatch") == 0 || - strcmp(error, "MissingAuthenticationToken") == 0 || - strcmp(error, "InvalidClientTokenId") == 0 || - strcmp(error, "InvalidToken") == 0 || - strcmp(error, "InvalidAccessKeyId") == 0 || - strcmp(error, "UnrecognizedClientException") == 0) { - flb_sds_destroy(error); - return FLB_TRUE; - } - flb_sds_destroy(error); - } - - return FLB_FALSE; -} - -struct flb_http_client *request_do(struct flb_aws_client *aws_client, - int method, const char *uri, - const char *body, size_t body_len, - struct flb_aws_header *dynamic_headers, - size_t dynamic_headers_len) -{ - size_t b_sent; - int ret; - struct flb_connection *u_conn = NULL; - flb_sds_t signature = NULL; - int i; - int normalize_uri; - struct flb_aws_header header; - struct flb_http_client *c = NULL; - flb_sds_t tmp; - flb_sds_t user_agent_prefix; - flb_sds_t user_agent = NULL; - char *buf; - struct flb_env *env; - - u_conn = flb_upstream_conn_get(aws_client->upstream); - if (!u_conn) { - if (aws_client->debug_only == FLB_TRUE) { - flb_debug("[aws_client] connection initialization error"); - } - else { - flb_error("[aws_client] connection initialization error"); - } - return NULL; - } - - /* Compose HTTP request */ - c = flb_http_client(u_conn, method, uri, - body, body_len, - aws_client->host, aws_client->port, - aws_client->proxy, aws_client->flags); - - if (!c) { - if (aws_client->debug_only == FLB_TRUE) { - flb_debug("[aws_client] could not initialize request"); - } - else { - flb_error("[aws_client] could not initialize request"); - } - goto error; - } - - /* Increase the maximum HTTP response buffer size to fit large responses from AWS services */ - ret = flb_http_buffer_size(c, FLB_MAX_AWS_RESP_BUFFER_SIZE); - if (ret != 0) { - flb_warn("[aws_http_client] failed to increase max response buffer size"); - } - - /* Set AWS Fluent Bit user agent */ - env = aws_client->upstream->base.config->env; - buf = (char *) flb_env_get(env, "FLB_AWS_USER_AGENT"); - if (buf == NULL) { - if (getenv(AWS_ECS_METADATA_URI) != NULL) { - user_agent = AWS_USER_AGENT_ECS; - } - else { - buf = (char *) flb_env_get(env, AWS_USER_AGENT_K8S); - if (buf && strcasecmp(buf, "enabled") == 0) { - user_agent = AWS_USER_AGENT_K8S; - } - } - - if (user_agent == NULL) { - user_agent = AWS_USER_AGENT_NONE; - } - - flb_env_set(env, "FLB_AWS_USER_AGENT", user_agent); - } - if (aws_client->extra_user_agent == NULL) { - buf = (char *) flb_env_get(env, "FLB_AWS_USER_AGENT"); - tmp = flb_sds_create(buf); - if (!tmp) { - flb_errno(); - goto error; - } - aws_client->extra_user_agent = tmp; - tmp = NULL; - } - - /* Add AWS Fluent Bit user agent header */ - if (strcasecmp(aws_client->extra_user_agent, AWS_USER_AGENT_NONE) == 0) { - ret = flb_http_add_header(c, "User-Agent", 10, - FLB_AWS_BASE_USER_AGENT, FLB_AWS_BASE_USER_AGENT_LEN); - } - else { - user_agent_prefix = flb_sds_create_size(64); - if (!user_agent_prefix) { - flb_errno(); - flb_error("[aws_client] failed to create user agent"); - goto error; - } - tmp = flb_sds_printf(&user_agent_prefix, FLB_AWS_BASE_USER_AGENT_FORMAT, - aws_client->extra_user_agent); - if (!tmp) { - flb_errno(); - flb_sds_destroy(user_agent_prefix); - flb_error("[aws_client] failed to create user agent"); - goto error; - } - user_agent_prefix = tmp; - - ret = flb_http_add_header(c, "User-Agent", 10, user_agent_prefix, - flb_sds_len(user_agent_prefix)); - flb_sds_destroy(user_agent_prefix); - } - - if (ret < 0) { - if (aws_client->debug_only == FLB_TRUE) { - flb_debug("[aws_client] failed to add header to request"); - } - else { - flb_error("[aws_client] failed to add header to request"); - } - goto error; - } - - /* add headers */ - for (i = 0; i < aws_client->static_headers_len; i++) { - header = aws_client->static_headers[i]; - ret = flb_http_add_header(c, - header.key, header.key_len, - header.val, header.val_len); - if (ret < 0) { - if (aws_client->debug_only == FLB_TRUE) { - flb_debug("[aws_client] failed to add header to request"); - } - else { - flb_error("[aws_client] failed to add header to request"); - } - goto error; - } - } - - for (i = 0; i < dynamic_headers_len; i++) { - header = dynamic_headers[i]; - ret = flb_http_add_header(c, - header.key, header.key_len, - header.val, header.val_len); - if (ret < 0) { - if (aws_client->debug_only == FLB_TRUE) { - flb_debug("[aws_client] failed to add header to request"); - } - else { - flb_error("[aws_client] failed to add header to request"); - } - goto error; - } - } - - if (aws_client->has_auth) { - if (aws_client->s3_mode == S3_MODE_NONE) { - normalize_uri = FLB_TRUE; - } - else { - normalize_uri = FLB_FALSE; - } - signature = flb_signv4_do(c, normalize_uri, FLB_TRUE, time(NULL), - aws_client->region, aws_client->service, - aws_client->s3_mode, NULL, - aws_client->provider); - if (!signature) { - if (aws_client->debug_only == FLB_TRUE) { - flb_debug("[aws_client] could not sign request"); - } - else { - flb_error("[aws_client] could not sign request"); - } - goto error; - } - } - - /* Perform request */ - ret = flb_http_do(c, &b_sent); - - if (ret != 0 || c->resp.status != 200) { - flb_debug("[aws_client] %s: http_do=%i, HTTP Status: %i", - aws_client->host, ret, c->resp.status); - } - - if (ret != 0 && c != NULL) { - flb_http_client_destroy(c); - c = NULL; - } - - flb_upstream_conn_release(u_conn); - flb_sds_destroy(signature); - return c; - -error: - if (u_conn) { - flb_upstream_conn_release(u_conn); - } - if (signature) { - flb_sds_destroy(signature); - } - if (c) { - flb_http_client_destroy(c); - } - return NULL; -} - -void flb_aws_print_xml_error(char *response, size_t response_len, - char *api, struct flb_output_instance *ins) -{ - flb_sds_t error; - flb_sds_t message; - - error = flb_aws_xml_get_val(response, response_len, "<Code>", "</Code>"); - if (!error) { - flb_plg_error(ins, "%s: Could not parse response", api); - return; - } - - message = flb_aws_xml_get_val(response, response_len, "<Message>", "</Message>"); - if (!message) { - /* just print the error */ - flb_plg_error(ins, "%s API responded with error='%s'", api, error); - } - else { - flb_plg_error(ins, "%s API responded with error='%s', message='%s'", - api, error, message); - flb_sds_destroy(message); - } - - flb_sds_destroy(error); -} - -/* Parses AWS XML API Error responses and returns the value of the <code> tag */ -flb_sds_t flb_aws_xml_error(char *response, size_t response_len) -{ - return flb_aws_xml_get_val(response, response_len, "<Code>", "</Code>"); -} - -/* - * Parses an XML document and returns the value of the given tag - * Param `tag` should include angle brackets; ex "<code>" - * And param `end` should include end brackets: "</code>" - */ -flb_sds_t flb_aws_xml_get_val(char *response, size_t response_len, char *tag, char *tag_end) -{ - flb_sds_t val = NULL; - char *node = NULL; - char *end; - int len; - - if (response_len == 0) { - return NULL; - } - node = strstr(response, tag); - if (!node) { - flb_debug("[aws] Could not find '%s' tag in API response", tag); - return NULL; - } - - /* advance to end of tag */ - node += strlen(tag); - - end = strstr(node, tag_end); - if (!end) { - flb_error("[aws] Could not find end of '%s' node in xml", tag); - return NULL; - } - len = end - node; - val = flb_sds_create_len(node, len); - if (!val) { - flb_errno(); - return NULL; - } - - return val; -} - -void flb_aws_print_error(char *response, size_t response_len, - char *api, struct flb_output_instance *ins) -{ - flb_sds_t error; - flb_sds_t message; - - error = flb_json_get_val(response, response_len, "__type"); - if (!error) { - return; - } - - message = flb_json_get_val(response, response_len, "message"); - if (!message) { - /* just print the error */ - flb_plg_error(ins, "%s API responded with error='%s'", api, error); - } - else { - flb_plg_error(ins, "%s API responded with error='%s', message='%s'", - api, error, message); - flb_sds_destroy(message); - } - - flb_sds_destroy(error); -} - -/* parses AWS JSON API error responses and returns the value of the __type field */ -flb_sds_t flb_aws_error(char *response, size_t response_len) -{ - return flb_json_get_val(response, response_len, "__type"); -} - -/* gets the value of a key in a json string */ -flb_sds_t flb_json_get_val(char *response, size_t response_len, char *key) -{ - jsmntok_t *tokens = NULL; - const jsmntok_t *t = NULL; - char *current_token = NULL; - jsmn_parser parser; - int tokens_size = 50; - size_t size; - int ret; - int i = 0; - int len; - flb_sds_t error_type = NULL; - - jsmn_init(&parser); - - size = sizeof(jsmntok_t) * tokens_size; - tokens = flb_calloc(1, size); - if (!tokens) { - flb_errno(); - return NULL; - } - - ret = jsmn_parse(&parser, response, response_len, - tokens, tokens_size); - - if (ret == JSMN_ERROR_INVAL || ret == JSMN_ERROR_PART) { - flb_free(tokens); - flb_debug("[aws_client] Unable to parse API response- response is not" - " valid JSON."); - return NULL; - } - - /* return value is number of tokens parsed */ - tokens_size = ret; - - /* - * jsmn will create an array of tokens like: - * key, value, key, value - */ - while (i < (tokens_size - 1)) { - t = &tokens[i]; - - if (t->start == -1 || t->end == -1 || (t->start == 0 && t->end == 0)) { - break; - } - - if (t->type == JSMN_STRING) { - current_token = &response[t->start]; - - if (strncmp(current_token, key, strlen(key)) == 0) { - i++; - t = &tokens[i]; - current_token = &response[t->start]; - len = t->end - t->start; - error_type = flb_sds_create_len(current_token, len); - if (!error_type) { - flb_errno(); - flb_free(tokens); - return NULL; - } - break; - } - } - - i++; - } - flb_free(tokens); - return error_type; -} - -/* Generic replace function for strings. */ -static char* replace_uri_tokens(const char* original_string, const char* current_word, - const char* new_word) -{ - char *result; - int i = 0; - int count = 0; - int new_word_len = strlen(new_word); - int old_word_len = strlen(current_word); - - for (i = 0; original_string[i] != '\0'; i++) { - if (strstr(&original_string[i], current_word) == &original_string[i]) { - count++; - i += old_word_len - 1; - } - } - - result = flb_sds_create_size(i + count * (new_word_len - old_word_len) + 1); - if (!result) { - flb_errno(); - return NULL; - } - - i = 0; - while (*original_string) { - if (strstr(original_string, current_word) == original_string) { - strncpy(&result[i], new_word, new_word_len); - i += new_word_len; - original_string += old_word_len; - } - else - result[i++] = *original_string++; - } - - result[i] = '\0'; - return result; -} - -/* - * Linux has strtok_r as the concurrent safe version - * Windows has strtok_s - */ -char* strtok_concurrent( - char* str, - char* delimiters, - char** context -) -{ -#ifdef FLB_SYSTEM_WINDOWS - return strtok_s(str, delimiters, context); -#else - return strtok_r(str, delimiters, context); -#endif -} - -/* Constructs S3 object key as per the format. */ -flb_sds_t flb_get_s3_key(const char *format, time_t time, const char *tag, - char *tag_delimiter, uint64_t seq_index) -{ - int i = 0; - int ret = 0; - int seq_index_len; - char *tag_token = NULL; - char *key; - char *random_alphanumeric; - char *seq_index_str; - /* concurrent safe strtok_r requires a tracking ptr */ - char *strtok_saveptr; - int len; - flb_sds_t tmp = NULL; - flb_sds_t buf = NULL; - flb_sds_t s3_key = NULL; - flb_sds_t tmp_key = NULL; - flb_sds_t tmp_tag = NULL; - struct tm gmt = {0}; - - if (strlen(format) > S3_KEY_SIZE){ - flb_warn("[s3_key] Object key length is longer than the 1024 character limit."); - } - - tmp_tag = flb_sds_create_len(tag, strlen(tag)); - if(!tmp_tag){ - goto error; - } - - s3_key = flb_sds_create_len(format, strlen(format)); - if (!s3_key) { - goto error; - } - - /* Check if delimiter(s) specifed exists in the tag. */ - for (i = 0; i < strlen(tag_delimiter); i++){ - if (strchr(tag, tag_delimiter[i])){ - ret = 1; - break; - } - } - - tmp = flb_sds_create_len(TAG_PART_DESCRIPTOR, 5); - if (!tmp) { - goto error; - } - if (strstr(s3_key, tmp)){ - if(ret == 0){ - flb_warn("[s3_key] Invalid Tag delimiter: does not exist in tag. " - "tag=%s, format=%s", tag, format); - } - } - - flb_sds_destroy(tmp); - tmp = NULL; - - /* Split the string on the delimiters */ - tag_token = strtok_concurrent(tmp_tag, tag_delimiter, &strtok_saveptr); - - /* Find all occurences of $TAG[*] and - * replaces it with the right token from tag. - */ - i = 0; - while(tag_token != NULL && i < MAX_TAG_PARTS) { - buf = flb_sds_create_size(10); - if (!buf) { - goto error; - } - tmp = flb_sds_printf(&buf, TAG_PART_DESCRIPTOR, i); - if (!tmp) { - goto error; - } - - tmp_key = replace_uri_tokens(s3_key, tmp, tag_token); - if (!tmp_key) { - goto error; - } - - if(strlen(tmp_key) > S3_KEY_SIZE){ - flb_warn("[s3_key] Object key length is longer than the 1024 character limit."); - } - - if (buf != tmp) { - flb_sds_destroy(buf); - } - flb_sds_destroy(tmp); - tmp = NULL; - buf = NULL; - flb_sds_destroy(s3_key); - s3_key = tmp_key; - tmp_key = NULL; - - tag_token = strtok_concurrent(NULL, tag_delimiter, &strtok_saveptr); - i++; - } - - tmp = flb_sds_create_len(TAG_PART_DESCRIPTOR, 5); - if (!tmp) { - goto error; - } - - /* A match against "$TAG[" indicates an invalid or out of bounds tag part. */ - if (strstr(s3_key, tmp)){ - flb_warn("[s3_key] Invalid / Out of bounds tag part: At most 10 tag parts " - "($TAG[0] - $TAG[9]) can be processed. tag=%s, format=%s, delimiters=%s", - tag, format, tag_delimiter); - } - - /* Find all occurences of $TAG and replace with the entire tag. */ - tmp_key = replace_uri_tokens(s3_key, TAG_DESCRIPTOR, tag); - if (!tmp_key) { - goto error; - } - - if(strlen(tmp_key) > S3_KEY_SIZE){ - flb_warn("[s3_key] Object key length is longer than the 1024 character limit."); - } - - flb_sds_destroy(s3_key); - s3_key = tmp_key; - tmp_key = NULL; - - /* Find all occurences of $INDEX and replace with the appropriate index. */ - if (strstr((char *) format, INDEX_STRING)) { - seq_index_len = snprintf(NULL, 0, "%"PRIu64, seq_index); - seq_index_str = flb_calloc(seq_index_len + 1, sizeof(char)); - if (seq_index_str == NULL) { - goto error; - } - - sprintf(seq_index_str, "%"PRIu64, seq_index); - seq_index_str[seq_index_len] = '\0'; - tmp_key = replace_uri_tokens(s3_key, INDEX_STRING, seq_index_str); - if (tmp_key == NULL) { - flb_free(seq_index_str); - goto error; - } - if (strlen(tmp_key) > S3_KEY_SIZE) { - flb_warn("[s3_key] Object key length is longer than the 1024 character limit."); - } - - flb_sds_destroy(s3_key); - s3_key = tmp_key; - tmp_key = NULL; - flb_free(seq_index_str); - } - - /* Find all occurences of $UUID and replace with a random string. */ - random_alphanumeric = flb_sts_session_name(); - if (!random_alphanumeric) { - goto error; - } - /* only use 8 chars of the random string */ - random_alphanumeric[8] = '\0'; - tmp_key = replace_uri_tokens(s3_key, RANDOM_STRING, random_alphanumeric); - if (!tmp_key) { - flb_free(random_alphanumeric); - goto error; - } - - if(strlen(tmp_key) > S3_KEY_SIZE){ - flb_warn("[s3_key] Object key length is longer than the 1024 character limit."); - } - - flb_sds_destroy(s3_key); - s3_key = tmp_key; - tmp_key = NULL; - flb_free(random_alphanumeric); - - if (!gmtime_r(&time, &gmt)) { - flb_error("[s3_key] Failed to create timestamp."); - goto error; - } - - flb_sds_destroy(tmp); - tmp = NULL; - - /* A string no longer than S3_KEY_SIZE + 1 is created to store the formatted timestamp. */ - key = flb_calloc(1, (S3_KEY_SIZE + 1) * sizeof(char)); - if (!key) { - goto error; - } - - ret = strftime(key, S3_KEY_SIZE, s3_key, &gmt); - if(ret == 0){ - flb_warn("[s3_key] Object key length is longer than the 1024 character limit."); - } - flb_sds_destroy(s3_key); - - len = strlen(key); - if (len > S3_KEY_SIZE) { - len = S3_KEY_SIZE; - } - - s3_key = flb_sds_create_len(key, len); - flb_free(key); - if (!s3_key) { - goto error; - } - - flb_sds_destroy(tmp_tag); - tmp_tag = NULL; - return s3_key; - - error: - flb_errno(); - if (tmp_tag){ - flb_sds_destroy(tmp_tag); - } - if (s3_key){ - flb_sds_destroy(s3_key); - } - if (buf && buf != tmp){ - flb_sds_destroy(buf); - } - if (tmp){ - flb_sds_destroy(tmp); - } - if (tmp_key){ - flb_sds_destroy(tmp_key); - } - return NULL; -} - -/* - * This function is an extension to strftime which can support milliseconds with %3N, - * support nanoseconds with %9N or %L. The return value is the length of formatted - * time string. - */ -size_t flb_aws_strftime_precision(char **out_buf, const char *time_format, - struct flb_time *tms) -{ - char millisecond_str[FLB_AWS_MILLISECOND_FORMATTER_LENGTH+1]; - char nanosecond_str[FLB_AWS_NANOSECOND_FORMATTER_LENGTH+1]; - char *tmp_parsed_time_str; - char *buf; - size_t out_size; - size_t tmp_parsed_time_str_len; - size_t time_format_len; - struct tm timestamp; - struct tm *tmp; - int i; - - /* - * Guess the max length needed for tmp_parsed_time_str and tmp_out_buf. The - * upper bound is 12*strlen(time_format) because the worst scenario will be only - * %c in time_format, and %c will be transfer to 24 chars long by function strftime(). - */ - time_format_len = strlen(time_format); - tmp_parsed_time_str_len = 12*time_format_len; - - /* - * Use tmp_parsed_time_str to buffer when replace %3N with milliseconds, replace - * %9N and %L with nanoseconds in time_format. - */ - tmp_parsed_time_str = (char *)flb_calloc(1, tmp_parsed_time_str_len*sizeof(char)); - if (!tmp_parsed_time_str) { - flb_errno(); - return 0; - } - - buf = (char *)flb_calloc(1, tmp_parsed_time_str_len*sizeof(char)); - if (!buf) { - flb_errno(); - flb_free(tmp_parsed_time_str); - return 0; - } - - /* Replace %3N to millisecond, %9N and %L to nanosecond in time_format. */ - snprintf(millisecond_str, FLB_AWS_MILLISECOND_FORMATTER_LENGTH+1, - "%" PRIu64, (uint64_t) tms->tm.tv_nsec / 1000000); - snprintf(nanosecond_str, FLB_AWS_NANOSECOND_FORMATTER_LENGTH+1, - "%" PRIu64, (uint64_t) tms->tm.tv_nsec); - for (i = 0; i < time_format_len; i++) { - if (strncmp(time_format+i, FLB_AWS_MILLISECOND_FORMATTER, 3) == 0) { - strncat(tmp_parsed_time_str, millisecond_str, - FLB_AWS_MILLISECOND_FORMATTER_LENGTH+1); - i += 2; - } - else if (strncmp(time_format+i, FLB_AWS_NANOSECOND_FORMATTER_N, 3) == 0) { - strncat(tmp_parsed_time_str, nanosecond_str, - FLB_AWS_NANOSECOND_FORMATTER_LENGTH+1); - i += 2; - } - else if (strncmp(time_format+i, FLB_AWS_NANOSECOND_FORMATTER_L, 2) == 0) { - strncat(tmp_parsed_time_str, nanosecond_str, - FLB_AWS_NANOSECOND_FORMATTER_LENGTH+1); - i += 1; - } - else { - strncat(tmp_parsed_time_str,time_format+i,1); - } - } - - tmp = gmtime_r(&tms->tm.tv_sec, ×tamp); - if (!tmp) { - return 0; - } - - out_size = strftime(buf, tmp_parsed_time_str_len, - tmp_parsed_time_str, ×tamp); - - /* Check whether tmp_parsed_time_str_len is enough for tmp_out_buff */ - if (out_size == 0) { - flb_free(tmp_parsed_time_str); - flb_free(buf); - return 0; - } - - *out_buf = buf; - flb_free(tmp_parsed_time_str); - - return out_size; -} diff --git a/fluent-bit/src/config_format/flb_cf_fluentbit.c b/fluent-bit/src/config_format/flb_cf_fluentbit.c deleted file mode 100644 index a775fffe0..000000000 --- a/fluent-bit/src/config_format/flb_cf_fluentbit.c +++ /dev/null @@ -1,804 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <fluent-bit/flb_info.h> -#include <fluent-bit/flb_config_format.h> -#include <fluent-bit/flb_sds.h> -#include <fluent-bit/flb_log.h> -#include <fluent-bit/flb_mem.h> -#include <fluent-bit/flb_kv.h> -#include <fluent-bit/flb_compat.h> - -#include <monkey/mk_core.h> - -#include <ctype.h> -#include <sys/types.h> -#include <sys/stat.h> -#ifndef _MSC_VER -#include <glob.h> -#endif - -#ifdef _WIN32 -#include <Windows.h> -#include <strsafe.h> -#define PATH_MAX MAX_PATH -#endif - -#define FLB_CF_FILE_NUM_LIMIT 1000 - -/* indent checker return codes */ -#define INDENT_ERROR -1 -#define INDENT_OK 0 -#define INDENT_GROUP_CONTENT 1 - -/* Included file by configuration */ -struct local_file { - flb_sds_t path; - struct mk_list _head; -}; - -/* Local context to keep state of variables and general */ -struct local_ctx { - int level; - char *file; - flb_sds_t root_path; - - /* included files */ - struct mk_list includes; - - /* meta instructions */ - struct mk_list metas; - - /* list of sections */ - struct mk_list sections; -}; - -static int read_config(struct flb_cf *cf, struct local_ctx *ctx, char *cfg_file, - char *buf, size_t size, ino_t *ino_table, int *ino_num); - -/* Raise a configuration schema error */ -static void config_error(const char *path, int line, const char *msg) -{ - flb_error("[config] error in %s:%i: %s", path, line, msg); -} - -/* Raise a warning */ -static void config_warn(const char *path, int line, const char *msg) -{ - mk_warn("Config file warning '%s':\n" - "\t\t\t\tat line %i: %s", - path, line, msg); -} - -static int char_search(const char *string, int c, int len) -{ - char *p; - - if (len < 0) { - len = strlen(string); - } - - p = memchr(string, c, len); - if (p) { - return (p - string); - } - - return -1; -} - -/* - * Helper function to simulate a fgets(2) but instead of using a real file stream - * uses the data buffer provided. - */ -#ifdef FLB_HAVE_STATIC_CONF - -static int static_fgets(char *out, size_t size, const char *data, size_t *off) -{ - size_t len; - const char *start = data + *off; - char *end; - - end = strchr(start, '\n'); - - if (!end || *off >= size) { - len = size - *off - 1; - memcpy(out, start, len); - out[len] = '\0'; - *off += len + 1; - return 0; - } - - len = end - start; - if (len >= size) { - len = size - 1; - } - memcpy(out, start, len); - out[len] = '\0'; - *off += len + 1; - - return 1; -} -#endif - -#ifndef _WIN32 -static int read_glob(struct flb_cf *cf, struct local_ctx *ctx, const char * path, - ino_t *ino_table, int *ino_num) -{ - int ret = -1; - glob_t glb; - char tmp[PATH_MAX]; - - const char *glb_path; - size_t i; - int ret_glb = -1; - - if (ctx->root_path && path[0] != '/') { - snprintf(tmp, PATH_MAX, "%s/%s", ctx->root_path, path); - glb_path = tmp; - } - else { - glb_path = path; - } - - ret_glb = glob(glb_path, GLOB_NOSORT, NULL, &glb); - if (ret_glb != 0) { - switch(ret_glb){ - case GLOB_NOSPACE: - flb_warn("[%s] glob: [%s] no space", __FUNCTION__, glb_path); - break; - case GLOB_NOMATCH: - flb_warn("[%s] glob: [%s] no match", __FUNCTION__, glb_path); - break; - case GLOB_ABORTED: - flb_warn("[%s] glob: [%s] aborted", __FUNCTION__, glb_path); - break; - default: - flb_warn("[%s] glob: [%s] other error", __FUNCTION__, glb_path); - } - return ret; - } - - for (i = 0; i < glb.gl_pathc; i++) { - ret = read_config(cf, ctx, glb.gl_pathv[i], NULL, 0, ino_table, ino_num); - if (ret < 0) { - break; - } - } - - globfree(&glb); - return ret; -} -#else -static int read_glob(struct flb_cf *cf, struct local_ctx *ctx, const char *path, - ino_t *ino_table, int *ino_num) -{ - char *star, *p0, *p1; - char pattern[MAX_PATH]; - char buf[MAX_PATH]; - int ret; - struct stat st; - HANDLE h; - WIN32_FIND_DATA data; - - if (strlen(path) > MAX_PATH - 1) { - return -1; - } - - star = strchr(path, '*'); - if (star == NULL) { - return -1; - } - - /* - * C:\data\tmp\input_*.conf - * 0<-----| - */ - p0 = star; - while (path <= p0 && *p0 != '\\') { - p0--; - } - - /* - * C:\data\tmp\input_*.conf - * |---->1 - */ - p1 = star; - while (*p1 && *p1 != '\\') { - p1++; - } - - memcpy(pattern, path, (p1 - path)); - pattern[p1 - path] = '\0'; - - h = FindFirstFileA(pattern, &data); - if (h == INVALID_HANDLE_VALUE) { - return 0; - } - - do { - /* Ignore the current and parent dirs */ - if (!strcmp(".", data.cFileName) || !strcmp("..", data.cFileName)) { - continue; - } - - /* Avoid an infinite loop */ - if (strchr(data.cFileName, '*')) { - continue; - } - - /* Create a path (prefix + filename + suffix) */ - memcpy(buf, path, p0 - path + 1); - buf[p0 - path + 1] = '\0'; - - if (FAILED(StringCchCatA(buf, MAX_PATH, data.cFileName))) { - continue; - } - if (FAILED(StringCchCatA(buf, MAX_PATH, p1))) { - continue; - } - - if (strchr(p1, '*')) { - read_glob(cf, ctx, buf, ino_table, ino_num); /* recursive */ - continue; - } - - ret = stat(buf, &st); - if (ret == 0 && (st.st_mode & S_IFMT) == S_IFREG) { - if (read_config(cf, ctx, buf, NULL, 0, ino_table, ino_num) < 0) { - return -1; - } - } - } while (FindNextFileA(h, &data) != 0); - - FindClose(h); - return 0; -} -#endif - -static int local_init(struct local_ctx *ctx, char *file) -{ - char *end; - char path[PATH_MAX + 1] = {0}; - -#ifndef FLB_HAVE_STATIC_CONF - char *p; - - if (file) { -#ifdef _MSC_VER - p = _fullpath(path, file, PATH_MAX + 1); -#else - p = realpath(file, path); -#endif - if (!p) { - flb_errno(); - flb_error("file=%s", file); - return -1; - } - } -#endif - - /* lookup path ending and truncate */ -#ifdef _MSC_VER - end = strrchr(path, '\\'); -#else - end = strrchr(path, '/'); -#endif - - if (end) { - end++; - *end = '\0'; - } - - if (file) { - ctx->file = flb_sds_create(file); - ctx->root_path = flb_sds_create(path); - } - else { - ctx->file = NULL; - ctx->root_path = NULL; - } - - ctx->level = 0; - mk_list_init(&ctx->metas); - mk_list_init(&ctx->sections); - mk_list_init(&ctx->includes); - - return 0; -} - -static void local_exit(struct local_ctx *ctx) -{ - struct mk_list *tmp; - struct mk_list *head; - struct local_file *f; - - mk_list_foreach_safe(head, tmp, &ctx->includes) { - f = mk_list_entry(head, struct local_file, _head); - flb_sds_destroy(f->path); - mk_list_del(&f->_head); - flb_free(f); - } - - if (ctx->file) { - flb_sds_destroy(ctx->file); - } - - if (ctx->root_path) { - flb_sds_destroy(ctx->root_path); - } -} - -static int is_file_included(struct local_ctx *ctx, const char *path) -{ - struct mk_list *head; - struct local_file *file; - - mk_list_foreach(head, &ctx->includes) { - file = mk_list_entry(head, struct local_file, _head); - if (strcmp(file->path, path) == 0) { - return FLB_TRUE; - } - } - - return FLB_FALSE; -} - -static int check_indent(const char *line, const char *indent, int *out_level) -{ - int extra = 0; - int level = 0; - - while (*line == *indent && *indent) { - line++; - indent++; - level++; - } - - if (*indent != '\0') { - if (isblank(*line)) { - flb_error("[config] inconsistent use of tab and space"); - } - else { - flb_error("[config] indentation level is too low"); - } - return INDENT_ERROR;; - } - - if (isblank(*line)) { - /* check if we have a 'group' key/value line */ - while (isblank(*line)) { - line++; - extra++; - } - - if (extra == level) { - *out_level = level + extra; - return INDENT_GROUP_CONTENT; - } - - flb_error("[config] extra indentation level found"); - return -1; - } - - *out_level = level; - return INDENT_OK; -} - -static int read_config(struct flb_cf *cf, struct local_ctx *ctx, - char *cfg_file, char *in_data, size_t in_size, - ino_t *ino_table, int *ino_num) -{ - int i; - int len; - int ret; - int end; - int level; - int line = 0; - int indent_len = -1; - int n_keys = 0; - char *key = NULL; - int key_len; - char *val = NULL; - int val_len; - char *buf; - char *fgets_ptr; - size_t bufsize = FLB_DEFAULT_CF_BUF_SIZE; - char tmp[PATH_MAX]; - flb_sds_t section = NULL; - flb_sds_t indent = NULL; - struct stat st; - struct local_file *file; - struct flb_cf_meta *meta; - struct flb_cf_section *current_section = NULL; - struct flb_cf_group *current_group = NULL; - struct cfl_variant *var; - unsigned long line_hard_limit; - - line_hard_limit = 32 * 1024 * 1024; /* 32MiB */ - - FILE *f = NULL; - - if (*ino_num >= FLB_CF_FILE_NUM_LIMIT) { - return -1; - } - - /* Check if the path exists (relative cases for included files) */ -#ifndef FLB_HAVE_STATIC_CONF - if (ctx->level >= 0) { - ret = stat(cfg_file, &st); - if (ret == -1 && errno == ENOENT) { - /* Try to resolve the real path (if exists) */ - if (cfg_file[0] == '/') { - return -1; - } - - if (ctx->root_path) { - snprintf(tmp, PATH_MAX, "%s/%s", ctx->root_path, cfg_file); - cfg_file = tmp; - } - /* stat again */ - ret = stat(cfg_file, &st); - if (ret < 0) { - flb_errno(); - return -1; - } - } -#ifndef _WIN32 - /* check if readed file */ - for (i=0; i<*ino_num; i++) { - if (st.st_ino == ino_table[i]) { - flb_warn("[config] Read twice. path=%s", cfg_file); - return -1; - } - } - ino_table[*ino_num] = st.st_ino; - *ino_num += 1; -#endif - } -#endif - - /* Check this file have not been included before */ - ret = is_file_included(ctx, cfg_file); - if (ret) { - flb_error("[config] file already included %s", cfg_file); - return -1; - } - ctx->level++; - -#ifndef FLB_HAVE_STATIC_CONF - /* Open configuration file */ - if ((f = fopen(cfg_file, "rb")) == NULL) { - flb_warn("[config] I cannot open %s file", cfg_file); - return -1; - } -#endif - - /* Allocate temporal buffer to read file content */ - buf = flb_malloc(bufsize); - if (!buf) { - flb_errno(); - goto error; - } - -#ifdef FLB_HAVE_STATIC_CONF - /* - * a static configuration comes from a buffer, so we use the static_fgets() - * workaround to retrieve the lines. - */ - size_t off = 0; - while (static_fgets(buf, FLB_CF_BUF_SIZE, in_data, &off)) { -#else - /* normal mode, read lines into a buffer */ - /* note that we use "fgets_ptr" so we can continue reading after realloc */ - fgets_ptr = buf; - while (fgets(fgets_ptr, bufsize - (fgets_ptr - buf), f)) { -#endif - len = strlen(buf); - if (len > 0 && buf[len - 1] == '\n') { - buf[--len] = 0; - if (len && buf[len - 1] == '\r') { - buf[--len] = 0; - } - /* after a successful line read, restore "fgets_ptr" to point to the - * beginning of buffer */ - fgets_ptr = buf; - } else if (feof(f)) { - /* handle EOF without a newline(CRLF or LF) */ - fgets_ptr = buf; - } -#ifndef FLB_HAVE_STATIC_CONF - else { - /* resize the line buffer */ - bufsize *= 2; - if (bufsize > line_hard_limit) { - flb_error("reading line is exceeded to the limit size of %lu. Current size is: %zu", - line_hard_limit, bufsize); - goto error; - } - buf = flb_realloc(buf, bufsize); - if (!buf) { - flb_error("failed to resize line buffer to %zu", bufsize); - flb_errno(); - goto error; - } - /* read more, starting at the buf + len position */ - fgets_ptr = buf + len; - continue; - } -#endif - - /* Line number */ - line++; - - if (!buf[0]) { - continue; - } - - /* Skip commented lines */ - if (buf[0] == '#') { - continue; - } - - if (len > 9 && strncasecmp(buf, "@INCLUDE ", 9) == 0) { - if (strchr(buf + 9, '*') != NULL) { - ret = read_glob(cf, ctx, buf + 9, ino_table, ino_num); - } - else { - ret = read_config(cf, ctx, buf + 9, NULL, 0, ino_table, ino_num); - } - if (ret == -1) { - ctx->level--; - if (indent) { - flb_sds_destroy(indent); - indent = NULL; - } - goto error; - } - continue; - } - else if (buf[0] == '@' && len > 3) { - meta = flb_cf_meta_property_add(cf, buf, len); - if (meta == NULL) { - goto error; - } - continue; - } - - /* Section definition */ - if (buf[0] == '[') { - current_group = NULL; - - end = char_search(buf, ']', len); - if (end > 0) { - /* - * Before to add a new section, lets check the previous - * one have at least one key set - */ - if (current_section && n_keys == 0) { - config_warn(cfg_file, line, - "previous section did not have keys"); - } - - /* Create new section */ - current_section = flb_cf_section_create(cf, buf + 1, end - 1); - if (!current_section) { - continue; - } - current_group = NULL; - n_keys = 0; - continue; - } - else { - config_error(cfg_file, line, "bad header definition"); - goto error; - } - } - - /* No separator defined */ - if (!indent) { - i = 0; - - do { i++; } while (i < len && isblank(buf[i])); - - indent = flb_sds_create_len(buf, i); - indent_len = flb_sds_len(indent); - - /* Blank indented line */ - if (i == len) { - continue; - } - } - - /* Validate indentation level */ - ret = check_indent(buf, indent, &level); - if (ret == INDENT_ERROR) { - config_error(cfg_file, line, "invalid indentation level"); - goto error; - } - else { - if (ret == INDENT_OK && current_group) { - current_group = NULL; - } - indent_len = level; - } - - if (buf[indent_len] == '#' || indent_len == len) { - continue; - } - - /* get the key value separator */ - i = char_search(buf + indent_len, ' ', len - indent_len); - - /* key */ - key = buf + indent_len; - key_len = i; - - if (!key) { - config_error(cfg_file, line, "undefined key - check config is in valid classic format"); - goto error; - } - else if(i < 0) { - config_error(cfg_file, line, "undefined value - check config is in valid classic format"); - goto error; - } - - /* Check possible start of a group */ - if (key[0] == '[') { - end = char_search(key, ']', len - indent_len); - if (end == -1) { - config_error(cfg_file, line, "expected a valid group name: [..]"); - goto error; - } - - if (!current_section) { - config_warn(cfg_file, line, - "current group don't have a parent section"); - goto error; - } - - /* check if a previous group exists with one key */ - if (current_group && n_keys == 0) { - config_warn(cfg_file, line, "previous group did not have keys"); - goto error; - } - - /* Create new group */ - current_group = flb_cf_group_create(cf, current_section, - key + 1, end - 1); - if (!current_group) { - continue; - } - n_keys = 0; - - /* continue processing since we need key/value pairs */ - continue; - } - - /* val */ - val = buf + indent_len + i + 1; - val_len = len - indent_len - i - 1; - - if (!key || !val || i < 0) { - config_error(cfg_file, line, "each key must have a value"); - goto error; - } - - if (val_len == 0) { - config_error(cfg_file, line, "key has an empty value"); - goto error; - } - - /* register entry: key and val are copied as duplicated */ - var = NULL; - if (current_group) { - var = flb_cf_section_property_add(cf, current_group->properties, - key, key_len, - val, val_len); - } - else if (current_section) { - var = flb_cf_section_property_add(cf, current_section->properties, - key, key_len, - val, val_len); - } - if (var == NULL) { - config_error(cfg_file, line, "could not allocate key value pair"); - goto error; - } - - /* Free temporary key and val */ - n_keys++; - } - - if (section && n_keys == 0) { - /* No key, no warning */ - } - - if (f) { - fclose(f); - } - - if (indent) { - flb_sds_destroy(indent); - indent = NULL; - } - flb_free(buf); - - /* Append this file to the list */ - file = flb_malloc(sizeof(struct local_file)); - if (!file) { - flb_errno(); - ctx->level--; - goto error; - } - file->path = flb_sds_create(cfg_file); - mk_list_add(&file->_head, &ctx->includes); - ctx->level--; - - return 0; - -error: - if (f) { - fclose(f); - } - if (indent) { - flb_sds_destroy(indent); - } - flb_free(buf); - return -1; -} - -struct flb_cf *flb_cf_fluentbit_create(struct flb_cf *cf, - char *file_path, char *buf, size_t size) -{ - int ret; - struct local_ctx ctx; - ino_t ino_table[FLB_CF_FILE_NUM_LIMIT]; - int ino_num = 0; - - if (!cf) { - cf = flb_cf_create(); - if (!cf) { - return NULL; - } - - flb_cf_set_origin_format(cf, FLB_CF_CLASSIC); - } - - ret = local_init(&ctx, file_path); - if (ret != 0) { - if (cf) { - flb_cf_destroy(cf); - } - return NULL; - } - - ret = read_config(cf, &ctx, file_path, buf, size, &ino_table[0], &ino_num); - - local_exit(&ctx); - - if (ret == -1) { - flb_cf_destroy(cf); - if (ino_num >= FLB_CF_FILE_NUM_LIMIT) { - flb_error("Too many config files. Limit = %d", FLB_CF_FILE_NUM_LIMIT); - } - return NULL; - } - - return cf; -} diff --git a/fluent-bit/src/config_format/flb_cf_yaml.c b/fluent-bit/src/config_format/flb_cf_yaml.c deleted file mode 100644 index 289760ec7..000000000 --- a/fluent-bit/src/config_format/flb_cf_yaml.c +++ /dev/null @@ -1,2110 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <fluent-bit/flb_info.h> -#include <fluent-bit/flb_config_format.h> -#include <fluent-bit/flb_sds.h> -#include <fluent-bit/flb_log.h> -#include <fluent-bit/flb_mem.h> -#include <fluent-bit/flb_kv.h> -#include <fluent-bit/flb_slist.h> - -#include <cfl/cfl.h> -#include <cfl/cfl_sds.h> -#include <cfl/cfl_variant.h> -#include <cfl/cfl_kvlist.h> - -#include <yaml.h> - -#include <ctype.h> -#include <sys/types.h> -#include <sys/stat.h> -#ifndef _MSC_VER -#include <glob.h> -#endif - -#ifdef _WIN32 -#include <Windows.h> -#include <strsafe.h> -#define PATH_MAX MAX_PATH -#endif - -#include <stdio.h> - -enum section { - SECTION_ENV, - SECTION_INCLUDE, - SECTION_SERVICE, - SECTION_PIPELINE, - SECTION_CUSTOM, - SECTION_INPUT, - SECTION_FILTER, - SECTION_OUTPUT, - SECTION_PROCESSOR, - SECTION_OTHER, -}; - -static char *section_names[] = { - "env", - "include", - "service", - "pipeline", - "custom", - "input", - "filter", - "output", - "processor", - "other" -}; - -struct file_state { - /* file */ - flb_sds_t name; /* file name */ - flb_sds_t path; /* file root path */ - - /* parent file state */ - struct file_state *parent; -}; - -struct local_ctx { - int level; /* inclusion level */ - - struct cfl_list states; - - struct mk_list includes; - - int service_set; -}; - -/* yaml_* functions return 1 on success and 0 on failure. */ -enum status { - YAML_SUCCESS = 1, - YAML_FAILURE = 0 -}; - -enum state { - STATE_START, /* start state */ - STATE_STREAM, /* start/end stream */ - STATE_DOCUMENT, /* start/end document */ - - STATE_SECTION, /* top level */ - STATE_SECTION_KEY, - STATE_SECTION_VAL, - - STATE_SERVICE, /* 'service' section */ - STATE_INCLUDE, /* 'includes' section */ - STATE_OTHER, /* any other unknown section */ - - STATE_CUSTOM, /* custom plugins */ - STATE_PIPELINE, /* pipeline groups customs inputs, filters and outputs */ - - STATE_PLUGIN_INPUT, /* input plugins section */ - STATE_PLUGIN_FILTER, /* filter plugins section */ - STATE_PLUGIN_OUTPUT, /* output plugins section */ - - STATE_PLUGIN_START, - STATE_PLUGIN_KEY, - STATE_PLUGIN_VAL, - STATE_PLUGIN_VAL_LIST, - - STATE_GROUP_KEY, - STATE_GROUP_VAL, - - STATE_INPUT_PROCESSORS, - STATE_INPUT_PROCESSOR, - - /* environment variables */ - STATE_ENV, - - - STATE_STOP /* end state */ -}; - -/* parser state allocation flags */ -#define HAS_KEY (1 << 0) -#define HAS_KEYVALS (1 << 1) - -struct parser_state { - /* tokens state */ - enum state state; - /* nesting level */ - int level; - - /* active section (if any) */ - enum section section; - - /* active section */ - struct flb_cf_section *cf_section; - /* active group */ - struct flb_cf_group *cf_group; - - /* key value */ - flb_sds_t key; - /* section key/value list */ - struct cfl_kvlist *keyvals; - /* pointer to current values in a list. */ - struct cfl_array *values; - /* are we the owner of the values? */ - int allocation_flags; - - struct file_state *file; - - struct cfl_list _head; -}; - -static struct parser_state *state_push(struct local_ctx *, enum state); -static struct parser_state *state_push_withvals(struct local_ctx *, - struct parser_state *, - enum state); -static struct parser_state *state_push_witharr(struct local_ctx *, - struct parser_state *, - enum state); -static struct parser_state *state_push_section(struct local_ctx *, enum state, - enum section); -static struct parser_state *state_push_key(struct local_ctx *, enum state, - const char *key); -static int state_create_section(struct flb_cf *, struct parser_state *, char *); -static int state_create_group(struct flb_cf *, struct parser_state *, char *); -static struct parser_state *state_pop(struct local_ctx *ctx); -static struct parser_state *state_create(struct file_state *parent, struct file_state *file); -static void state_destroy(struct parser_state *s); - - -static int read_config(struct flb_cf *cf, struct local_ctx *ctx, - struct file_state *parent, char *cfg_file); - -static char *state_str(enum state val) -{ - switch (val) { - case STATE_START: - return "start"; - case STATE_STREAM: - return "stream"; - case STATE_DOCUMENT: - return "document"; - case STATE_SECTION: - return "section"; - case STATE_SECTION_KEY: - return "section-key"; - case STATE_SECTION_VAL: - return "section-value"; - case STATE_SERVICE: - return "service"; - case STATE_INCLUDE: - return "include"; - case STATE_OTHER: - return "other"; - case STATE_CUSTOM: - return "custom"; - case STATE_PIPELINE: - return "pipeline"; - case STATE_PLUGIN_INPUT: - return "input"; - case STATE_PLUGIN_FILTER: - return "filter"; - case STATE_PLUGIN_OUTPUT: - return "output"; - case STATE_PLUGIN_START: - return "plugin-start"; - case STATE_PLUGIN_KEY: - return "plugin-key"; - case STATE_PLUGIN_VAL: - return "plugin-value"; - case STATE_PLUGIN_VAL_LIST: - return "plugin-values"; - case STATE_GROUP_KEY: - return "group-key"; - case STATE_GROUP_VAL: - return "group-val"; - case STATE_INPUT_PROCESSORS: - return "processors"; - case STATE_INPUT_PROCESSOR: - return "processor"; - case STATE_ENV: - return "env"; - case STATE_STOP: - return "stop"; - default: - return "unknown"; - } -} - -static int add_section_type(struct flb_cf *conf, struct parser_state *state) -{ - if (conf == NULL || state == NULL) { - return -1; - } - - if (state->section == SECTION_INPUT) { - state->cf_section = flb_cf_section_create(conf, "INPUT", 0); - } - else if (state->section == SECTION_FILTER) { - state->cf_section = flb_cf_section_create(conf, "FILTER", 0); - } - else if (state->section == SECTION_OUTPUT) { - state->cf_section = flb_cf_section_create(conf, "OUTPUT", 0); - } - else if (state->section == SECTION_CUSTOM) { - state->cf_section = flb_cf_section_create(conf, "customs", 0); - } - - if (!state->cf_section) { - return -1; - } - - return 0; -} - -static char *event_type_str(yaml_event_t *event) -{ - switch (event->type) { - case YAML_NO_EVENT: - return "no-event"; - case YAML_STREAM_START_EVENT: - return "stream-start-event"; - case YAML_STREAM_END_EVENT: - return "stream-end-event"; - case YAML_DOCUMENT_START_EVENT: - return "document-start-event"; - case YAML_DOCUMENT_END_EVENT: - return "document-end-event"; - case YAML_ALIAS_EVENT: - return "alias-event"; - case YAML_SCALAR_EVENT: - return "scalar-event"; - case YAML_SEQUENCE_START_EVENT: - return "sequence-start-event"; - break; - case YAML_SEQUENCE_END_EVENT: - return "sequence-end-event"; - case YAML_MAPPING_START_EVENT: - return "mapping-start-event"; - case YAML_MAPPING_END_EVENT: - return "mapping-end-event"; - default: - return "unknown"; - } -} - -static char *state_get_last(struct local_ctx *ctx) -{ - struct flb_slist_entry *entry; - - entry = mk_list_entry_last(&ctx->includes, struct flb_slist_entry, _head); - - if (entry == NULL) { - return NULL; - } - return entry->str; -} - -static void yaml_error_event(struct local_ctx *ctx, struct parser_state *state, - yaml_event_t *event) -{ - struct flb_slist_entry *entry; - - if (event == NULL) { - flb_error("[config] YAML error found but with no state or event"); - return; - } - - if (state == NULL) { - flb_error("[config] YAML error found but with no state, line %zu, column %zu: " - "unexpected event '%s' (%d).", - event->start_mark.line + 1, event->start_mark.column, - event_type_str(event), event->type); - return; - } - - entry = mk_list_entry_last(&ctx->includes, struct flb_slist_entry, _head); - - if (entry == NULL) { - flb_error("[config] YAML error found (no file info), line %zu, column %zu: " - "unexpected event '%s' (%d) in state '%s' (%d).", - event->start_mark.line + 1, event->start_mark.column, - event_type_str(event), event->type, state_str(state->state), state->state); - return; - } - - flb_error("[config] YAML error found in file \"%s\", line %zu, column %zu: " - "unexpected event '%s' (%d) in state '%s' (%d).", - entry->str, event->start_mark.line + 1, event->start_mark.column, - event_type_str(event), event->type, state_str(state->state), state->state); -} - -static void yaml_error_definition(struct local_ctx *ctx, struct parser_state *state, - yaml_event_t *event, char *value) -{ - flb_error("[config] YAML error found in file \"%s\", line %zu, column %zu: " - "duplicated definition of '%s'", - state->file->name, event->start_mark.line + 1, event->start_mark.column, - value); -} - -static void yaml_error_plugin_category(struct local_ctx *ctx, struct parser_state *state, - yaml_event_t *event, char *value) -{ - flb_error("[config] YAML error found in file \"%s\", line %zu, column %zu: " - "the pipeline component '%s' is not valid. Try one of these values: " - "customs, inputs, filters or outputs.", - state->file->name, event->start_mark.line + 1, event->start_mark.column, - value); -} - -static int is_file_included(struct local_ctx *ctx, const char *path) -{ - struct mk_list *head; - struct flb_slist_entry *entry; - - mk_list_foreach(head, &ctx->includes) { - entry = mk_list_entry(head, struct flb_slist_entry, _head); - - if (strcmp(entry->str, path) == 0) { - return FLB_TRUE; - } - } - - return FLB_FALSE; -} - -#ifndef _WIN32 -static int read_glob(struct flb_cf *conf, struct local_ctx *ctx, - struct parser_state *state, const char *path) -{ - int ret = -1; - glob_t glb; - char tmp[PATH_MAX+1]; - - const char *glb_path; - size_t idx; - int ret_glb = -1; - - if (state->file->path && path[0] != '/') { - ret = snprintf(tmp, PATH_MAX, "%s/%s", state->file->path, path); - - if (ret > PATH_MAX) { - return -1; - } - glb_path = tmp; - } - else { - glb_path = path; - } - - ret_glb = glob(glb_path, GLOB_NOSORT, NULL, &glb); - - if (ret_glb != 0) { - switch(ret_glb){ - case GLOB_NOSPACE: - flb_warn("[%s] glob: [%s] no space", __FUNCTION__, glb_path); - break; - case GLOB_NOMATCH: - flb_warn("[%s] glob: [%s] no match", __FUNCTION__, glb_path); - break; - case GLOB_ABORTED: - flb_warn("[%s] glob: [%s] aborted", __FUNCTION__, glb_path); - break; - default: - flb_warn("[%s] glob: [%s] other error", __FUNCTION__, glb_path); - } - return ret; - } - - for (idx = 0; idx < glb.gl_pathc; idx++) { - ret = read_config(conf, ctx, state->file, glb.gl_pathv[idx]); - - if (ret < 0) { - break; - } - } - - globfree(&glb); - return ret; -} -#else -static char *dirname(char *path) -{ - char *ptr; - - - ptr = strrchr(path, '\\'); - - if (ptr == NULL) { - return path; - } - *ptr++='\0'; - return path; -} - -static int read_glob(struct flb_cf *conf, struct local_ctx *ctx, - struct parser_state *state, const char *path) -{ - char *star, *p0, *p1; - char pattern[MAX_PATH]; - char buf[MAX_PATH]; - int ret; - struct stat st; - HANDLE hnd; - WIN32_FIND_DATA data; - - if (strlen(path) > MAX_PATH - 1) { - return -1; - } - - star = strchr(path, '*'); - - if (star == NULL) { - return -1; - } - - /* - * C:\data\tmp\input_*.conf - * 0<-----| - */ - p0 = star; - while (path <= p0 && *p0 != '\\') { - p0--; - } - - /* - * C:\data\tmp\input_*.conf - * |---->1 - */ - p1 = star; - while (*p1 && *p1 != '\\') { - p1++; - } - - memcpy(pattern, path, (p1 - path)); - pattern[p1 - path] = '\0'; - - hnd = FindFirstFileA(pattern, &data); - - if (hnd == INVALID_HANDLE_VALUE) { - return 0; - } - - do { - /* Ignore the current and parent dirs */ - if (!strcmp(".", data.cFileName) || !strcmp("..", data.cFileName)) { - continue; - } - - /* Avoid an infinite loop */ - if (strchr(data.cFileName, '*')) { - continue; - } - - /* Create a path (prefix + filename + suffix) */ - memcpy(buf, path, p0 - path + 1); - buf[p0 - path + 1] = '\0'; - - if (FAILED(StringCchCatA(buf, MAX_PATH, data.cFileName))) { - continue; - } - - if (FAILED(StringCchCatA(buf, MAX_PATH, p1))) { - continue; - } - - if (strchr(p1, '*')) { - read_glob(conf, ctx, state, buf); /* recursive */ - continue; - } - - ret = stat(buf, &st); - - if (ret == 0 && (st.st_mode & S_IFMT) == S_IFREG) { - - if (read_config(conf, ctx, state, buf) < 0) { - return -1; - } - } - } while (FindNextFileA(hnd, &data) != 0); - - FindClose(hnd); - return 0; -} -#endif - -static void print_current_state(struct local_ctx *ctx, struct parser_state *state, - yaml_event_t *event) -{ - flb_debug("%*s%s->%s", state->level*2, "", state_str(state->state), - event_type_str(event)); -} - -static void print_current_properties(struct parser_state *state) -{ - struct cfl_list *head; - struct cfl_kvpair *prop; - struct cfl_variant *var; - int idx; - - flb_debug("%*s[%s] PROPERTIES:", state->level*2, "", section_names[state->section]); - - cfl_list_foreach(head, &state->keyvals->list) { - prop = cfl_list_entry(head, struct cfl_kvpair, _head); - switch (prop->val->type) { - case CFL_VARIANT_STRING: - flb_debug("%*s%s: %s", (state->level+2)*2, "", prop->key, prop->val->data.as_string); - break; - case CFL_VARIANT_ARRAY: - flb_debug("%*s%s: [", (state->level+2)*2, "", prop->key); - for (idx = 0; idx < prop->val->data.as_array->entry_count; idx++) { - var = cfl_array_fetch_by_index(prop->val->data.as_array, idx); - flb_debug("%*s%s", (state->level+3)*2, "", var->data.as_string); - } - flb_debug("%*s]", (state->level+2)*2, ""); - break; - } - } -} - -static struct parser_state *get_current_state(struct local_ctx *ctx) -{ - struct parser_state *state; - - if (cfl_list_size(&ctx->states) <= 0) { - return NULL; - } - state = cfl_list_entry_last(&ctx->states, struct parser_state, _head); - return state; -} - -static enum status state_copy_into_config_group(struct parser_state *state, struct flb_cf_group *cf_group) -{ - struct cfl_list *head; - struct cfl_kvpair *kvp; - struct cfl_variant *var; - struct cfl_variant *varr; - struct cfl_array *arr; - struct cfl_array *carr; - struct cfl_kvlist *copy; - int idx; - - if (cf_group == NULL) { - flb_error("no group for processor properties"); - return YAML_FAILURE; - } - - varr = cfl_kvlist_fetch(cf_group->properties, state->key); - - if (varr == NULL) { - arr = cfl_array_create(1); - - if (arr == NULL) { - flb_error("unable to allocate array"); - return YAML_FAILURE; - } - - cfl_array_resizable(arr, CFL_TRUE); - - if (cfl_kvlist_insert_array(cf_group->properties, state->key, arr) < 0) { - cfl_array_destroy(arr); - flb_error("unable to insert into array"); - return YAML_FAILURE; - } - } - else { - arr = varr->data.as_array; - } - - copy = cfl_kvlist_create(); - - if (copy == NULL) { - cfl_array_destroy(arr); - flb_error("unable to allocate kvlist"); - return YAML_FAILURE; - } - - cfl_list_foreach(head, &state->keyvals->list) { - kvp = cfl_list_entry(head, struct cfl_kvpair, _head); - switch (kvp->val->type) { - case CFL_VARIANT_STRING: - - if (cfl_kvlist_insert_string(copy, kvp->key, kvp->val->data.as_string) < 0) { - flb_error("unable to allocate kvlist"); - cfl_kvlist_destroy(copy); - return YAML_FAILURE; - } - break; - case CFL_VARIANT_ARRAY: - carr = cfl_array_create(kvp->val->data.as_array->entry_count); - - if (carr) { - flb_error("unable to allocate array"); - cfl_kvlist_destroy(copy); - return YAML_FAILURE; - } - for (idx = 0; idx < kvp->val->data.as_array->entry_count; idx++) { - var = cfl_array_fetch_by_index(kvp->val->data.as_array, idx); - - if (var == NULL) { - cfl_array_destroy(arr); - flb_error("unable to fetch from array by index"); - return YAML_FAILURE; - } - switch (var->type) { - case CFL_VARIANT_STRING: - - if (cfl_array_append_string(carr, var->data.as_string) < 0) { - flb_error("unable to append string"); - cfl_kvlist_destroy(copy); - cfl_array_destroy(carr); - return YAML_FAILURE; - } - break; - default: - cfl_array_destroy(arr); - flb_error("unable to copy value for property"); - cfl_kvlist_destroy(copy); - cfl_array_destroy(carr); - return YAML_FAILURE; - } - } - - if (cfl_kvlist_insert_array(copy, kvp->key, carr) < 0) { - cfl_array_destroy(arr); - flb_error("unabelt to insert into array"); - flb_error("unable to insert array into kvlist"); - } - break; - default: - flb_error("unknown value type for properties: %d", kvp->val->type); - cfl_kvlist_destroy(copy); - return YAML_FAILURE; - } - } - - if (cfl_array_append_kvlist(arr, copy) < 0) { - flb_error("unable to insert array into kvlist"); - cfl_kvlist_destroy(copy); - return YAML_FAILURE; - } - return YAML_SUCCESS; -} - -static enum status state_copy_into_properties(struct parser_state *state, struct flb_cf *conf, struct cfl_kvlist *properties) -{ - struct cfl_list *head; - struct cfl_kvpair *kvp; - struct cfl_variant *var; - struct cfl_array *arr; - int idx; - - cfl_list_foreach(head, &state->keyvals->list) { - kvp = cfl_list_entry(head, struct cfl_kvpair, _head); - switch (kvp->val->type) { - case CFL_VARIANT_STRING: - var = flb_cf_section_property_add(conf, - properties, - kvp->key, - cfl_sds_len(kvp->key), - kvp->val->data.as_string, - cfl_sds_len(kvp->val->data.as_string)); - - if (var == NULL) { - flb_error("unable to add variant value property"); - return YAML_FAILURE; - } - break; - case CFL_VARIANT_ARRAY: - arr = flb_cf_section_property_add_list(conf, properties, - kvp->key, cfl_sds_len(kvp->key)); - - if (arr == NULL) { - flb_error("unable to add property list"); - return YAML_FAILURE; - } - for (idx = 0; idx < kvp->val->data.as_array->entry_count; idx++) { - var = cfl_array_fetch_by_index(kvp->val->data.as_array, idx); - - if (var == NULL) { - flb_error("unable to retrieve from array by index"); - return YAML_FAILURE; - } - switch (var->type) { - case CFL_VARIANT_STRING: - - if (cfl_array_append_string(arr, var->data.as_string) < 0) { - flb_error("unable to append string to array"); - return YAML_FAILURE; - } - break; - default: - flb_error("unable to copy value for property"); - return YAML_FAILURE; - } - } - break; - default: - flb_error("unknown value type for properties: %d", kvp->val->type); - return YAML_FAILURE; - } - } - return YAML_SUCCESS; -} - -static int consume_event(struct flb_cf *conf, struct local_ctx *ctx, - yaml_event_t *event) -{ - struct parser_state *state; - enum status status; - int ret; - char *value; - struct flb_kv *keyval; - char *last_included; - - last_included = state_get_last(ctx); - - if (last_included == NULL) { - last_included = "**unknown**"; - } - - state = get_current_state(ctx); - - if (state == NULL) { - flb_error("unable to parse yaml: no state"); - return YAML_FAILURE; - } - print_current_state(ctx, state, event); - - switch (state->state) { - case STATE_START: - switch (event->type) { - case YAML_STREAM_START_EVENT: - state = state_push(ctx, STATE_STREAM); - - if (state == NULL) { - flb_error("unable to allocate state"); - return YAML_FAILURE; - } - break; - case YAML_NO_EVENT: - state->state = STATE_STOP; - break; - default: - yaml_error_event(ctx, state, event); - return YAML_FAILURE; - } - break; - - case STATE_STREAM: - switch (event->type) { - case YAML_DOCUMENT_START_EVENT: - state = state_push(ctx, STATE_DOCUMENT); - - if (state == NULL) { - flb_error("unable to allocate state"); - return YAML_FAILURE; - } - break; - case YAML_STREAM_END_EVENT: - state = state_pop(ctx); - - if (state == NULL) { - flb_error("no state left"); - return YAML_FAILURE; - } - break; - default: - yaml_error_event(ctx, state, event); - return YAML_FAILURE; - } - break; - - case STATE_DOCUMENT: - switch (event->type) { - case YAML_MAPPING_START_EVENT: - state = state_push(ctx, STATE_SECTION); - - if (state == NULL) { - flb_error("unable to allocate state"); - return YAML_FAILURE; - } - break; - case YAML_DOCUMENT_END_EVENT: - state = state_pop(ctx); - - if (state == NULL) { - flb_error("no state left"); - return YAML_FAILURE; - } - break; - default: - yaml_error_event(ctx, state, event); - return YAML_FAILURE; - } - break; - - /* - * 'includes' - * -------- - */ - case STATE_INCLUDE: - switch (event->type) { - case YAML_SEQUENCE_START_EVENT: - break; - case YAML_SEQUENCE_END_EVENT: - state = state_pop(ctx); - - if (state == NULL) { - flb_error("no state left"); - return YAML_FAILURE; - } - - if (state->state != STATE_SECTION) { - yaml_error_event(ctx, state, event); - return YAML_FAILURE; - } - break; - case YAML_SCALAR_EVENT: - value = (char *) event->data.scalar.value; - flb_error("[config yaml] including: %s", value); - - if (strchr(value, '*') != NULL) { - ret = read_glob(conf, ctx, state, value); - } - else { - ret = read_config(conf, ctx, state->file, value); - } - - if (ret == -1) { - flb_error("[config] including file '%s' at %s:%zu", - value, - last_included, event->start_mark.line + 1); - return YAML_FAILURE; - } - ctx->level++; - break; - default: - yaml_error_event(ctx, state, event); - return YAML_FAILURE; - } - break; - /* end of 'includes' */ - - /* - * 'customs' - * -------- - */ - case STATE_CUSTOM: - switch (event->type) { - case YAML_SEQUENCE_START_EVENT: - break; - case YAML_MAPPING_START_EVENT: - state = state_push_withvals(ctx, state, STATE_PLUGIN_START); - if (state == NULL) { - flb_error("unable to allocate state"); - return YAML_FAILURE; - } - if (add_section_type(conf, state) == -1) { - flb_error("unable to add section type"); - return YAML_FAILURE; - } - break; - case YAML_SEQUENCE_END_EVENT: - state = state_pop(ctx); - - if (state == NULL) { - flb_error("no state left"); - return YAML_FAILURE; - } - break; - default: - yaml_error_event(ctx, state, event); - return YAML_FAILURE; - } - break; - /* end of 'customs' */ - - case STATE_PIPELINE: - switch (event->type) { - case YAML_SCALAR_EVENT: - value = (char *)event->data.scalar.value; - - if (strcasecmp(value, "inputs") == 0) { - state = state_push_section(ctx, STATE_PLUGIN_INPUT, SECTION_INPUT); - } - else if (strcasecmp(value, "filters") == 0) { - state = state_push_section(ctx, STATE_PLUGIN_FILTER, SECTION_FILTER); - } - else if (strcasecmp(value, "outputs") == 0) { - state = state_push_section(ctx, STATE_PLUGIN_OUTPUT, SECTION_OUTPUT); - } - else { - yaml_error_plugin_category(ctx, state, event, value); - return YAML_FAILURE; - } - - if (state == NULL) { - flb_error("unable to allocate state"); - return YAML_FAILURE; - } - break; - case YAML_MAPPING_START_EVENT: - break; - case YAML_MAPPING_END_EVENT: - state_pop(ctx); - - if (state == NULL) { - flb_error("no state left"); - return YAML_FAILURE; - } - break; - default: - yaml_error_event(ctx, state, event); - return YAML_FAILURE; - } - break; - - case STATE_SECTION: - switch (event->type) { - case YAML_SCALAR_EVENT: - value = (char *)event->data.scalar.value; - - if (strcasecmp(value, "env") == 0) { - state = state_push_section(ctx, STATE_ENV, SECTION_ENV); - if (state == NULL) { - flb_error("unable to allocate state"); - return YAML_FAILURE; - } - } - else if (strcasecmp(value, "pipeline") == 0) { - state = state_push_section(ctx, STATE_PIPELINE, SECTION_PIPELINE); - if (state == NULL) { - flb_error("unable to allocate state"); - return YAML_FAILURE; - } - } - else if (strcasecmp(value, "service") == 0) { - - if (ctx->service_set) { - yaml_error_definition(ctx, state, event, value); - return YAML_FAILURE; - } - - state = state_push_section(ctx, STATE_SERVICE, SECTION_SERVICE); - - if (state == NULL) { - flb_error("unable to allocate state"); - return YAML_FAILURE; - } - - if (state_create_section(conf, state, value) == -1) { - flb_error("unable to allocate section: %s", value); - return YAML_FAILURE; - } - ctx->service_set = 1; - } - else if (strcasecmp(value, "customs") == 0) { - state = state_push_section(ctx, STATE_CUSTOM, SECTION_CUSTOM); - - if (state == NULL) { - flb_error("unable to allocate state"); - return YAML_FAILURE; - } - } - else if (strcasecmp(value, "includes") == 0) { - state = state_push_section(ctx, STATE_INCLUDE, SECTION_INCLUDE); - - if (state == NULL) { - flb_error("unable to allocate state"); - return YAML_FAILURE; - } - } - else { - /* any other main section definition (e.g: similar to STATE_SERVICE) */ - state = state_push(ctx, STATE_OTHER); - - if (state == NULL) { - flb_error("unable to allocate state"); - return YAML_FAILURE; - } - - if (state_create_section(conf, state, value) == -1) { - flb_error("unable to allocate section: %s", value); - return YAML_FAILURE; - } - } - break; - case YAML_MAPPING_END_EVENT: - state = state_pop(ctx); - - if (state == NULL) { - flb_error("no state left"); - return YAML_FAILURE; - } - break; - case YAML_DOCUMENT_END_EVENT: - state = state_pop(ctx); - - if (state == NULL) { - flb_error("no state left"); - return YAML_FAILURE; - } - break; - default: - yaml_error_event(ctx, state, event); - return YAML_FAILURE; - } - break; - - /* service or others */ - case STATE_ENV: - case STATE_SERVICE: - case STATE_OTHER: - switch(event->type) { - case YAML_MAPPING_START_EVENT: - state = state_push(ctx, STATE_SECTION_KEY); - - if (state == NULL) { - flb_error("unable to allocate state"); - return YAML_FAILURE; - } - break; - case YAML_MAPPING_END_EVENT: - state = state_pop(ctx); - - if (state == NULL) { - flb_error("no state left"); - return YAML_FAILURE; - } - - if (state->state != STATE_SECTION) { - yaml_error_event(ctx, state, event); - return YAML_FAILURE; - } - break; - default: - yaml_error_event(ctx, state, event); - return YAML_FAILURE; - } - break; - - case STATE_SECTION_KEY: - switch(event->type) { - case YAML_SCALAR_EVENT: - value = (char *) event->data.scalar.value; - state = state_push_key(ctx, STATE_SECTION_VAL, value); - - if (state == NULL) { - flb_error("unable to allocate state"); - return YAML_FAILURE; - } - break; - case YAML_MAPPING_END_EVENT: - state = state_pop(ctx); - switch (state->state) { - case STATE_SERVICE: - case STATE_ENV: - case STATE_OTHER: - break; - default: - printf("BAD STATE FOR SECTION KEY POP=%s\n", state_str(state->state)); - yaml_error_event(ctx, state, event); - return YAML_FAILURE; - } - state = state_pop(ctx); - - if (state == NULL) { - flb_error("no state left"); - return YAML_FAILURE; - } - break; - default: - yaml_error_event(ctx, state, event); - return YAML_FAILURE; - } - break; - - case STATE_SECTION_VAL: - switch(event->type) { - case YAML_SCALAR_EVENT: - value = (char *) event->data.scalar.value; - - /* Check if the incoming k/v pair set a config environment variable */ - if (state->section == SECTION_ENV) { - keyval = flb_cf_env_property_add(conf, - state->key, flb_sds_len(state->key), - value, strlen(value)); - - if (keyval == NULL) { - flb_error("unable to add key value"); - return YAML_FAILURE; - } - } - else { - - /* register key/value pair as a property */ - if (state->cf_section == NULL) { - flb_error("no section to register key value to"); - return YAML_FAILURE; - } - - if (flb_cf_section_property_add(conf, state->cf_section->properties, - state->key, flb_sds_len(state->key), - value, strlen(value)) < 0) { - flb_error("unable to add property"); - return YAML_FAILURE; - } - } - - state = state_pop(ctx); - - if (state == NULL) { - flb_error("no state left"); - return YAML_FAILURE; - } - - if (state->state != STATE_SECTION_KEY) { - yaml_error_event(ctx, state, event); - return YAML_FAILURE; - } - break; - default: - yaml_error_event(ctx, state, event); - return YAML_FAILURE; - } - break; - - /* Plugin types */ - case STATE_PLUGIN_INPUT: - case STATE_PLUGIN_FILTER: - case STATE_PLUGIN_OUTPUT: - switch(event->type) { - case YAML_SEQUENCE_START_EVENT: - break; - case YAML_SEQUENCE_END_EVENT: - state = state_pop(ctx); - - if (state == NULL) { - flb_error("no state left"); - return YAML_FAILURE; - } - break; - case YAML_MAPPING_START_EVENT: - state = state_push_withvals(ctx, state, STATE_PLUGIN_START); - - if (state == NULL) { - flb_error("unable to allocate state"); - return YAML_FAILURE; - } - - if (add_section_type(conf, state) == -1) { - flb_error("unable to add section type"); - return YAML_FAILURE; - } - break; - case YAML_SCALAR_EVENT: - state = state_pop(ctx); - - if (state == NULL) { - flb_error("no state left"); - return YAML_FAILURE; - } - - if (state->state != STATE_SECTION) { - yaml_error_event(ctx, state, event); - return YAML_FAILURE; - } - break; - case YAML_MAPPING_END_EVENT: - state = state_pop(ctx); - - if (state == NULL) { - flb_error("no state left"); - return YAML_FAILURE; - } - - if (state->state != STATE_SECTION_KEY) { - yaml_error_event(ctx, state, event); - return YAML_FAILURE; - } - break; - default: - yaml_error_event(ctx, state, event); - return YAML_FAILURE; - } - break; - - case STATE_PLUGIN_START: - switch(event->type) { - case YAML_SCALAR_EVENT: - /* Here is where we process all the plugin properties for customs, pipelines - * and processors. - */ - state = state_push_key(ctx, STATE_PLUGIN_VAL, (char *) event->data.scalar.value); - - if (state == NULL) { - flb_error("unable to allocate state"); - return YAML_FAILURE; - } - break; - case YAML_MAPPING_END_EVENT: - print_current_properties(state); - - if (state->section == SECTION_PROCESSOR) { - status = state_copy_into_config_group(state, state->cf_group); - - if (status != YAML_SUCCESS) { - return status; - } - } - else { - status = state_copy_into_properties(state, conf, state->cf_section->properties); - - if (status != YAML_SUCCESS) { - return status; - } - } - - state = state_pop(ctx); - - if (state == NULL) { - flb_error("no state left"); - return YAML_FAILURE; - } - break; - case YAML_SEQUENCE_START_EVENT: /* start a new group */ - - if (state->key == NULL) { - flb_error("no key"); - return YAML_FAILURE; - } - - if (strcmp(state->key, "processors") == 0) { - yaml_error_event(ctx, state, event); - return YAML_FAILURE; - } - - state = state_push_witharr(ctx, state, STATE_PLUGIN_VAL_LIST); - - if (state == NULL) { - flb_error("unable to allocate state"); - return YAML_FAILURE; - } - break; - case YAML_SEQUENCE_END_EVENT: - state = state_pop(ctx); - - if (state == NULL) { - flb_error("no state left"); - return YAML_FAILURE; - } - break; - default: - yaml_error_event(ctx, state, event); - return YAML_FAILURE; - } - break; - - case STATE_PLUGIN_KEY: - switch(event->type) { - case YAML_SCALAR_EVENT: - /* Here is where we process all the plugin properties for customs, pipelines - * and processors. - */ - state = state_push_key(ctx, STATE_PLUGIN_VAL, (char *) event->data.scalar.value); - - if (state == NULL) { - flb_error("unable to allocate state"); - return YAML_FAILURE; - } - break; - case YAML_MAPPING_START_EVENT: - state = state_pop(ctx); - - if (state == NULL) { - flb_error("no state left"); - return YAML_FAILURE; - } - - if (state->state != STATE_PLUGIN_START) { - yaml_error_event(ctx, state, event); - return YAML_FAILURE; - } - break; - case YAML_MAPPING_END_EVENT: - state = state_pop(ctx); - - if (state == NULL) { - flb_error("no state left"); - return YAML_FAILURE; - } - - if (state->state != STATE_PLUGIN_START) { - yaml_error_event(ctx, state, event); - return YAML_FAILURE; - } - break; - case YAML_SEQUENCE_END_EVENT: - state = state_pop(ctx); - - if (state == NULL) { - flb_error("no state left"); - return YAML_FAILURE; - } - break; - default: - yaml_error_event(ctx, state, event); - return YAML_FAILURE; - } - break; - - case STATE_PLUGIN_VAL: - switch(event->type) { - case YAML_SCALAR_EVENT: - - /* register key/value pair as a property */ - if (cfl_kvlist_insert_string(state->keyvals, state->key, (char *)event->data.scalar.value) < 0) { - flb_error("unable to insert string"); - return YAML_FAILURE; - } - - state = state_pop(ctx); - - if (state == NULL) { - flb_error("no state left"); - return YAML_FAILURE; - } - break; - case YAML_SEQUENCE_START_EVENT: /* start a new group */ - state = state_push_witharr(ctx, state, STATE_PLUGIN_VAL_LIST); - - if (state == NULL) { - flb_error("unable to allocate state"); - return YAML_FAILURE; - } - break; - case YAML_MAPPING_START_EVENT: - - if (strcmp(state->key, "processors") == 0) { - state = state_push(ctx, STATE_INPUT_PROCESSORS); - - if (state == NULL) { - flb_error("unable to allocate state"); - return YAML_FAILURE; - } - - if (state_create_group(conf, state, "processors") == YAML_FAILURE) { - return YAML_FAILURE; - } - break; - } - - state = state_push(ctx, STATE_GROUP_KEY); - - if (state == NULL) { - flb_error("unable to allocate state"); - return YAML_FAILURE; - } - /* create group */ - state->values = flb_cf_section_property_add_list(conf, - state->cf_section->properties, - state->key, flb_sds_len(state->key)); - - if (state->values == NULL) { - flb_error("no values"); - return YAML_FAILURE; - } - - state->cf_group = flb_cf_group_create(conf, state->cf_section, state->key, strlen(state->key)); - - if (state->cf_group == NULL) { - flb_error("unable to create group"); - return YAML_FAILURE; - } - break; - case YAML_SEQUENCE_END_EVENT: /* end of group */ - state = state_pop(ctx); - - if (state == NULL) { - flb_error("no state left"); - return YAML_FAILURE; - } - - if (state->state != STATE_PLUGIN_KEY) { - yaml_error_event(ctx, state, event); - return YAML_FAILURE; - } - - state = state_pop(ctx); - - if (state == NULL) { - flb_error("no state left"); - return YAML_FAILURE; - } - - if (state->state != STATE_PLUGIN_KEY) { - yaml_error_event(ctx, state, event); - return YAML_FAILURE; - } - break; - case YAML_MAPPING_END_EVENT: - state = state_pop(ctx); - - if (state == NULL) { - flb_error("no state left"); - return YAML_FAILURE; - } - break; - default: - yaml_error_event(ctx, state, event); - return YAML_FAILURE; - } - break; - - case STATE_PLUGIN_VAL_LIST: - switch(event->type) { - case YAML_SCALAR_EVENT: - - if (state->values == NULL) { - flb_error("unable to add values to list"); - return YAML_FAILURE; - } - - if (cfl_array_append_string(state->values, (char *)event->data.scalar.value) < 0) { - flb_error("unable to add values to list"); - return YAML_FAILURE; - } - break; - case YAML_SEQUENCE_END_EVENT: - - /* register key/value pair as a property */ - if (cfl_kvlist_insert_array(state->keyvals, state->key, state->values) < 0) { - flb_error("unable to insert key values"); - return YAML_FAILURE; - } - - state = state_pop(ctx); - - if (state == NULL) { - flb_error("no state left"); - return YAML_FAILURE; - } - - state = state_pop(ctx); - - if (state == NULL) { - flb_error("no state left"); - return YAML_FAILURE; - } - break; - default: - yaml_error_event(ctx, state, event); - return YAML_FAILURE; - } - break; - - case STATE_INPUT_PROCESSORS: - switch(event->type) { - case YAML_MAPPING_START_EVENT: - break; - case YAML_MAPPING_END_EVENT: - - state = state_pop(ctx); - - if (state == NULL) { - flb_error("no state left"); - return YAML_FAILURE; - } - - state = state_pop(ctx); - - if (state == NULL) { - flb_error("no state left"); - return YAML_FAILURE; - } - break; - case YAML_SCALAR_EVENT: - - /* Check if we are entering a 'logs', 'metrics' or 'traces' section */ - value = (char *) event->data.scalar.value; - - if (strcasecmp(value, "logs") == 0) { - /* logs state */ - state = state_push_key(ctx, STATE_INPUT_PROCESSOR, "logs"); - } - else if (strcasecmp(value, "metrics") == 0) { - /* metrics state */ - state = state_push_key(ctx, STATE_INPUT_PROCESSOR, "metrics"); - } - else if (strcasecmp(value, "traces") == 0) { - /* metrics state */ - state = state_push_key(ctx, STATE_INPUT_PROCESSOR, "traces"); - } - else { - flb_error("[config] unknown processor '%s'", value); - yaml_error_event(ctx, state, event); - return YAML_FAILURE; - } - - if (state == NULL) { - flb_error("unable to allocate state"); - return YAML_FAILURE; - } - break; - default: - yaml_error_event(ctx, state, event); - return YAML_FAILURE; - }; - break; - - case STATE_INPUT_PROCESSOR: - switch(event->type) { - case YAML_SEQUENCE_START_EVENT: - break; - case YAML_SEQUENCE_END_EVENT: - - state = state_pop(ctx); - - if (state == NULL) { - flb_error("no state left"); - return YAML_FAILURE; - } - break; - case YAML_MAPPING_START_EVENT: - - state = state_push_withvals(ctx, state, STATE_PLUGIN_START); - - if (state == NULL) { - flb_error("unable to allocate state"); - return YAML_FAILURE; - } - state->section = SECTION_PROCESSOR; - break; - case YAML_MAPPING_END_EVENT: - return YAML_FAILURE; - break; - default: - yaml_error_event(ctx, state, event); - return YAML_FAILURE; - }; - break; - - /* groups: a group is a sub-section and here we handle the key/value pairs. */ - case STATE_GROUP_KEY: - switch(event->type) { - case YAML_SCALAR_EVENT: - /* grab current value (key) */ - value = (char *) event->data.scalar.value; - - state = state_push_key(ctx, STATE_GROUP_VAL, value); - - if (state == NULL) { - flb_error("unable to allocate state"); - return YAML_FAILURE; - } - break; - case YAML_MAPPING_START_EVENT: - break; - case YAML_MAPPING_END_EVENT: - - state = state_pop(ctx); - - if (state == NULL) { - flb_error("no state left"); - return YAML_FAILURE; - } - - /* This is also the end of the plugin values mapping. - * So we pop an additional state off the stack. - */ - state = state_pop(ctx); - - if (state == NULL) { - flb_error("no state left"); - return YAML_FAILURE; - } - break; - default: - yaml_error_event(ctx, state, event); - return YAML_FAILURE; - } - break; - - case STATE_GROUP_VAL: - switch(event->type) { - case YAML_SCALAR_EVENT: - value = (char *) event->data.scalar.value; - - /* add the kv pair to the active group properties */ - if (flb_cf_section_property_add(conf, state->cf_group->properties, - state->key, flb_sds_len(state->key), - value, strlen(value)) == NULL) { - flb_error("unable to add property"); - return YAML_FAILURE; - } - - state = state_pop(ctx); - - if (state == NULL) { - flb_error("no state left"); - return YAML_FAILURE; - } - break; - default: - yaml_error_event(ctx, state, event); - return YAML_FAILURE; - } - break; - - case STATE_STOP: - break; - } - - return YAML_SUCCESS; -} - -static struct parser_state *state_start(struct local_ctx *ctx, struct file_state *file) -{ - struct parser_state *state; - - state = state_create(NULL, file); - - if (state != NULL) { - cfl_list_add(&state->_head, &ctx->states); - } - - return state; -} - -static struct parser_state *state_push(struct local_ctx *ctx, enum state state_num) -{ - struct parser_state *last = NULL; - struct parser_state *state; - - if (cfl_list_size(&ctx->states) <= 0) { - return NULL; - } - - last = cfl_list_entry_last(&ctx->states, struct parser_state, _head); - - if (last == NULL) { - return NULL; - } - - state = state_create(last->file, last->file); - - if (state == NULL) { - return NULL; - } - state->section = last->section; - state->keyvals = last->keyvals; - state->cf_section = last->cf_section; - state->cf_group = last->cf_group; - state->values = last->values; - state->file = last->file; - state->state = state_num; - state->level = last->level + 1; - state->key = last->key; - - cfl_list_add(&state->_head, &ctx->states); - return state; -} - -static struct parser_state *state_push_section(struct local_ctx *ctx, - enum state state_num, - enum section section) -{ - struct parser_state *state; - - state = state_push(ctx, state_num); - - if (state == NULL) { - return NULL; - } - state->section = section; - - return state; -} - -static struct parser_state *state_push_key(struct local_ctx *ctx, - enum state state_num, - const char *key) -{ - struct parser_state *state; - flb_sds_t skey; - - if (key == NULL) { - return NULL; - } - - skey = flb_sds_create(key); - - if (skey == NULL) { - return NULL; - } - - state = state_push(ctx, state_num); - - if (state == NULL) { - flb_sds_destroy(skey); - return NULL; - } - - state->key = skey; - state->allocation_flags |= HAS_KEY; - return state; -} - -static struct parser_state *state_push_withvals(struct local_ctx *ctx, - struct parser_state *parent, - enum state state_num) -{ - struct parser_state *state; - struct cfl_kvlist *kvlist; - - kvlist = cfl_kvlist_create(); - - if (kvlist == NULL) { - return NULL; - } - - state = state_push(ctx, state_num); - - if (state == NULL) { - cfl_kvlist_destroy(kvlist); - return NULL; - } - - state->keyvals = kvlist; - state->allocation_flags |= HAS_KEYVALS; - - return state; -} - -static struct parser_state *state_push_witharr(struct local_ctx *ctx, - struct parser_state *parent, - enum state state_num) -{ - struct parser_state *state; - - parent->values = cfl_array_create(4); - - if (parent->values == NULL) { - flb_error("parent has no values"); - return NULL; - } - - cfl_array_resizable(parent->values, CFL_TRUE); - - state = state_push(ctx, state_num); - - return state; -} - -static int state_create_section(struct flb_cf *conf, struct parser_state *state, char *name) -{ - - if (state == NULL || conf == NULL || name == NULL) { - return -1; - } - - state->cf_section = flb_cf_section_create(conf, name, 0); - - if (state->cf_section == NULL) { - return -1; - } - - return 0; -} - -static int state_create_group(struct flb_cf *conf, struct parser_state *state, char *name) -{ - if (state == NULL || conf == NULL || name == NULL) { - return -1; - } - - state->cf_group = flb_cf_group_create(conf, state->cf_section, - "processors", strlen("processors")); - - if (state->cf_group == NULL) { - return -1; - } - - return YAML_SUCCESS; -} - -static struct parser_state *state_pop(struct local_ctx *ctx) -{ - struct parser_state *last; - - if (ctx == NULL) { - return NULL; - } - - if (cfl_list_size(&ctx->states) <= 0) { - return NULL; - } - - last = cfl_list_entry_last(&ctx->states, struct parser_state, _head); - cfl_list_del(&last->_head); - - if (last->allocation_flags & HAS_KEY) { - flb_sds_destroy(last->key); - } - - if (last->allocation_flags & HAS_KEYVALS) { - cfl_kvlist_destroy(last->keyvals); - } - - state_destroy(last); - - if (cfl_list_size(&ctx->states) <= 0) { - return NULL; - } - - return cfl_list_entry_last(&ctx->states, struct parser_state, _head); -} - -static void state_destroy(struct parser_state *s) -{ - flb_free(s); -} - -static struct parser_state *state_create(struct file_state *parent, struct file_state *file) -{ - struct parser_state *state; - - /* allocate context */ - state = flb_calloc(1, sizeof(struct parser_state)); - - if (!state) { - flb_errno(); - return NULL; - } - - state->file = file; -#ifndef FLB_HAVE_STATIC_CONF - - if (parent) { - state->file->parent = parent; - } - -#else - - s->file->name = flb_sds_create("***static***"); - s->file->path = flb_sds_create("***static***"); - -#endif - - return state; -} - -static int read_config(struct flb_cf *conf, struct local_ctx *ctx, - struct file_state *parent, char *cfg_file) -{ - int ret; - int status; - int code = 0; - struct parser_state *state; - flb_sds_t include_dir = NULL; - flb_sds_t include_file = NULL; - yaml_parser_t parser; - yaml_event_t event; - FILE *fh; - struct file_state fstate; - - if (parent && cfg_file[0] != '/') { - - include_dir = flb_sds_create_size(strlen(cfg_file) + strlen(parent->path)); - - if (include_dir == NULL) { - flb_error("unable to create filename"); - return -1; - } - - if (flb_sds_printf(&include_dir, "%s/%s", parent->path, cfg_file) == NULL) { - flb_error("unable to create full filename"); - return -1; - } - - } - else { - - include_dir = flb_sds_create(cfg_file); - - if (include_dir == NULL) { - flb_error("unable to create filename"); - return -1; - } - } - - include_file = flb_sds_create(include_dir); - - if (include_file == NULL) { - flb_error("unable to create include filename"); - flb_sds_destroy(include_dir); - return -1; - } - - fstate.name = basename(include_dir); - fstate.path = dirname(include_dir); - - fstate.parent = parent; - - state = state_start(ctx, &fstate); - - if (!state) { - flb_error("unable to push initial include file state: %s", cfg_file); - flb_sds_destroy(include_dir); - flb_sds_destroy(include_file); - return -1; - } - - /* check if this file has been included before */ - ret = is_file_included(ctx, include_file); - - if (ret) { - flb_error("[config] file '%s' is already included", cfg_file); - flb_sds_destroy(include_dir); - flb_sds_destroy(include_file); - state_destroy(state); - return -1; - } - - flb_debug("============ %s ============", cfg_file); - fh = fopen(include_file, "r"); - - if (!fh) { - flb_errno(); - flb_sds_destroy(include_dir); - flb_sds_destroy(include_file); - state_destroy(state); - return -1; - } - - /* add file to the list of included files */ - ret = flb_slist_add(&ctx->includes, include_file); - - if (ret == -1) { - flb_error("[config] could not register file %s", cfg_file); - fclose(fh); - flb_sds_destroy(include_dir); - flb_sds_destroy(include_file); - state_destroy(state); - return -1; - } - ctx->level++; - - yaml_parser_initialize(&parser); - yaml_parser_set_input_file(&parser, fh); - - do { - status = yaml_parser_parse(&parser, &event); - - if (status == YAML_FAILURE) { - flb_error("[config] invalid YAML format in file %s", cfg_file); - code = -1; - goto done; - } - - status = consume_event(conf, ctx, &event); - - if (status == YAML_FAILURE) { - flb_error("yaml error"); - code = -1; - goto done; - } - - yaml_event_delete(&event); - state = cfl_list_entry_last(&ctx->states, struct parser_state, _head); - - } while (state->state != STATE_STOP); - - flb_debug("=============================="); -done: - - if (code == -1) { - yaml_event_delete(&event); - } - - yaml_parser_delete(&parser); - - /* free all remaining states */ - if (code == -1) { - while (state = state_pop(ctx)); - } - else { - state = state_pop(ctx); - } - - fclose(fh); - ctx->level--; - - flb_sds_destroy(include_file); - flb_sds_destroy(include_dir); - - return code; -} - -static int local_init(struct local_ctx *ctx) -{ - /* reset the state */ - memset(ctx, '\0', sizeof(struct local_ctx)); - cfl_list_init(&ctx->states); - ctx->level = 0; - flb_slist_create(&ctx->includes); - - return 0; -} - -static void local_exit(struct local_ctx *ctx) -{ - flb_slist_destroy(&ctx->includes); -} - -struct flb_cf *flb_cf_yaml_create(struct flb_cf *conf, char *file_path, - char *buf, size_t size) -{ - int ret; - struct local_ctx ctx; - - if (!conf) { - conf = flb_cf_create(); - if (!conf) { - return NULL; - } - - flb_cf_set_origin_format(conf, FLB_CF_YAML); - } - - /* initialize the parser state */ - ret = local_init(&ctx); - - if (ret == -1) { - flb_cf_destroy(conf); - return NULL; - } - - /* process the entry poing config file */ - ret = read_config(conf, &ctx, NULL, file_path); - - if (ret == -1) { - flb_cf_destroy(conf); - local_exit(&ctx); - return NULL; - } - - local_exit(&ctx); - return conf; -} diff --git a/fluent-bit/src/config_format/flb_config_format.c b/fluent-bit/src/config_format/flb_config_format.c deleted file mode 100644 index 3a62fc9e2..000000000 --- a/fluent-bit/src/config_format/flb_config_format.c +++ /dev/null @@ -1,878 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include <ctype.h> - -#include <fluent-bit/flb_info.h> -#include <fluent-bit/flb_mem.h> -#include <fluent-bit/flb_log.h> -#include <fluent-bit/flb_kv.h> -#include <fluent-bit/flb_config_format.h> - -#include <cfl/cfl.h> -#include <cfl/cfl_sds.h> -#include <cfl/cfl_variant.h> -#include <cfl/cfl_kvlist.h> - -int flb_cf_file_read() -{ - return 0; -} - -/* Retrieve the proper key name, it tries to auto-detect Yaml camelcase and convert it to snake_case */ -flb_sds_t flb_cf_key_translate(struct flb_cf *cf, char *key, int len) -{ - int i; - int x = 0; - int is_upper; - flb_sds_t out; - - if (!key || len <= 0) { - return NULL; - } - - /* If we got something in classic format, just convert it to lowercase and return */ - if (cf->format == FLB_CF_CLASSIC) { - out = flb_sds_create_len(key, len); - if (!out) { - return NULL; - } - - for (i = 0; i < len; i++) { - out[i] = tolower(key[i]); - } - flb_sds_len_set(out, len); - return out; - } - - /* - * First step is try to identify if the incoming key is a strict Yaml camelcase format - */ - - /* does it start with a lowercase character ? */ - if (!islower(key[0])) { - return flb_sds_create_len(key, len); - } - - /* copy content and check if we have underscores */ - out = flb_sds_create_size(len * 2); - flb_sds_cat_safe(&out, key, len); - - for (i = 0; i < len; i++) { - if (key[i] == '_') { - /* the config is classic mode, so it's safe to return the same copy of the content */ - for (i = 0; i < len; i++) { - out[i] = tolower(key[i]); - } - flb_sds_len_set(out, len); - return out; - } - } - - /* translate from camelCase to snake_case */ - for (i = 0; i < len; i++) { - is_upper = isupper(key[i]); - if (is_upper && i > 0) { - out[x++] = '_'; - } - out[x++] = tolower(key[i]); - - } - out[x] = '\0'; - flb_sds_len_set(out, x); - return out; -} - -struct flb_cf *flb_cf_create() -{ - struct flb_cf *ctx; - - ctx = flb_calloc(1, sizeof(struct flb_cf)); - if (!ctx) { - flb_errno(); - return NULL; - } - ctx->format = FLB_CF_CLASSIC; - - /* env vars */ - mk_list_init(&ctx->env); - - /* meta commands */ - mk_list_init(&ctx->metas); - - /* parsers */ - mk_list_init(&ctx->parsers); - mk_list_init(&ctx->multiline_parsers); - - /* custom plugins */ - mk_list_init(&ctx->customs); - - /* pipeline */ - mk_list_init(&ctx->inputs); - mk_list_init(&ctx->filters); - mk_list_init(&ctx->outputs); - - /* other sections */ - mk_list_init(&ctx->others); - - /* general list for sections */ - mk_list_init(&ctx->sections); - - return ctx; -} - -void flb_cf_destroy(struct flb_cf *cf) -{ - flb_kv_release(&cf->env); - flb_kv_release(&cf->metas); - flb_cf_section_destroy_all(cf); - flb_free(cf); -} - -int flb_cf_set_origin_format(struct flb_cf *cf, int format) -{ -#ifdef FLB_HAVE_LIBYAML - if (format != FLB_CF_CLASSIC && format != FLB_CF_YAML) { -#else - if (format != FLB_CF_CLASSIC) { -#endif - return -1; - } - - cf->format = format; - return 0; -} - -static enum section_type get_section_type(char *name, int len) -{ - if (strncasecmp(name, "SERVICE", len) == 0) { - return FLB_CF_SERVICE; - } - else if (strncasecmp(name, "PARSER", len) == 0) { - return FLB_CF_PARSER; - } - else if (strncasecmp(name, "MULTILINE_PARSER", len) == 0) { - return FLB_CF_MULTILINE_PARSER; - } - else if (strncasecmp(name, "CUSTOM", len) == 0 || - strncasecmp(name, "CUSTOMS", len) == 0) { - return FLB_CF_CUSTOM; - } - else if (strncasecmp(name, "INPUT", len) == 0 || - strncasecmp(name, "INPUTS", len) == 0) { - return FLB_CF_INPUT; - } - else if (strncasecmp(name, "FILTER", len) == 0 || - strncasecmp(name, "FILTERS", len) == 0) { - return FLB_CF_FILTER; - } - else if (strncasecmp(name, "OUTPUT", len) == 0 || - strncasecmp(name, "OUTPUTS", len) == 0) { - return FLB_CF_OUTPUT; - } - - return FLB_CF_OTHER; -} - -int flb_cf_plugin_property_add(struct flb_cf *cf, - struct cfl_kvlist *kv_list, - char *k_buf, size_t k_len, - char *v_buf, size_t v_len) -{ - int ret; - flb_sds_t key; - flb_sds_t val; - - if (k_len == 0) { - k_len = strlen(k_buf); - } - if (v_len == 0) { - v_len = strlen(v_buf); - } - - - key = flb_cf_key_translate(cf, k_buf, k_len); - if (key == NULL) { - return -1; - } - - val = flb_sds_create_len(v_buf, v_len); - if (val == NULL) { - flb_sds_destroy(key); - return -1; - } - - /* sanitize key and value by removing empty spaces */ - ret = flb_sds_trim(key); - if (ret == -1) { - flb_cf_error_set(cf, FLB_CF_ERROR_KV_INVALID_KEY); - flb_sds_destroy(key); - flb_sds_destroy(val); - return -1; - } - - ret = flb_sds_trim(val); - if (ret == -1) { - flb_cf_error_set(cf, FLB_CF_ERROR_KV_INVALID_VAL); - flb_sds_destroy(key); - flb_sds_destroy(val); - return ret; - } - - ret = cfl_kvlist_insert_string(kv_list, key, val); - flb_sds_destroy(key); - flb_sds_destroy(val); - return ret; -} - -struct cfl_variant *flb_cf_section_property_add(struct flb_cf *cf, - struct cfl_kvlist *kv_list, - char *k_buf, size_t k_len, - char *v_buf, size_t v_len) -{ - int rc; - flb_sds_t key; - flb_sds_t val; - struct cfl_variant *var; - - if (k_len == 0) { - k_len = strlen(k_buf); - } - - key = flb_cf_key_translate(cf, k_buf, k_len); - if (key == NULL) { - goto key_error; - } - - /* sanitize key and value by removing empty spaces */ - rc = flb_sds_trim(key); - if (rc == -1) { - flb_cf_error_set(cf, FLB_CF_ERROR_KV_INVALID_KEY); - goto val_error; - } - - if (v_len == 0) { - v_len = strlen(v_buf); - } - val = flb_sds_create_len(v_buf, v_len); - if (val == NULL) { - goto val_error; - } - /* sanitize key and value by removing empty spaces */ - rc = flb_sds_trim(val); - if (rc == -1) { - flb_cf_error_set(cf, FLB_CF_ERROR_KV_INVALID_VAL); - goto var_error; - } - - var = cfl_variant_create_from_string(val); - if (var == NULL) { - goto var_error; - } - - rc = cfl_kvlist_insert(kv_list, key, var); - if (rc < 0) { - goto insert_error; - } - - flb_sds_destroy(val); - flb_sds_destroy(key); - return var; - -insert_error: - cfl_variant_destroy(var); -var_error: - flb_sds_destroy(val); -val_error: - flb_sds_destroy(key); -key_error: - return NULL; -} - -struct cfl_array *flb_cf_section_property_add_list(struct flb_cf *cf, - struct cfl_kvlist *kv_list, - char *k_buf, size_t k_len) -{ - int rc; - flb_sds_t key; - struct cfl_array *arr; - - - if (k_len == 0) { - k_len = strlen(k_buf); - } - - key = flb_cf_key_translate(cf, k_buf, k_len); - if (key == NULL) { - goto key_error; - } - - arr = cfl_array_create(10); - if (arr == NULL) { - goto array_error; - } - cfl_array_resizable(arr, 1); - - /* sanitize key and value by removing empty spaces */ - rc = flb_sds_trim(key); - if (rc == -1) { - flb_cf_error_set(cf, FLB_CF_ERROR_KV_INVALID_KEY); - goto cfg_error; - } - - rc = cfl_kvlist_insert_array(kv_list, key, arr); - if (rc < 0) { - goto cfg_error; - } - - flb_sds_destroy(key); - return arr; - -cfg_error: - cfl_array_destroy(arr); -array_error: - flb_sds_destroy(key); -key_error: - return NULL; -} - -flb_sds_t flb_cf_section_property_get_string(struct flb_cf *cf, struct flb_cf_section *s, - char *key) -{ - (void) cf; - flb_sds_t tkey; - struct cfl_variant *val; - flb_sds_t ret = NULL; - struct cfl_variant *entry; - int i; - int len; - - len = strlen(key); - tkey = flb_cf_key_translate(cf, key, len); - - val = cfl_kvlist_fetch(s->properties, key); - flb_sds_destroy(tkey); - if (val == NULL) { - return NULL; - } - - if (val->type == CFL_VARIANT_STRING) { - ret = flb_sds_create(val->data.as_string); - } - if (val->type == CFL_VARIANT_ARRAY) { - // recreate the format SLISTS are expecting... - ret = flb_sds_create(" "); - for (i = 0; i < val->data.as_array->entry_count; i++) { - entry = val->data.as_array->entries[i]; - if (entry->type != CFL_VARIANT_STRING) { - flb_sds_destroy(ret); - return NULL; - } - if ((i+1) < val->data.as_array->entry_count) { - flb_sds_printf(&ret, "%s ", entry->data.as_string); - } else { - flb_sds_printf(&ret, "%s", entry->data.as_string); - } - } - } - return ret; -} - -struct cfl_variant * flb_cf_section_property_get(struct flb_cf *cf, struct flb_cf_section *s, - char *key) -{ - (void) cf; - return cfl_kvlist_fetch(s->properties, key); -} - -struct flb_kv *flb_cf_env_property_add(struct flb_cf *cf, - char *k_buf, size_t k_len, - char *v_buf, size_t v_len) -{ - int ret; - struct flb_kv *kv; - - if (k_len == 0) { - k_len = strlen(k_buf); - } - if (v_len == 0) { - v_len = strlen(v_buf); - } - - kv = flb_kv_item_create_len(&cf->env, k_buf, k_len, v_buf, v_len); - if (!kv) { - return NULL; - } - - /* sanitize key and value by removing empty spaces */ - ret = flb_sds_trim(kv->key); - if (ret == -1) { - flb_cf_error_set(cf, FLB_CF_ERROR_KV_INVALID_KEY); - flb_kv_item_destroy(kv); - return NULL; - } - - ret = flb_sds_trim(kv->val); - if (ret == -1) { - flb_cf_error_set(cf, FLB_CF_ERROR_KV_INVALID_VAL); - flb_kv_item_destroy(kv); - return NULL; - } - - return kv; -} - -static struct flb_kv *meta_property_add(struct flb_cf *cf, - char *k_buf, size_t k_len, - char *v_buf, size_t v_len) -{ - int ret; - struct flb_kv *kv; - - if (k_len == 0) { - k_len = strlen(k_buf); - } - if (v_len == 0) { - v_len = strlen(v_buf); - } - - kv = flb_kv_item_create_len(&cf->metas, k_buf, k_len, v_buf, v_len); - if (!kv) { - return NULL; - } - - /* sanitize key and value by removing empty spaces */ - ret = flb_sds_trim(kv->key); - if (ret == -1) { - flb_cf_error_set(cf, FLB_CF_ERROR_KV_INVALID_KEY); - flb_kv_item_destroy(kv); - return NULL; - } - - ret = flb_sds_trim(kv->val); - if (ret == -1) { - flb_cf_error_set(cf, FLB_CF_ERROR_KV_INVALID_VAL); - flb_kv_item_destroy(kv); - return NULL; - } - - return kv; -} - -struct flb_kv *flb_cf_meta_property_add(struct flb_cf *cf, char *meta, int len) -{ - int xlen; - char *p; - char *tmp; - - if (len <= 0) { - len = strlen(meta); - if (len == 0) { - return NULL; - } - } - - if (meta[0] != '@') { - flb_cf_error_set(cf, FLB_CF_ERROR_META_CHAR); - return NULL; - } - - p = meta; - tmp = strchr(p, ' '); - if (tmp == NULL) { - return NULL; - } - xlen = (tmp - p); - - /* create k/v pair */ - return meta_property_add(cf, - meta + 1, xlen - 1, - meta + xlen + 1, len - xlen - 1); -} - -struct flb_cf_group *flb_cf_group_create(struct flb_cf *cf, struct flb_cf_section *s, - char *name, int len) -{ - struct flb_cf_group *g; - - if (!name || strlen(name) == 0 || len < 1) { - return NULL; - } - - /* section context */ - g = flb_malloc(sizeof(struct flb_cf_group)); - if (!g) { - flb_errno(); - return NULL; - } - - /* initialize lists */ - g->properties = cfl_kvlist_create(); - if (g->properties == NULL) { - flb_free(g); - return NULL; - } - - /* determinate type by name */ - if (len <= 0) { - len = strlen(name); - } - - /* create a NULL terminated name */ - g->name = flb_sds_create_len(name, len); - if (!g->name) { - cfl_kvlist_destroy(g->properties); - flb_free(g); - return NULL; - } - - /* link to global section */ - mk_list_add(&g->_head, &s->groups); - - return g; -} - -struct flb_cf_group *flb_cf_group_get(struct flb_cf *cf, struct flb_cf_section *s, char *name) -{ - struct mk_list *head; - struct flb_cf_group *g; - - mk_list_foreach(head, &s->groups) { - g = mk_list_entry(head, struct flb_cf_group, _head); - if (strcasecmp(g->name, name) == 0){ - return g; - } - } - - return NULL; -} - -void flb_cf_group_print(struct flb_cf_group *g) -{ - cfl_kvlist_print(stdout, g->properties); -} - -void flb_cf_group_destroy(struct flb_cf_group *g) -{ - if (g->name) { - flb_sds_destroy(g->name); - } - - cfl_kvlist_destroy(g->properties); - mk_list_del(&g->_head); - flb_free(g); -} - -struct flb_cf_section *flb_cf_section_create(struct flb_cf *cf, char *name, int len) -{ - int type; - struct flb_cf_section *s; - - if (!name) { - return NULL; - } - - /* determinate type by name */ - if (len <= 0) { - len = strlen(name); - } - - /* get the section type */ - type = get_section_type(name, len); - - /* check if 'service' already exists */ - if (type == FLB_CF_SERVICE && cf->service) { - return cf->service; - } - - /* section context */ - s = flb_malloc(sizeof(struct flb_cf_section)); - if (!s) { - flb_errno(); - return NULL; - } - - /* initialize lists */ - s->properties = cfl_kvlist_create(); - mk_list_init(&s->groups); - - /* create a NULL terminated name */ - s->name = flb_sds_create_len(name, len); - if (!s->name) { - flb_free(s->properties); - flb_free(s); - return NULL; - } - s->type = type; - - if (type == FLB_CF_SERVICE && !cf->service) { - cf->service = s; - } - - /* link to global section */ - mk_list_add(&s->_head, &cf->sections); - - /* link to list per type */ - if (type == FLB_CF_PARSER) { - mk_list_add(&s->_head_section, &cf->parsers); - } - else if (type == FLB_CF_MULTILINE_PARSER) { - mk_list_add(&s->_head_section, &cf->multiline_parsers); - } - else if (type == FLB_CF_CUSTOM) { - mk_list_add(&s->_head_section, &cf->customs); - } - else if (type == FLB_CF_INPUT) { - mk_list_add(&s->_head_section, &cf->inputs); - } - else if (type == FLB_CF_FILTER) { - mk_list_add(&s->_head_section, &cf->filters); - } - else if (type == FLB_CF_OUTPUT) { - mk_list_add(&s->_head_section, &cf->outputs); - } - else if (type == FLB_CF_OTHER) { - mk_list_add(&s->_head_section, &cf->others); - } - - return s; -} - -/* returns the first match of a section that it name matches 'name' parameter */ -struct flb_cf_section *flb_cf_section_get_by_name(struct flb_cf *cf, char *name) -{ - struct mk_list *head; - struct flb_cf_section *s; - - mk_list_foreach(head, &cf->sections) { - s = mk_list_entry(head, struct flb_cf_section, _head); - if (strcasecmp(s->name, name) == 0) { - return s; - } - } - - return NULL; -} - -void flb_cf_section_destroy(struct flb_cf *cf, struct flb_cf_section *s) -{ - struct mk_list *tmp; - struct mk_list *head; - struct flb_cf_group *g; - - if (s->name) { - flb_sds_destroy(s->name); - s->name = NULL; - } - cfl_kvlist_destroy(s->properties); - - /* groups */ - mk_list_foreach_safe(head, tmp, &s->groups) { - g = mk_list_entry(head, struct flb_cf_group, _head); - flb_cf_group_destroy(g); - } - - /* unlink */ - mk_list_del(&s->_head); - - if (s->type != FLB_CF_SERVICE) { - mk_list_del(&s->_head_section); - } - - flb_free(s); -} - -static void section_destroy_list(struct flb_cf *cf, struct mk_list *list) -{ - struct mk_list *tmp; - struct mk_list *head; - struct flb_cf_section *s; - - mk_list_foreach_safe(head, tmp, list) { - s = mk_list_entry(head, struct flb_cf_section, _head); - flb_cf_section_destroy(cf, s); - } -} - -void flb_cf_section_destroy_all(struct flb_cf *cf) -{ - section_destroy_list(cf, &cf->sections); -} - -/* - * Helpers - * ------- - */ - -static char *section_type_str(int type) -{ - switch (type) { - case FLB_CF_SERVICE: - return "SERVICE"; - case FLB_CF_PARSER: - return "PARSER"; - case FLB_CF_MULTILINE_PARSER: - return "MULTILINE_PARSER"; - case FLB_CF_CUSTOM: - return "CUSTOM"; - case FLB_CF_INPUT: - return "INPUT"; - case FLB_CF_FILTER: - return "FILTER"; - case FLB_CF_OUTPUT: - return "OUTPUT"; - case FLB_CF_OTHER: - return "OTHER"; - default: - return "error / unknown"; - } - - return NULL; -} - -static void dump_section(struct flb_cf_section *s) -{ - struct mk_list *head; - struct cfl_list *p_head; - struct cfl_kvpair *kv; - struct flb_cf_group *g; - - printf("> section:\n name: %s\n type: %s\n", - s->name, section_type_str(s->type)); - - if (cfl_list_size(&s->properties->list) > 0) { - printf(" properties:\n"); - cfl_list_foreach(p_head, &s->properties->list) { - kv = cfl_list_entry(p_head, struct cfl_kvpair, _head); - printf(" - %-15s: %s\n", kv->key, kv->val->data.as_string); - } - } - else { - printf(" properties: NONE\n"); - } - - if (mk_list_size(&s->groups) <= 0) { - printf(" groups : NONE\n"); - return; - } - - mk_list_foreach(head, &s->groups) { - g = mk_list_entry(head, struct flb_cf_group, _head); - printf(" > group:\n name: %s\n", g->name); - - if (cfl_list_size(&g->properties->list) > 0) { - printf(" properties:\n"); - cfl_list_foreach(p_head, &g->properties->list) { - kv = cfl_list_entry(p_head, struct cfl_kvpair, _head); - printf(" - %-11s: %s\n", kv->key, kv->val->data.as_string); - } - } - else { - printf(" properties: NONE\n"); - } - } -} - -static void dump_env(struct mk_list *list) -{ - struct mk_list *head; - struct flb_kv *kv; - - if (mk_list_size(list) == 0) { - return; - } - - printf("> env:\n"); - - mk_list_foreach(head, list) { - kv = mk_list_entry(head, struct flb_kv, _head); - printf(" - %-15s: %s\n", kv->key, kv->val); - } -} - -static void dump_metas(struct mk_list *list) -{ - struct mk_list *head; - struct flb_kv *kv; - - if (mk_list_size(list) == 0) { - return; - } - - printf("> metas:\n"); - - mk_list_foreach(head, list) { - kv = mk_list_entry(head, struct flb_kv, _head); - printf(" - %-15s: %s\n", kv->key, kv->val); - } -} - -static void dump_section_list(struct mk_list *list) -{ - struct mk_list *head; - struct flb_cf_section *s; - - mk_list_foreach(head, list) { - s = mk_list_entry(head, struct flb_cf_section, _head); - dump_section(s); - } -} - -void flb_cf_dump(struct flb_cf *cf) -{ - dump_metas(&cf->metas); - dump_env(&cf->env); - dump_section_list(&cf->sections); -} - -struct flb_cf *flb_cf_create_from_file(struct flb_cf *cf, char *file) -{ - int format = FLB_CF_FLUENTBIT; - char *ptr; - - if (!file) { - return NULL; - } - - ptr = strrchr(file, '.'); - if (!ptr) { - format = FLB_CF_FLUENTBIT; - } - else { - if (strcasecmp(ptr, ".conf") == 0) { - format = FLB_CF_FLUENTBIT; - } -#ifdef FLB_HAVE_LIBYAML - else if (strcasecmp(ptr, ".yaml") == 0 || strcasecmp(ptr, ".yml") == 0) { - format = FLB_CF_YAML; - } -#endif - } - - if (format == FLB_CF_FLUENTBIT) { - cf = flb_cf_fluentbit_create(cf, file, NULL, 0); - } -#ifdef FLB_HAVE_LIBYAML - else if (format == FLB_CF_YAML) { - cf = flb_cf_yaml_create(cf, file, NULL, 0); - } -#endif - - return cf; - } - diff --git a/fluent-bit/src/flb_api.c b/fluent-bit/src/flb_api.c deleted file mode 100644 index 6c6a60ea6..000000000 --- a/fluent-bit/src/flb_api.c +++ /dev/null @@ -1,56 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <fluent-bit/flb_info.h> -#include <fluent-bit/flb_api.h> -#include <fluent-bit/flb_mem.h> -#include <fluent-bit/flb_log.h> - -#include <fluent-bit/flb_input.h> -#include <fluent-bit/flb_output.h> - -struct flb_api *flb_api_create() -{ - struct flb_api *api; - - api = flb_malloc(sizeof(struct flb_api)); - if (!api) { - flb_errno(); - return NULL; - } - - api->output_get_property = flb_output_get_property; - api->input_get_property = flb_input_get_property; - -#ifdef FLB_HAVE_METRICS - api->output_get_cmt_instance = flb_output_get_cmt_instance; - api->input_get_cmt_instance = flb_input_get_cmt_instance; -#endif - - api->log_print = flb_log_print; - api->input_log_check = flb_input_log_check; - api->output_log_check = flb_output_log_check; - - return api; -} - -void flb_api_destroy(struct flb_api *api) -{ - flb_free(api); -} diff --git a/fluent-bit/src/flb_avro.c b/fluent-bit/src/flb_avro.c deleted file mode 100644 index fe45fb5da..000000000 --- a/fluent-bit/src/flb_avro.c +++ /dev/null @@ -1,397 +0,0 @@ -/*-*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <assert.h> -#include <stdlib.h> -#include <string.h> - -#include <fluent-bit/flb_macros.h> -#include <fluent-bit/flb_log.h> -#include <fluent-bit/flb_mem.h> -#include <fluent-bit/flb_error.h> -#include <fluent-bit/flb_sds.h> -#include <fluent-bit/flb_avro.h> - -static inline int do_avro(bool call, const char *msg) { - if (call) { - flb_error("%s:\n %s\n", msg, avro_strerror()); - return FLB_FALSE; - } - return FLB_TRUE; -} - -avro_value_iface_t *flb_avro_init(avro_value_t *aobject, char *json, size_t json_len, avro_schema_t *aschema) -{ - - flb_debug("in flb_avro_init:before error:%s:json len:%zu:\n", avro_strerror(), json_len); - - if (avro_schema_from_json_length(json, json_len, aschema)) { - flb_error("Unable to parse aobject schema:%s:error:%s:\n", json, avro_strerror()); - return NULL; - } - - avro_value_iface_t *aclass = avro_generic_class_from_schema(*aschema); - - if(aclass == NULL) { - flb_error("Unable to instantiate class from schema:%s:\n", avro_strerror()); - return NULL; - } - - if(avro_generic_value_new(aclass, aobject) != 0) { - flb_error("Unable to allocate new avro value:%s:\n", avro_strerror()); - return NULL; - } - - return aclass; -} - -int msgpack2avro(avro_value_t *val, msgpack_object *o) -{ - int ret = FLB_FALSE; - flb_debug("in msgpack2avro\n"); - - assert(val != NULL); - assert(o != NULL); - - switch(o->type) { - case MSGPACK_OBJECT_NIL: - flb_debug("got a nil:\n"); - ret = do_avro(avro_value_set_null(val), "failed on nil"); - break; - - case MSGPACK_OBJECT_BOOLEAN: - flb_debug("got a bool:%s:\n", (o->via.boolean ? "true" : "false")); - ret = do_avro(avro_value_set_boolean(val, o->via.boolean), "failed on bool"); - break; - - case MSGPACK_OBJECT_POSITIVE_INTEGER: - // for reference src/objectc.c +/msgpack_pack_object -#if defined(PRIu64) - // msgpack_pack_fix_uint64 - flb_debug("got a posint: %" PRIu64 "\n", o->via.u64); - ret = do_avro(avro_value_set_int(val, o->via.u64), "failed on posint"); -#else - if (o.via.u64 > ULONG_MAX) - flb_warn("over \"%lu\"", ULONG_MAX); - ret = do_avro(avro_value_set_int(val, ULONG_MAX), "failed on posint"); - else - flb_debug("got a posint: %lu\n", (unsigned long)o->via.u64); - ret = do_avro(avro_value_set_int(val, o->via.u64), "failed on posint"); -#endif - - break; - - case MSGPACK_OBJECT_NEGATIVE_INTEGER: -#if defined(PRIi64) - flb_debug("got a negint: %" PRIi64 "\n", o->via.i64); - ret = do_avro(avro_value_set_int(val, o->via.i64), "failed on negint"); -#else - if (o->via.i64 > LONG_MAX) - flb_warn("over +\"%ld\"", LONG_MAX); - ret = do_avro(avro_value_set_int(val, LONG_MAX), "failed on negint"); - else if (o->via.i64 < LONG_MIN) - flb_warn("under -\"%ld\"", LONG_MIN); - ret = do_avro(avro_value_set_int(val, LONG_MIN), "failed on negint"); - else - flb_debug("got a negint: %ld\n", (signed long)o->via.i64); - ret = do_avro(avro_value_set_int(val, o->via.i64), "failed on negint"); -#endif - break; - - case MSGPACK_OBJECT_FLOAT32: - case MSGPACK_OBJECT_FLOAT64: - flb_debug("got a float: %f\n", o->via.f64); - ret = do_avro(avro_value_set_float(val, o->via.f64), "failed on float"); - break; - - case MSGPACK_OBJECT_STR: - { - flb_debug("got a string: \""); - - if (flb_log_check(FLB_LOG_DEBUG)) - fwrite(o->via.str.ptr, o->via.str.size, 1, stderr); - flb_debug("\"\n"); - - flb_debug("setting string:%.*s:\n", o->via.str.size, o->via.str.ptr); - flb_sds_t cstr = flb_sds_create_len(o->via.str.ptr, o->via.str.size); - ret = do_avro(avro_value_set_string_len(val, cstr, flb_sds_len(cstr) + 1), "failed on string"); - flb_sds_destroy(cstr); - flb_debug("set string\n"); - } - break; - - case MSGPACK_OBJECT_BIN: - flb_debug("got a binary\n"); - ret = do_avro(avro_value_set_bytes(val, (void *)o->via.bin.ptr, o->via.bin.size), "failed on bin"); - break; - - case MSGPACK_OBJECT_EXT: -#if defined(PRIi8) - flb_debug("got an ext: %" PRIi8 ")", o->via.ext.type); -#else - flb_debug("got an ext: %d)", (int)o->via.ext.type); -#endif - ret = do_avro(avro_value_set_bytes(val, (void *)o->via.bin.ptr, o->via.bin.size), "failed on ext"); - break; - - case MSGPACK_OBJECT_ARRAY: - { - - flb_debug("got a array:size:%u:\n", o->via.array.size); - if(o->via.array.size != 0) { - msgpack_object* p = o->via.array.ptr; - msgpack_object* const pend = o->via.array.ptr + o->via.array.size; - int i = 0; - for(; p < pend; ++p) { - avro_value_t element; - flb_debug("processing array\n"); - if ( - !do_avro(avro_value_append(val, &element, NULL), "Cannot append to array") || - !do_avro(avro_value_get_by_index(val, i++, &element, NULL), "Cannot get element")) { - goto msg2avro_end; - } - ret = flb_msgpack_to_avro(&element, p); - } - } - } - break; - - case MSGPACK_OBJECT_MAP: - flb_debug("got a map\n"); - if(o->via.map.size != 0) { - msgpack_object_kv* p = o->via.map.ptr; - msgpack_object_kv* const pend = o->via.map.ptr + o->via.map.size; - for(; p < pend; ++p) { - avro_value_t element; - if (p->key.type != MSGPACK_OBJECT_STR) { - flb_debug("the key of in a map must be string.\n"); - continue; - } - flb_sds_t key = flb_sds_create_len(p->key.via.str.ptr, p->key.via.str.size); - flb_debug("got key:%s:\n", key); - - if (val == NULL) { - flb_debug("got a null val\n"); - flb_sds_destroy(key); - continue; - } - // this does not always return 0 for succcess - if (avro_value_add(val, key, &element, NULL, NULL) != 0) { - flb_debug("avro_value_add:key:%s:avro error:%s:\n", key, avro_strerror()); - } - flb_debug("added\n"); - - flb_debug("calling avro_value_get_by_name\n"); - if (!do_avro(avro_value_get_by_name(val, key, &element, NULL), "Cannot get field")) { - flb_sds_destroy(key); - goto msg2avro_end; - } - flb_debug("called avro_value_get_by_index\n"); - - ret = flb_msgpack_to_avro(&element, &p->val); - - flb_sds_destroy(key); - } - } - break; - - default: - // FIXME -#if defined(PRIu64) - flb_warn(" #<UNKNOWN %i %" PRIu64 ">\n", o->type, o->via.u64); -#else - if (o.via.u64 > ULONG_MAX) - flb_warn(" #<UNKNOWN %i over 4294967295>", o->type); - else - flb_warn(" #<UNKNOWN %i %lu>", o.type, (unsigned long)o->via.u64); -#endif - // noop - break; - } - -msg2avro_end: - return ret; - -} - -/** - * convert msgpack to an avro object. - * it will fill the avro value with whatever comes in from the msgpack - * instantiate the avro value properly according to avro-c style - * - avro_schema_from_json_literal - * - avro_generic_class_from_schema - * - avro_generic_value_new - * - * or use flb_avro_init for the initialization - * - * refer to avro docs - * http://avro.apache.org/docs/current/api/c/index.html#_avro_values - * - * @param val An initialized avro value, an instiatied instance of the class to be unpacked. - * @param data The msgpack_unpacked data. - * @return success FLB_TRUE on success - */ -int flb_msgpack_to_avro(avro_value_t *val, msgpack_object *o) -{ - int ret = -1; - - if (val == NULL || o == NULL) { - flb_error("flb_msgpack_to_avro called with NULL\n"); - return ret; - } - - ret = msgpack2avro(val, o); - - return ret; -} - -bool flb_msgpack_raw_to_avro_sds(const void *in_buf, size_t in_size, struct flb_avro_fields *ctx, char *out_buff, size_t *out_size) -{ - msgpack_unpacked result; - msgpack_object *root; - - avro_writer_t awriter; - flb_debug("in flb_msgpack_raw_to_avro_sds\n"); - flb_debug("schemaID:%s:\n", ctx->schema_id); - flb_debug("schema string:%s:\n", ctx->schema_str); - - size_t schema_json_len = flb_sds_len(ctx->schema_str); - - avro_value_t aobject; - - assert(in_buf != NULL); - - avro_value_iface_t *aclass = NULL; - avro_schema_t aschema; - - aclass = flb_avro_init(&aobject, (char *)ctx->schema_str, schema_json_len, &aschema); - - if (!aclass) { - flb_error("Failed init avro:%s:n", avro_strerror()); - return false; - } - - msgpack_unpacked_init(&result); - if (msgpack_unpack_next(&result, in_buf, in_size, NULL) != MSGPACK_UNPACK_SUCCESS) { - flb_error("msgpack_unpack problem\n"); - avro_value_decref(&aobject); - avro_value_iface_decref(aclass); - avro_schema_decref(aschema); - return false; - } - - root = &result.data; - - // create the avro object - // then serialize it into a buffer for the downstream - flb_debug("calling flb_msgpack_to_avro\n"); - - if (flb_msgpack_to_avro(&aobject, root) != FLB_TRUE) { - flb_errno(); - flb_error("Failed msgpack to avro\n"); - msgpack_unpacked_destroy(&result); - avro_value_decref(&aobject); - avro_value_iface_decref(aclass); - avro_schema_decref(aschema); - return false; - } - - flb_debug("before avro_writer_memory\n"); - awriter = avro_writer_memory(out_buff, *out_size); - if (awriter == NULL) { - flb_error("Unable to init avro writer\n"); - msgpack_unpacked_destroy(&result); - avro_value_decref(&aobject); - avro_value_iface_decref(aclass); - avro_schema_decref(aschema); - return false; - } - - // write the magic byte stuff - // write one bye of \0 - // this is followed by - // 16 bytes of the schemaid where the schemaid is in hex - // in this implementation the schemaid is the md5hash of the avro schema - int rval; - rval = avro_write(awriter, "\0", 1); - if (rval != 0) { - flb_error("Unable to write magic byte\n"); - avro_writer_free(awriter); - avro_value_decref(&aobject); - avro_value_iface_decref(aclass); - avro_schema_decref(aschema); - msgpack_unpacked_destroy(&result); - return false; - } - - // write the schemaid - // its md5hash of the avro schema - // it looks like this c4b52aaf22429c7f9eb8c30270bc1795 - const char *pos = ctx->schema_id; - unsigned char val[16]; - size_t count; - for (count = 0; count < sizeof val/sizeof *val; count++) { - sscanf(pos, "%2hhx", &val[count]); - pos += 2; - } - - // write it into a buffer which can be passed to librdkafka - rval = avro_write(awriter, val, 16); - if (rval != 0) { - flb_error("Unable to write schemaid\n"); - avro_writer_free(awriter); - avro_value_decref(&aobject); - avro_value_iface_decref(aclass); - avro_schema_decref(aschema); - msgpack_unpacked_destroy(&result); - return false; - } - - if (avro_value_write(awriter, &aobject)) { - flb_error("Unable to write avro value to memory buffer\nMessage: %s\n", avro_strerror()); - avro_writer_free(awriter); - avro_value_decref(&aobject); - avro_value_iface_decref(aclass); - avro_schema_decref(aschema); - msgpack_unpacked_destroy(&result); - return false; - } - - // null terminate it - avro_write(awriter, "\0", 1); - - flb_debug("before avro_writer_flush\n"); - - avro_writer_flush(awriter); - - int64_t bytes_written = avro_writer_tell(awriter); - - // by here the entire object should be fully serialized into the sds buffer - avro_writer_free(awriter); - avro_value_decref(&aobject); - avro_value_iface_decref(aclass); - avro_schema_decref(aschema); - msgpack_unpacked_destroy(&result); - - flb_debug("after memory free:bytes written:%zu:\n", bytes_written); - *out_size = bytes_written; - - return true; - -} diff --git a/fluent-bit/src/flb_base64.c b/fluent-bit/src/flb_base64.c deleted file mode 100644 index 2bf442fba..000000000 --- a/fluent-bit/src/flb_base64.c +++ /dev/null @@ -1,239 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2019-2021 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* This code is based on base64.c from the mbedtls-2.25.0 Library distribution, - * as originally written by Paul Bakker, et al., and forked by the Fluent Bit - * project to provide performant base64 encoding and decoding routines. - * The 2.25.0 implementation is included rather than 2.26.0+ implementation due - * to performance degradation introduced in 2.26.0. - * - * Method and variable names are changed by the Fluent Bit authors to maintain - * consistency with the Fluent Bit project. - * The self test section of the code was removed by the Fluent Bit authors. - * Other minor changes are made by the Fluent Bit authors. - * - * The original source file base64.c is copyright and licensed as follows; - * - * RFC 1521 base64 encoding/decoding - * - * Copyright The Mbed TLS Contributors - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <fluent-bit/flb_base64.h> - -#include <stdint.h> - -static const unsigned char base64_enc_map[64] = -{ - 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', - 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', - 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', - 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', - 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', - 'y', 'z', '0', '1', '2', '3', '4', '5', '6', '7', - '8', '9', '+', '/' -}; - -static const unsigned char base64_dec_map[128] = -{ - 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, - 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, - 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, - 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, - 127, 127, 127, 62, 127, 127, 127, 63, 52, 53, - 54, 55, 56, 57, 58, 59, 60, 61, 127, 127, - 127, 64, 127, 127, 127, 0, 1, 2, 3, 4, - 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, - 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, - 25, 127, 127, 127, 127, 127, 127, 26, 27, 28, - 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, - 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, - 49, 50, 51, 127, 127, 127, 127, 127 -}; - -#define BASE64_SIZE_T_MAX ( (size_t) -1 ) /* SIZE_T_MAX is not standard */ - -/* - * Encode a buffer into base64 format - */ -int flb_base64_encode( unsigned char *dst, size_t dlen, size_t *olen, - const unsigned char *src, size_t slen ) -{ - size_t i, n; - int C1, C2, C3; - unsigned char *p; - - if( slen == 0 ) - { - *olen = 0; - return( 0 ); - } - - n = slen / 3 + ( slen % 3 != 0 ); - - if( n > ( BASE64_SIZE_T_MAX - 1 ) / 4 ) - { - *olen = BASE64_SIZE_T_MAX; - return( FLB_BASE64_ERR_BUFFER_TOO_SMALL ); - } - - n *= 4; - - if( ( dlen < n + 1 ) || ( NULL == dst ) ) - { - *olen = n + 1; - return( FLB_BASE64_ERR_BUFFER_TOO_SMALL ); - } - - n = ( slen / 3 ) * 3; - - for( i = 0, p = dst; i < n; i += 3 ) - { - C1 = *src++; - C2 = *src++; - C3 = *src++; - - *p++ = base64_enc_map[(C1 >> 2) & 0x3F]; - *p++ = base64_enc_map[(((C1 & 3) << 4) + (C2 >> 4)) & 0x3F]; - *p++ = base64_enc_map[(((C2 & 15) << 2) + (C3 >> 6)) & 0x3F]; - *p++ = base64_enc_map[C3 & 0x3F]; - } - - if( i < slen ) - { - C1 = *src++; - C2 = ( ( i + 1 ) < slen ) ? *src++ : 0; - - *p++ = base64_enc_map[(C1 >> 2) & 0x3F]; - *p++ = base64_enc_map[(((C1 & 3) << 4) + (C2 >> 4)) & 0x3F]; - - if( ( i + 1 ) < slen ) - *p++ = base64_enc_map[((C2 & 15) << 2) & 0x3F]; - else *p++ = '='; - - *p++ = '='; - } - - *olen = p - dst; - *p = 0; - - return( 0 ); -} - -/* - * Decode a base64-formatted buffer - */ -int flb_base64_decode( unsigned char *dst, size_t dlen, size_t *olen, - const unsigned char *src, size_t slen ) -{ - size_t i, n; - uint32_t j, x; - unsigned char *p; - - /* First pass: check for validity and get output length */ - for( i = n = j = 0; i < slen; i++ ) - { - /* Skip spaces before checking for EOL */ - x = 0; - while( i < slen && src[i] == ' ' ) - { - ++i; - ++x; - } - - /* Spaces at end of buffer are OK */ - if( i == slen ) - break; - - if( ( slen - i ) >= 2 && - src[i] == '\r' && src[i + 1] == '\n' ) - continue; - - if( src[i] == '\n' ) - continue; - - /* Space inside a line is an error */ - if( x != 0 ) - return( FLB_BASE64_ERR_INVALID_CHARACTER ); - - if( src[i] == '=' && ++j > 2 ) - return( FLB_BASE64_ERR_INVALID_CHARACTER ); - - if( src[i] > 127 || base64_dec_map[src[i]] == 127 ) - return( FLB_BASE64_ERR_INVALID_CHARACTER ); - - if( base64_dec_map[src[i]] < 64 && j != 0 ) - return( FLB_BASE64_ERR_INVALID_CHARACTER ); - - n++; - } - - if( n == 0 ) - { - *olen = 0; - return( 0 ); - } - - /* The following expression is to calculate the following formula without - * risk of integer overflow in n: - * n = ( ( n * 6 ) + 7 ) >> 3; - */ - n = ( 6 * ( n >> 3 ) ) + ( ( 6 * ( n & 0x7 ) + 7 ) >> 3 ); - n -= j; - - if( dst == NULL || dlen < n ) - { - *olen = n; - return( FLB_BASE64_ERR_BUFFER_TOO_SMALL ); - } - - for( j = 3, n = x = 0, p = dst; i > 0; i--, src++ ) - { - if( *src == '\r' || *src == '\n' || *src == ' ' ) - continue; - - j -= ( base64_dec_map[*src] == 64 ); - x = ( x << 6 ) | ( base64_dec_map[*src] & 0x3F ); - - if( ++n == 4 ) - { - n = 0; - if( j > 0 ) *p++ = (unsigned char)( x >> 16 ); - if( j > 1 ) *p++ = (unsigned char)( x >> 8 ); - if( j > 2 ) *p++ = (unsigned char)( x ); - } - } - - *olen = p - dst; - - return( 0 ); -} diff --git a/fluent-bit/src/flb_callback.c b/fluent-bit/src/flb_callback.c deleted file mode 100644 index 760604ce7..000000000 --- a/fluent-bit/src/flb_callback.c +++ /dev/null @@ -1,134 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <fluent-bit/flb_info.h> -#include <fluent-bit/flb_mem.h> -#include <fluent-bit/flb_log.h> -#include <fluent-bit/flb_callback.h> - -struct flb_callback *flb_callback_create(char *name) -{ - struct flb_callback *ctx; - - /* Create context */ - ctx = flb_malloc(sizeof(struct flb_callback)); - if (!ctx) { - flb_errno(); - return NULL; - } - - ctx->ht = flb_hash_table_create(FLB_HASH_TABLE_EVICT_NONE, 16, 0); - if (!ctx->ht) { - flb_error("[callback] error allocating hash table"); - flb_free(ctx); - return NULL; - } - mk_list_init(&ctx->entries); - - return ctx; -} - -int flb_callback_set(struct flb_callback *ctx, char *name, - void (*cb)(char *, void *, void *)) -{ - int ret; - int len; - struct flb_callback_entry *entry; - - entry = flb_malloc(sizeof(struct flb_callback_entry)); - if (!entry) { - flb_errno(); - return -1; - } - entry->name = flb_sds_create(name); - if (!entry->name) { - flb_free(entry); - return -1; - } - entry->cb = cb; - - len = strlen(name); - ret = flb_hash_table_add(ctx->ht, name, len, - (char *) &entry, sizeof(struct flb_callback_entry *)); - if (ret == -1) { - flb_sds_destroy(entry->name); - flb_free(entry); - return -1; - } - mk_list_add(&entry->_head, &ctx->entries); - - return ret; -} - -int flb_callback_exists(struct flb_callback *ctx, char *name) -{ - int ret; - int len; - size_t out_size; - void *cb_addr; - - len = strlen(name); - ret = flb_hash_table_get(ctx->ht, name, len, &cb_addr, &out_size); - if (ret == -1) { - return FLB_FALSE; - } - - return FLB_TRUE; -} - -int flb_callback_do(struct flb_callback *ctx, char *name, void *p1, void *p2) -{ - int ret; - int len; - size_t out_size; - void *cb_addr; - struct flb_callback_entry *entry; - - if (!ctx) { - return -1; - } - - len = strlen(name); - ret = flb_hash_table_get(ctx->ht, name, len, &cb_addr, &out_size); - if (ret == -1) { - return -1; - } - - memcpy(&entry, cb_addr, sizeof(struct flb_callback_entry *)); - entry->cb(entry->name, p1, p2); - return 0; -} - -void flb_callback_destroy(struct flb_callback *ctx) -{ - struct mk_list *tmp; - struct mk_list *head; - struct flb_callback_entry *entry; - - flb_hash_table_destroy(ctx->ht); - - mk_list_foreach_safe(head, tmp, &ctx->entries) { - entry = mk_list_entry(head, struct flb_callback_entry, _head); - mk_list_del(&entry->_head); - flb_sds_destroy(entry->name); - flb_free(entry); - } - - flb_free(ctx); -} diff --git a/fluent-bit/src/flb_chunk_trace.c b/fluent-bit/src/flb_chunk_trace.c deleted file mode 100644 index adacb73f2..000000000 --- a/fluent-bit/src/flb_chunk_trace.c +++ /dev/null @@ -1,692 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <fcntl.h> - -#include <msgpack.h> -#include <chunkio/chunkio.h> - -#include <fluent-bit/flb_info.h> -#include <fluent-bit/flb_input_chunk.h> -#include <fluent-bit/flb_input.h> -#include <fluent-bit/flb_output.h> -#include <fluent-bit/flb_chunk_trace.h> -#include <fluent-bit/flb_pack.h> -#include <fluent-bit/flb_base64.h> -#include <fluent-bit/flb_storage.h> -#include <fluent-bit/flb_router.h> -#include <fluent-bit/flb_kv.h> - - -/* Register external function to emit records, check 'plugins/in_emitter' */ -int in_emitter_add_record(const char *tag, int tag_len, - const char *buf_data, size_t buf_size, - struct flb_input_instance *in); - -/****************************************************************************/ -/* To avoid double frees when enabling and disabling tracing as well */ -/* as avoiding race conditions when stopping fluent-bit while someone is */ -/* toggling tracing via the HTTP API this set of APIS with a mutex lock */ -/* is used: */ -/* * flb_chunk_trace_to_be_destroyed - query to see if the trace context */ -/* is slated to be freed */ -/* * flb_chunk_trace_set_destroy - set the trace context to be destroyed */ -/* once all chunks are freed (executed in flb_chunk_trace_destroy). */ -/* * flb_chunk_trace_has_chunks - see if there are still chunks using */ -/* using the tracing context */ -/* * flb_chunk_trace_add - increment the traces chunk count */ -/* * flb_chunk_trace_sub - decrement the traces chunk count */ -/****************************************************************************/ -static inline int flb_chunk_trace_to_be_destroyed(struct flb_chunk_trace_context *ctxt) -{ - int ret = FLB_FALSE; - - ret = (ctxt->to_destroy == 1 ? FLB_TRUE : FLB_FALSE); - return ret; -} - -static inline int flb_chunk_trace_has_chunks(struct flb_chunk_trace_context *ctxt) -{ - int ret = FLB_FALSE; - - ret = ((ctxt->chunks > 0) ? FLB_TRUE : FLB_FALSE); - return ret; -} - -static inline void flb_chunk_trace_add(struct flb_chunk_trace_context *ctxt) -{ - ctxt->chunks++; -} - -static inline void flb_chunk_trace_sub(struct flb_chunk_trace_context *ctxt) -{ - ctxt->chunks--; -} - -static inline void flb_chunk_trace_set_destroy(struct flb_chunk_trace_context *ctxt) -{ - ctxt->to_destroy = 1; -} - -static struct flb_output_instance *find_calyptia_output_instance(struct flb_config *config) -{ - struct mk_list *head = NULL; - struct flb_output_instance *output = NULL; - - mk_list_foreach(head, &config->outputs) { - output = mk_list_entry(head, struct flb_output_instance, _head); - if (strcmp(output->p->name, "calyptia") == 0) { - return output; - } - } - return NULL; -} - -static void trace_chunk_context_destroy(struct flb_chunk_trace_context *ctxt) -{ - int i; - - - if (flb_chunk_trace_has_chunks(ctxt) == FLB_TRUE) { - flb_chunk_trace_set_destroy(ctxt); - flb_input_pause_all(ctxt->flb->config); - return; - } - - /* pause all inputs, then destroy the input storage. */ - flb_input_pause_all(ctxt->flb->config); - /* waiting for all tasks to end is key to safely stopping and destroying */ - /* the fluent-bit pipeline. */ - for (i = 0; i < 5 && flb_task_running_count(ctxt->flb->config) > 0; i++) { - usleep(10 * 1000); - } - - flb_sds_destroy(ctxt->trace_prefix); - flb_stop(ctxt->flb); - flb_destroy(ctxt->flb); - flb_free(ctxt); -} - -void flb_chunk_trace_context_destroy(void *input) -{ - struct flb_input_instance *in = (struct flb_input_instance *)input; - pthread_mutex_lock(&in->chunk_trace_lock); - if (in->chunk_trace_ctxt != NULL) { - trace_chunk_context_destroy(in->chunk_trace_ctxt); - in->chunk_trace_ctxt = NULL; - } - pthread_mutex_unlock(&in->chunk_trace_lock); -} - -struct flb_chunk_trace_context *flb_chunk_trace_context_new(void *trace_input, - const char *output_name, - const char *trace_prefix, - void *data, struct mk_list *props) -{ - struct flb_input_instance *in = (struct flb_input_instance *)trace_input; - struct flb_config *config = in->config; - struct flb_input_instance *input = NULL; - struct flb_output_instance *output = NULL; - struct flb_output_instance *calyptia = NULL; - struct flb_chunk_trace_context *ctx = NULL; - struct mk_list *head = NULL; - struct flb_kv *prop = NULL; - int ret; - - if (config->enable_chunk_trace == FLB_FALSE) { - flb_warn("[chunk trace] enable chunk tracing via the configuration or " - " command line to be able to activate tracing."); - return NULL; - } - - pthread_mutex_lock(&in->chunk_trace_lock); - - if (in->chunk_trace_ctxt) { - trace_chunk_context_destroy(in->chunk_trace_ctxt); - } - - ctx = flb_calloc(1, sizeof(struct flb_chunk_trace_context)); - if (ctx == NULL) { - flb_errno(); - pthread_mutex_unlock(&in->chunk_trace_lock); - return NULL; - } - - ctx->flb = flb_create(); - if (ctx->flb == NULL) { - flb_errno(); - goto error_ctxt; - } - - flb_service_set(ctx->flb, "flush", "1", "grace", "1", NULL); - - input = (void *)flb_input_new(ctx->flb->config, "emitter", NULL, FLB_FALSE); - if (input == NULL) { - flb_error("could not load trace emitter"); - goto error_flb; - } - - ret = flb_input_set_property(input, "alias", "trace-emitter"); - if (ret != 0) { - flb_error("unable to set alias for trace emitter"); - goto error_input; - } - - ret = flb_input_set_property(input, "ring_buffer_size", "4096"); - if (ret != 0) { - flb_error("unable to set ring buffer size for trace emitter"); - goto error_input; - } - - output = flb_output_new(ctx->flb->config, output_name, data, 1); - if (output == NULL) { - flb_error("could not create trace output"); - goto error_input; - } - - /* special handling for the calyptia plugin so we can copy the API */ - /* key and other configuration properties. */ - if (strcmp(output_name, "calyptia") == 0) { - calyptia = find_calyptia_output_instance(config); - if (calyptia == NULL) { - flb_error("unable to find calyptia output instance"); - goto error_output; - } - mk_list_foreach(head, &calyptia->properties) { - prop = mk_list_entry(head, struct flb_kv, _head); - flb_output_set_property(output, prop->key, prop->val); - } - } - else if (props != NULL) { - mk_list_foreach(head, props) { - prop = mk_list_entry(head, struct flb_kv, _head); - flb_output_set_property(output, prop->key, prop->val); - } - } - - ret = flb_router_connect_direct(input, output); - if (ret != 0) { - flb_error("unable to route traces"); - goto error_output; - } - - ctx->output = (void *)output; - ctx->input = (void *)input; - ctx->trace_prefix = flb_sds_create(trace_prefix); - - flb_start_trace(ctx->flb); - - in->chunk_trace_ctxt = ctx; - pthread_mutex_unlock(&in->chunk_trace_lock); - return ctx; - -error_output: - flb_output_instance_destroy(output); -error_input: - if (ctx->cio) { - cio_destroy(ctx->cio); - } - flb_input_instance_destroy(input); -error_flb: - flb_destroy(ctx->flb); -error_ctxt: - flb_free(ctx); - pthread_mutex_unlock(&in->chunk_trace_lock); - return NULL; -} - -struct flb_chunk_trace *flb_chunk_trace_new(struct flb_input_chunk *chunk) -{ - struct flb_chunk_trace *trace = NULL; - struct flb_input_instance *f_ins = (struct flb_input_instance *)chunk->in; - - pthread_mutex_lock(&f_ins->chunk_trace_lock); - - if (flb_chunk_trace_to_be_destroyed(f_ins->chunk_trace_ctxt) == FLB_TRUE) { - pthread_mutex_unlock(&f_ins->chunk_trace_lock); - return NULL; - } - - trace = flb_calloc(1, sizeof(struct flb_chunk_trace)); - if (trace == NULL) { - flb_errno(); - pthread_mutex_unlock(&f_ins->chunk_trace_lock); - return NULL; - } - - trace->ctxt = f_ins->chunk_trace_ctxt; - flb_chunk_trace_add(trace->ctxt); - - trace->trace_id = flb_sds_create(""); - if (flb_sds_printf(&trace->trace_id, "%s%d", trace->ctxt->trace_prefix, - trace->ctxt->trace_count++) == NULL) { - pthread_mutex_unlock(&f_ins->chunk_trace_lock); - flb_sds_destroy(trace->trace_id); - flb_free(trace); - return NULL; - } - - trace->ic = chunk; - - pthread_mutex_unlock(&f_ins->chunk_trace_lock); - return trace; -} - -void flb_chunk_trace_destroy(struct flb_chunk_trace *trace) -{ - pthread_mutex_lock(&trace->ic->in->chunk_trace_lock); - flb_chunk_trace_sub(trace->ctxt); - - /* check to see if we need to free the trace context. */ - if (flb_chunk_trace_has_chunks(trace->ctxt) == FLB_FALSE && - flb_chunk_trace_to_be_destroyed(trace->ctxt) == FLB_TRUE) { - trace_chunk_context_destroy(trace->ctxt); - } - else if (flb_chunk_trace_has_chunks(trace->ctxt) == FLB_TRUE && - flb_chunk_trace_to_be_destroyed(trace->ctxt) == FLB_TRUE) { - } - pthread_mutex_unlock(&trace->ic->in->chunk_trace_lock); - - flb_sds_destroy(trace->trace_id); - flb_free(trace); -} - -int flb_chunk_trace_context_set_limit(void *input, int limit_type, int limit_arg) -{ - struct flb_input_instance *in = (struct flb_input_instance *)input; - struct flb_chunk_trace_context *ctxt = NULL; - struct flb_time tm; - - pthread_mutex_lock(&in->chunk_trace_lock); - - ctxt = in->chunk_trace_ctxt; - if (ctxt == NULL) { - pthread_mutex_unlock(&in->chunk_trace_lock); - return -1; - } - - switch(limit_type) { - case FLB_CHUNK_TRACE_LIMIT_TIME: - flb_time_get(&tm); - ctxt->limit.type = FLB_CHUNK_TRACE_LIMIT_TIME; - ctxt->limit.seconds_started = tm.tm.tv_sec; - ctxt->limit.seconds = limit_arg; - - pthread_mutex_unlock(&in->chunk_trace_lock); - return 0; - case FLB_CHUNK_TRACE_LIMIT_COUNT: - ctxt->limit.type = FLB_CHUNK_TRACE_LIMIT_COUNT; - ctxt->limit.count = limit_arg; - - pthread_mutex_unlock(&in->chunk_trace_lock); - return 0; - } - - pthread_mutex_unlock(&in->chunk_trace_lock); - return -1; -} - -int flb_chunk_trace_context_hit_limit(void *input) -{ - struct flb_input_instance *in = (struct flb_input_instance *)input; - struct flb_time tm; - struct flb_chunk_trace_context *ctxt = NULL; - - pthread_mutex_lock(&in->chunk_trace_lock); - - ctxt = in->chunk_trace_ctxt; - if (ctxt == NULL) { - pthread_mutex_unlock(&in->chunk_trace_lock); - return FLB_FALSE; - } - - switch(ctxt->limit.type) { - case FLB_CHUNK_TRACE_LIMIT_TIME: - flb_time_get(&tm); - if ((tm.tm.tv_sec - ctxt->limit.seconds_started) > ctxt->limit.seconds) { - pthread_mutex_unlock(&in->chunk_trace_lock); - return FLB_TRUE; - } - return FLB_FALSE; - case FLB_CHUNK_TRACE_LIMIT_COUNT: - if (ctxt->limit.count <= ctxt->trace_count) { - pthread_mutex_unlock(&in->chunk_trace_lock); - return FLB_TRUE; - } - pthread_mutex_unlock(&in->chunk_trace_lock); - return FLB_FALSE; - } - pthread_mutex_unlock(&in->chunk_trace_lock); - return FLB_FALSE; -} - -void flb_chunk_trace_do_input(struct flb_input_chunk *ic) -{ - pthread_mutex_lock(&ic->in->chunk_trace_lock); - if (ic->in->chunk_trace_ctxt == NULL) { - pthread_mutex_unlock(&ic->in->chunk_trace_lock); - return; - } - pthread_mutex_unlock(&ic->in->chunk_trace_lock); - - if (ic->trace == NULL) { - ic->trace = flb_chunk_trace_new(ic); - } - - if (ic->trace) { - flb_chunk_trace_input(ic->trace); - if (flb_chunk_trace_context_hit_limit(ic->in) == FLB_TRUE) { - flb_chunk_trace_context_destroy(ic->in); - } - } -} - -int flb_chunk_trace_input(struct flb_chunk_trace *trace) -{ - msgpack_packer mp_pck; - msgpack_sbuffer mp_sbuf; - msgpack_unpacked result; - msgpack_object *record = NULL; - char *buf = NULL; - size_t buf_size; - struct flb_time tm; - struct flb_time tm_end; - struct flb_input_instance *input = (struct flb_input_instance *)trace->ic->in; - int rc = -1; - size_t off = 0; - flb_sds_t tag = flb_sds_create("trace"); - int records = 0; - - - /* initiailize start time */ - flb_time_get(&tm); - flb_time_get(&tm_end); - - msgpack_sbuffer_init(&mp_sbuf); - msgpack_packer_init(&mp_pck, &mp_sbuf, msgpack_sbuffer_write); - msgpack_unpacked_init(&result); - - cio_chunk_get_content(trace->ic->chunk, &buf, &buf_size); - - msgpack_pack_array(&mp_pck, 2); - flb_pack_time_now(&mp_pck); - if (input->alias != NULL) { - msgpack_pack_map(&mp_pck, 7); - } - else { - msgpack_pack_map(&mp_pck, 6); - } - - msgpack_pack_str_with_body(&mp_pck, "type", 4); - msgpack_pack_int(&mp_pck, FLB_CHUNK_TRACE_TYPE_INPUT); - - msgpack_pack_str_with_body(&mp_pck, "trace_id", strlen("trace_id")); - msgpack_pack_str_with_body(&mp_pck, trace->trace_id, strlen(trace->trace_id)); - - msgpack_pack_str_with_body(&mp_pck, "plugin_instance", strlen("plugin_instance")); - msgpack_pack_str_with_body(&mp_pck, input->name, strlen(input->name)); - - if (input->alias != NULL) { - msgpack_pack_str_with_body(&mp_pck, "plugin_alias", strlen("plugin_alias")); - msgpack_pack_str_with_body(&mp_pck, input->alias, strlen(input->alias)); - } - - msgpack_pack_str_with_body(&mp_pck, "records", strlen("records")); - - if (buf_size > 0) { - do { - rc = msgpack_unpack_next(&result, buf, buf_size, &off); - if (rc != MSGPACK_UNPACK_SUCCESS) { - flb_error("unable to unpack record"); - goto sbuffer_error; - } - records++; - } while (rc == MSGPACK_UNPACK_SUCCESS && off < buf_size); - - msgpack_pack_array(&mp_pck, records); - - off = 0; - do { - rc = msgpack_unpack_next(&result, buf, buf_size, &off); - if (rc != MSGPACK_UNPACK_SUCCESS) { - flb_error("unable to unpack record"); - goto sbuffer_error; - } - flb_time_pop_from_msgpack(&tm, &result, &record); - - msgpack_pack_map(&mp_pck, 2); - msgpack_pack_str_with_body(&mp_pck, "timestamp", strlen("timestamp")); - flb_time_append_to_msgpack(&tm, &mp_pck, FLB_TIME_ETFMT_INT); - msgpack_pack_str_with_body(&mp_pck, "record", strlen("record")); - msgpack_pack_object(&mp_pck, *record); - - } while (rc == MSGPACK_UNPACK_SUCCESS && off < buf_size); - } - - msgpack_pack_str_with_body(&mp_pck, "start_time", strlen("start_time")); - flb_time_append_to_msgpack(&tm, &mp_pck, FLB_TIME_ETFMT_INT); - msgpack_pack_str_with_body(&mp_pck, "end_time", strlen("end_time")); - flb_time_append_to_msgpack(&tm_end, &mp_pck, FLB_TIME_ETFMT_INT); - in_emitter_add_record(tag, flb_sds_len(tag), mp_sbuf.data, mp_sbuf.size, - trace->ctxt->input); -sbuffer_error: - flb_sds_destroy(tag); - msgpack_unpacked_destroy(&result); - msgpack_sbuffer_destroy(&mp_sbuf); - return rc; -} - -int flb_chunk_trace_pre_output(struct flb_chunk_trace *trace) -{ - msgpack_packer mp_pck; - msgpack_sbuffer mp_sbuf; - msgpack_unpacked result; - msgpack_object *record = NULL; - char *buf = NULL; - size_t buf_size; - struct flb_time tm; - struct flb_time tm_end; - struct flb_input_instance *input = (struct flb_input_instance *)trace->ic->in; - int rc = -1; - size_t off = 0; - flb_sds_t tag = flb_sds_create("trace"); - int records = 0; - - - /* initiailize start time */ - flb_time_get(&tm); - flb_time_get(&tm_end); - - msgpack_sbuffer_init(&mp_sbuf); - msgpack_packer_init(&mp_pck, &mp_sbuf, msgpack_sbuffer_write); - msgpack_unpacked_init(&result); - - cio_chunk_get_content(trace->ic->chunk, &buf, &buf_size); - - msgpack_pack_array(&mp_pck, 2); - flb_pack_time_now(&mp_pck); - if (input->alias != NULL) { - msgpack_pack_map(&mp_pck, 7); - } - else { - msgpack_pack_map(&mp_pck, 6); - } - - msgpack_pack_str_with_body(&mp_pck, "type", 4); - msgpack_pack_int(&mp_pck, FLB_CHUNK_TRACE_TYPE_PRE_OUTPUT); - - msgpack_pack_str_with_body(&mp_pck, "trace_id", strlen("trace_id")); - msgpack_pack_str_with_body(&mp_pck, trace->trace_id, strlen(trace->trace_id)); - - msgpack_pack_str_with_body(&mp_pck, "plugin_instance", strlen("plugin_instance")); - msgpack_pack_str_with_body(&mp_pck, input->name, strlen(input->name)); - - if (input->alias != NULL) { - msgpack_pack_str_with_body(&mp_pck, "plugin_alias", strlen("plugin_alias")); - msgpack_pack_str_with_body(&mp_pck, input->alias, strlen(input->alias)); - } - - msgpack_pack_str_with_body(&mp_pck, "records", strlen("records")); - - if (buf_size > 0) { - do { - rc = msgpack_unpack_next(&result, buf, buf_size, &off); - if (rc != MSGPACK_UNPACK_SUCCESS) { - flb_error("unable to unpack record"); - goto sbuffer_error; - } - records++; - } while (rc == MSGPACK_UNPACK_SUCCESS && off < buf_size); - - msgpack_pack_array(&mp_pck, records); - off = 0; - do { - rc = msgpack_unpack_next(&result, buf, buf_size, &off); - if (rc != MSGPACK_UNPACK_SUCCESS) { - flb_error("unable to unpack record"); - goto sbuffer_error; - } - flb_time_pop_from_msgpack(&tm, &result, &record); - - msgpack_pack_map(&mp_pck, 2); - msgpack_pack_str_with_body(&mp_pck, "timestamp", strlen("timestamp")); - flb_time_append_to_msgpack(&tm, &mp_pck, FLB_TIME_ETFMT_INT); - msgpack_pack_str_with_body(&mp_pck, "record", strlen("record")); - msgpack_pack_object(&mp_pck, *record); - - } while (rc == MSGPACK_UNPACK_SUCCESS && off < buf_size); - } - - msgpack_pack_str_with_body(&mp_pck, "start_time", strlen("start_time")); - flb_time_append_to_msgpack(&tm, &mp_pck, FLB_TIME_ETFMT_INT); - msgpack_pack_str_with_body(&mp_pck, "end_time", strlen("end_time")); - flb_time_append_to_msgpack(&tm_end, &mp_pck, FLB_TIME_ETFMT_INT); - in_emitter_add_record(tag, flb_sds_len(tag), mp_sbuf.data, mp_sbuf.size, - trace->ctxt->input); -sbuffer_error: - flb_sds_destroy(tag); - msgpack_unpacked_destroy(&result); - msgpack_sbuffer_destroy(&mp_sbuf); - return rc; -} - -int flb_chunk_trace_filter(struct flb_chunk_trace *tracer, void *pfilter, struct flb_time *tm_start, struct flb_time *tm_end, char *buf, size_t buf_size) -{ - msgpack_packer mp_pck; - msgpack_sbuffer mp_sbuf; - msgpack_unpacked result; - msgpack_object *record = NULL; - int rc = -1; - struct flb_filter_instance *filter = (struct flb_filter_instance *)pfilter; - flb_sds_t tag = flb_sds_create("trace"); - struct flb_time tm; - size_t off = 0; - int records = 0; - - - if (tracer == NULL) { - goto tracer_error; - } - - msgpack_sbuffer_init(&mp_sbuf); - msgpack_packer_init(&mp_pck, &mp_sbuf, msgpack_sbuffer_write); - - msgpack_pack_array(&mp_pck, 2); - flb_pack_time_now(&mp_pck); - if (filter->alias == NULL) { - msgpack_pack_map(&mp_pck, 6); - } - else { - msgpack_pack_map(&mp_pck, 7); - } - - msgpack_pack_str_with_body(&mp_pck, "type", strlen("type")); - rc = msgpack_pack_int(&mp_pck, FLB_CHUNK_TRACE_TYPE_FILTER); - if (rc == -1) { - goto sbuffer_error; - } - - msgpack_pack_str_with_body(&mp_pck, "start_time", strlen("start_time")); - //msgpack_pack_double(&mp_pck, flb_time_to_double(tm_start)); - flb_time_append_to_msgpack(tm_start, &mp_pck, FLB_TIME_ETFMT_INT); - msgpack_pack_str_with_body(&mp_pck, "end_time", strlen("end_time")); - //msgpack_pack_double(&mp_pck, flb_time_to_double(tm_end)); - flb_time_append_to_msgpack(tm_end, &mp_pck, FLB_TIME_ETFMT_INT); - - msgpack_pack_str_with_body(&mp_pck, "trace_id", strlen("trace_id")); - msgpack_pack_str_with_body(&mp_pck, tracer->trace_id, strlen(tracer->trace_id)); - - - msgpack_pack_str_with_body(&mp_pck, "plugin_instance", strlen("plugin_instance")); - rc = msgpack_pack_str_with_body(&mp_pck, filter->name, strlen(filter->name)); - if (rc == -1) { - goto sbuffer_error; - } - - if (filter->alias != NULL) { - msgpack_pack_str_with_body(&mp_pck, "plugin_alias", strlen("plugin_alias")); - msgpack_pack_str_with_body(&mp_pck, filter->alias, strlen(filter->alias)); - } - - msgpack_pack_str_with_body(&mp_pck, "records", strlen("records")); - - msgpack_unpacked_init(&result); - - if (buf_size > 0) { - do { - rc = msgpack_unpack_next(&result, buf, buf_size, &off); - if (rc != MSGPACK_UNPACK_SUCCESS) { - flb_error("unable to unpack record"); - goto unpack_error; - } - records++; - } while (rc == MSGPACK_UNPACK_SUCCESS && off < buf_size); - - msgpack_pack_array(&mp_pck, records); - off = 0; - do { - rc = msgpack_unpack_next(&result, buf, buf_size, &off); - if (rc != MSGPACK_UNPACK_SUCCESS) { - flb_error("unable to unpack record"); - goto unpack_error; - } - flb_time_pop_from_msgpack(&tm, &result, &record); - - msgpack_pack_map(&mp_pck, 2); - msgpack_pack_str_with_body(&mp_pck, "timestamp", strlen("timestamp")); - flb_time_append_to_msgpack(&tm, &mp_pck, FLB_TIME_ETFMT_INT); - msgpack_pack_str_with_body(&mp_pck, "record", strlen("record")); - msgpack_pack_object(&mp_pck, *record); - - } while (rc == MSGPACK_UNPACK_SUCCESS && off < buf_size); - } - - in_emitter_add_record(tag, flb_sds_len(tag), mp_sbuf.data, mp_sbuf.size, - tracer->ctxt->input); - - rc = 0; - -unpack_error: - msgpack_unpacked_destroy(&result); -sbuffer_error: - msgpack_sbuffer_destroy(&mp_sbuf); -tracer_error: - flb_sds_destroy(tag); - return rc; -} diff --git a/fluent-bit/src/flb_compression.c b/fluent-bit/src/flb_compression.c deleted file mode 100644 index 86c94c90b..000000000 --- a/fluent-bit/src/flb_compression.c +++ /dev/null @@ -1,221 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <fluent-bit/flb_info.h> -#include <fluent-bit/flb_mem.h> -#include <fluent-bit/flb_log.h> -#include <fluent-bit/flb_gzip.h> -#include <fluent-bit/flb_compression.h> - -static size_t flb_decompression_context_get_read_buffer_offset( - struct flb_decompression_context *context) -{ - uintptr_t input_buffer_offset; - - if (context == NULL) { - return 0; - } - - input_buffer_offset = (uintptr_t) context->read_buffer; - input_buffer_offset -= (uintptr_t) context->input_buffer; - - return input_buffer_offset; -} - -static void flb_decompression_context_adjust_buffer( - struct flb_decompression_context *context) -{ - uintptr_t input_buffer_offset; - - if (context != NULL) { - input_buffer_offset = \ - flb_decompression_context_get_read_buffer_offset(context); - - if (input_buffer_offset >= (context->input_buffer_size / 2)) { - memmove(context->input_buffer, - context->read_buffer, - context->input_buffer_length); - - context->read_buffer = context->input_buffer; - } - } -} - -uint8_t *flb_decompression_context_get_append_buffer( - struct flb_decompression_context *context) -{ - if (context != NULL) { - flb_decompression_context_adjust_buffer(context); - - return &context->read_buffer[context->input_buffer_length]; - } - - return NULL; -} - -size_t flb_decompression_context_get_available_space( - struct flb_decompression_context *context) -{ - uintptr_t available_buffer_space; - uintptr_t input_buffer_offset; - - if (context == NULL) { - return 0; - } - - flb_decompression_context_adjust_buffer(context); - - input_buffer_offset = \ - flb_decompression_context_get_read_buffer_offset(context); - - available_buffer_space = context->input_buffer_size; - available_buffer_space -= input_buffer_offset; - available_buffer_space -= context->input_buffer_length; - - return available_buffer_space; -} - -int flb_decompression_context_resize_buffer( - struct flb_decompression_context *context, size_t new_size) -{ - void *new_buffer_address; - - if (new_size > context->input_buffer_length) { - new_buffer_address = flb_realloc(context->input_buffer, - new_size); - - if (new_buffer_address == NULL) { - return FLB_DECOMPRESSOR_FAILURE; - } - - if (new_buffer_address != context->input_buffer) { - context->read_buffer = (uint8_t *) \ - (((uintptr_t) context->read_buffer - - (uintptr_t) context->input_buffer) + - (uintptr_t) new_buffer_address); - context->input_buffer = (uint8_t *) new_buffer_address; - context->input_buffer_size = new_size; - } - } - else if (new_size < context->input_buffer_length) { - return FLB_DECOMPRESSOR_FAILURE; - } - - return FLB_DECOMPRESSOR_SUCCESS; -} - - -void flb_decompression_context_destroy(struct flb_decompression_context *context) -{ - if (context != NULL) { - if (context->input_buffer != NULL) { - flb_free(context->input_buffer); - - context->input_buffer = NULL; - } - - if (context->inner_context != NULL) { - flb_gzip_decompression_context_destroy(context->inner_context); - - context->inner_context = NULL; - } - - context->read_buffer = NULL; - - flb_free(context); - } -} - -struct flb_decompression_context *flb_decompression_context_create(int algorithm, - size_t input_buffer_size) -{ - struct flb_decompression_context *context; - - if (input_buffer_size == 0) { - input_buffer_size = FLB_DECOMPRESSION_BUFFER_SIZE; - } - - context = - flb_calloc(1, sizeof(struct flb_decompression_context)); - - if (context == NULL) { - flb_errno(); - - flb_error("error allocating decompression context"); - - return NULL; - } - - context->input_buffer = - flb_calloc(input_buffer_size, sizeof(uint8_t)); - - if (context->input_buffer == NULL) { - flb_errno(); - - flb_error("error allocating decompression buffer"); - - flb_decompression_context_destroy(context); - - return NULL; - } - - if (algorithm == FLB_COMPRESSION_ALGORITHM_GZIP) { - context->inner_context = flb_gzip_decompression_context_create(); - } - else { - flb_error("invalid compression algorithm : %d", algorithm); - - flb_decompression_context_destroy(context); - - return NULL; - } - - if (context->inner_context == NULL) { - flb_errno(); - - flb_error("error allocating internal decompression context"); - - flb_decompression_context_destroy(context); - - return NULL; - } - - context->input_buffer_size = input_buffer_size; - context->read_buffer = context->read_buffer; - context->algorithm = algorithm; - context->state = FLB_DECOMPRESSOR_STATE_EXPECTING_HEADER; - - return context; -} - -int flb_decompress(struct flb_decompression_context *context, - void *output_buffer, - size_t *output_length) -{ - if (context != NULL) { - if (context->algorithm == FLB_COMPRESSION_ALGORITHM_GZIP) { - return flb_gzip_decompressor_dispatch(context, - output_buffer, - output_length); - - } - } - - return FLB_DECOMPRESSOR_FAILURE; -} diff --git a/fluent-bit/src/flb_config.c b/fluent-bit/src/flb_config.c deleted file mode 100644 index 882a93c7c..000000000 --- a/fluent-bit/src/flb_config.c +++ /dev/null @@ -1,942 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <stdio.h> -#include <stdlib.h> -#include <signal.h> -#include <stddef.h> - -#include <monkey/mk_core.h> -#include <fluent-bit/flb_info.h> -#include <fluent-bit/flb_mem.h> -#include <fluent-bit/flb_str.h> -#include <fluent-bit/flb_kv.h> -#include <fluent-bit/flb_env.h> -#include <fluent-bit/flb_meta.h> -#include <fluent-bit/flb_macros.h> -#include <fluent-bit/flb_config.h> -#include <fluent-bit/flb_parser.h> -#include <fluent-bit/flb_plugin.h> -#include <fluent-bit/flb_plugins.h> -#include <fluent-bit/flb_slist.h> -#include <fluent-bit/flb_kernel.h> -#include <fluent-bit/flb_worker.h> -#include <fluent-bit/flb_scheduler.h> -#include <fluent-bit/flb_http_server.h> -#include <fluent-bit/flb_plugin.h> -#include <fluent-bit/flb_utils.h> -#include <fluent-bit/flb_config_format.h> -#include <fluent-bit/multiline/flb_ml.h> -#include <fluent-bit/flb_bucket_queue.h> - -const char *FLB_CONF_ENV_LOGLEVEL = "FLB_LOG_LEVEL"; - -int flb_regex_init(); - -struct flb_service_config service_configs[] = { - {FLB_CONF_STR_FLUSH, - FLB_CONF_TYPE_DOUBLE, - offsetof(struct flb_config, flush)}, - - {FLB_CONF_STR_GRACE, - FLB_CONF_TYPE_INT, - offsetof(struct flb_config, grace)}, - - {FLB_CONF_STR_CONV_NAN, - FLB_CONF_TYPE_BOOL, - offsetof(struct flb_config, convert_nan_to_null)}, - - {FLB_CONF_STR_DAEMON, - FLB_CONF_TYPE_BOOL, - offsetof(struct flb_config, daemon)}, - - {FLB_CONF_STR_LOGFILE, - FLB_CONF_TYPE_STR, - offsetof(struct flb_config, log_file)}, - - {FLB_CONF_STR_PARSERS_FILE, - FLB_CONF_TYPE_STR, - offsetof(struct flb_config, parsers_file)}, - - {FLB_CONF_STR_PLUGINS_FILE, - FLB_CONF_TYPE_STR, - offsetof(struct flb_config, plugins_file)}, - - {FLB_CONF_STR_LOGLEVEL, - FLB_CONF_TYPE_STR, - offsetof(struct flb_config, log)}, - -#ifdef FLB_HAVE_HTTP_SERVER - {FLB_CONF_STR_HTTP_SERVER, - FLB_CONF_TYPE_BOOL, - offsetof(struct flb_config, http_server)}, - - {FLB_CONF_STR_HTTP_LISTEN, - FLB_CONF_TYPE_STR, - offsetof(struct flb_config, http_listen)}, - - {FLB_CONF_STR_HTTP_PORT, - FLB_CONF_TYPE_STR, - offsetof(struct flb_config, http_port)}, - - {FLB_CONF_STR_HEALTH_CHECK, - FLB_CONF_TYPE_BOOL, - offsetof(struct flb_config, health_check)}, - - {FLB_CONF_STR_HC_ERRORS_COUNT, - FLB_CONF_TYPE_INT, - offsetof(struct flb_config, hc_errors_count)}, - - {FLB_CONF_STR_HC_RETRIES_FAILURE_COUNT, - FLB_CONF_TYPE_INT, - offsetof(struct flb_config, hc_retry_failure_count)}, - - {FLB_CONF_STR_HC_PERIOD, - FLB_CONF_TYPE_INT, - offsetof(struct flb_config, health_check_period)}, - -#endif - /* DNS*/ - {FLB_CONF_DNS_MODE, - FLB_CONF_TYPE_STR, - offsetof(struct flb_config, dns_mode)}, - - {FLB_CONF_DNS_RESOLVER, - FLB_CONF_TYPE_STR, - offsetof(struct flb_config, dns_resolver)}, - - {FLB_CONF_DNS_PREFER_IPV4, - FLB_CONF_TYPE_BOOL, - offsetof(struct flb_config, dns_prefer_ipv4)}, - - /* Storage */ - {FLB_CONF_STORAGE_PATH, - FLB_CONF_TYPE_STR, - offsetof(struct flb_config, storage_path)}, - {FLB_CONF_STORAGE_SYNC, - FLB_CONF_TYPE_STR, - offsetof(struct flb_config, storage_sync)}, - {FLB_CONF_STORAGE_METRICS, - FLB_CONF_TYPE_BOOL, - offsetof(struct flb_config, storage_metrics)}, - {FLB_CONF_STORAGE_CHECKSUM, - FLB_CONF_TYPE_BOOL, - offsetof(struct flb_config, storage_checksum)}, - {FLB_CONF_STORAGE_BL_MEM_LIMIT, - FLB_CONF_TYPE_STR, - offsetof(struct flb_config, storage_bl_mem_limit)}, - {FLB_CONF_STORAGE_MAX_CHUNKS_UP, - FLB_CONF_TYPE_INT, - offsetof(struct flb_config, storage_max_chunks_up)}, - {FLB_CONF_STORAGE_DELETE_IRRECOVERABLE_CHUNKS, - FLB_CONF_TYPE_BOOL, - offsetof(struct flb_config, storage_del_bad_chunks)}, - {FLB_CONF_STORAGE_TRIM_FILES, - FLB_CONF_TYPE_BOOL, - offsetof(struct flb_config, storage_trim_files)}, - - /* Coroutines */ - {FLB_CONF_STR_CORO_STACK_SIZE, - FLB_CONF_TYPE_INT, - offsetof(struct flb_config, coro_stack_size)}, - - /* Scheduler */ - {FLB_CONF_STR_SCHED_CAP, - FLB_CONF_TYPE_INT, - offsetof(struct flb_config, sched_cap)}, - {FLB_CONF_STR_SCHED_BASE, - FLB_CONF_TYPE_INT, - offsetof(struct flb_config, sched_base)}, - -#ifdef FLB_HAVE_STREAM_PROCESSOR - {FLB_CONF_STR_STREAMS_FILE, - FLB_CONF_TYPE_STR, - offsetof(struct flb_config, stream_processor_file)}, - {FLB_CONF_STR_STREAMS_STR_CONV, - FLB_CONF_TYPE_BOOL, - offsetof(struct flb_config, stream_processor_str_conv)}, -#endif - -#ifdef FLB_HAVE_CHUNK_TRACE - {FLB_CONF_STR_ENABLE_CHUNK_TRACE, - FLB_CONF_TYPE_BOOL, - offsetof(struct flb_config, enable_chunk_trace)}, -#endif - - {FLB_CONF_STR_HOT_RELOAD, - FLB_CONF_TYPE_BOOL, - offsetof(struct flb_config, enable_hot_reload)}, - - {FLB_CONF_STR_HOT_RELOAD_ENSURE_THREAD_SAFETY, - FLB_CONF_TYPE_BOOL, - offsetof(struct flb_config, ensure_thread_safety_on_hot_reloading)}, - - {NULL, FLB_CONF_TYPE_OTHER, 0} /* end of array */ -}; - - -struct flb_config *flb_config_init() -{ - int ret; - struct flb_config *config; - struct flb_cf *cf; - struct flb_cf_section *section; - - config = flb_calloc(1, sizeof(struct flb_config)); - if (!config) { - flb_errno(); - return NULL; - } - - MK_EVENT_ZERO(&config->ch_event); - MK_EVENT_ZERO(&config->event_flush); - MK_EVENT_ZERO(&config->event_shutdown); - - /* is data ingestion active ? */ - config->is_ingestion_active = FLB_TRUE; - - /* Is the engine (event loop) actively running ? */ - config->is_running = FLB_TRUE; - - /* Initialize config_format context */ - cf = flb_cf_create(); - if (!cf) { - flb_free(config); - return NULL; - } - config->cf_main = cf; - - section = flb_cf_section_create(cf, "service", 0); - if (!section) { - flb_cf_destroy(cf); - flb_free(config); - return NULL; - } - - /* Flush */ - config->flush = FLB_CONFIG_FLUSH_SECS; - config->daemon = FLB_FALSE; - config->init_time = time(NULL); - config->kernel = flb_kernel_info(); - config->verbose = 3; - config->grace = 5; - config->grace_count = 0; - config->exit_status_code = 0; - - /* json */ - config->convert_nan_to_null = FLB_FALSE; - -#ifdef FLB_HAVE_HTTP_SERVER - config->http_ctx = NULL; - config->http_server = FLB_FALSE; - config->http_listen = flb_strdup(FLB_CONFIG_HTTP_LISTEN); - config->http_port = flb_strdup(FLB_CONFIG_HTTP_PORT); - config->health_check = FLB_FALSE; - config->hc_errors_count = HC_ERRORS_COUNT_DEFAULT; - config->hc_retry_failure_count = HC_RETRY_FAILURE_COUNTS_DEFAULT; - config->health_check_period = HEALTH_CHECK_PERIOD; -#endif - - config->http_proxy = getenv("HTTP_PROXY"); - if (flb_str_emptyval(config->http_proxy) == FLB_TRUE) { - config->http_proxy = getenv("http_proxy"); - if (flb_str_emptyval(config->http_proxy) == FLB_TRUE) { - /* Proxy should not be set when `HTTP_PROXY` or `http_proxy` are set to "" */ - config->http_proxy = NULL; - } - } - config->no_proxy = getenv("NO_PROXY"); - if (flb_str_emptyval(config->no_proxy) == FLB_TRUE || config->http_proxy == NULL) { - config->no_proxy = getenv("no_proxy"); - if (flb_str_emptyval(config->no_proxy) == FLB_TRUE || config->http_proxy == NULL) { - /* NoProxy should not be set when `NO_PROXY` or `no_proxy` are set to "" or there is no Proxy. */ - config->no_proxy = NULL; - } - } - - config->cio = NULL; - config->storage_path = NULL; - config->storage_input_plugin = NULL; - config->storage_metrics = FLB_TRUE; - - config->sched_cap = FLB_SCHED_CAP; - config->sched_base = FLB_SCHED_BASE; - - /* reload */ - config->ensure_thread_safety_on_hot_reloading = FLB_TRUE; - config->hot_reloaded_count = 0; - -#ifdef FLB_HAVE_SQLDB - mk_list_init(&config->sqldb_list); -#endif - -#ifdef FLB_HAVE_LUAJIT - mk_list_init(&config->luajit_list); -#endif - -#ifdef FLB_HAVE_STREAM_PROCESSOR - flb_slist_create(&config->stream_processor_tasks); - config->stream_processor_str_conv = FLB_TRUE; -#endif - - flb_slist_create(&config->external_plugins); - - /* Set default coroutines stack size */ - config->coro_stack_size = FLB_CORO_STACK_SIZE_BYTE; - if (config->coro_stack_size < getpagesize()) { - flb_info("[config] changing coro_stack_size from %u to %u bytes", - config->coro_stack_size, getpagesize()); - config->coro_stack_size = (unsigned int)getpagesize(); - } - - /* collectors */ - pthread_mutex_init(&config->collectors_mutex, NULL); - - /* Initialize linked lists */ - mk_list_init(&config->processor_plugins); - mk_list_init(&config->custom_plugins); - mk_list_init(&config->in_plugins); - mk_list_init(&config->parser_plugins); - mk_list_init(&config->filter_plugins); - mk_list_init(&config->out_plugins); - mk_list_init(&config->customs); - mk_list_init(&config->inputs); - mk_list_init(&config->parsers); - mk_list_init(&config->filters); - mk_list_init(&config->outputs); - mk_list_init(&config->proxies); - mk_list_init(&config->workers); - mk_list_init(&config->upstreams); - mk_list_init(&config->downstreams); - mk_list_init(&config->cmetrics); - mk_list_init(&config->cf_parsers_list); - - memset(&config->tasks_map, '\0', sizeof(config->tasks_map)); - - /* Initialize multiline-parser list. We need this here, because from now - * on we use flb_config_exit to cleanup the config, which requires - * the config->multiline_parsers list to be initialized. */ - mk_list_init(&config->multiline_parsers); - - /* Environment */ - config->env = flb_env_create(); - if (config->env == NULL) { - flb_error("[config] environment creation failed"); - flb_config_exit(config); - return NULL; - } - - /* Multiline core */ - ret = flb_ml_init(config); - if (ret == -1) { - flb_error("[config] multiline core initialization failed"); - flb_config_exit(config); - return NULL; - } - - /* Register static plugins */ - ret = flb_plugins_register(config); - if (ret == -1) { - flb_error("[config] plugins registration failed"); - flb_config_exit(config); - return NULL; - } - - /* Create environment for dynamic plugins */ - config->dso_plugins = flb_plugin_create(); - - /* Ignoring SIGPIPE on Windows (scary) */ -#ifndef _WIN32 - /* Ignore SIGPIPE */ - signal(SIGPIPE, SIG_IGN); -#endif - - /* Prepare worker interface */ - flb_worker_init(config); - -#ifdef FLB_HAVE_REGEX - /* Regex support */ - flb_regex_init(); -#endif - - return config; -} - -void flb_config_exit(struct flb_config *config) -{ - struct mk_list *tmp; - struct mk_list *head; - struct flb_cf *cf; - - if (config->log_file) { - flb_free(config->log_file); - } - - if (config->log) { - flb_log_destroy(config->log, config); - } - - if (config->parsers_file) { - flb_free(config->parsers_file); - } - - if (config->plugins_file) { - flb_free(config->plugins_file); - } - - if (config->kernel) { - flb_kernel_destroy(config->kernel); - } - - /* release resources */ - if (config->ch_event.fd) { - mk_event_closesocket(config->ch_event.fd); - } - - /* Pipe */ - if (config->ch_data[0]) { - mk_event_closesocket(config->ch_data[0]); - mk_event_closesocket(config->ch_data[1]); - } - - /* Channel manager */ - if (config->ch_manager[0] > 0) { - mk_event_closesocket(config->ch_manager[0]); - if (config->ch_manager[0] != config->ch_manager[1]) { - mk_event_closesocket(config->ch_manager[1]); - } - } - - /* Channel notifications */ - if (config->ch_notif[0] > 0) { - mk_event_closesocket(config->ch_notif[0]); - if (config->ch_notif[0] != config->ch_notif[1]) { - mk_event_closesocket(config->ch_notif[1]); - } - } - - if (config->env) { - flb_env_destroy(config->env); - } - - /* Program name */ - if (config->program_name) { - flb_sds_destroy(config->program_name); - } - - /* Conf path */ - if (config->conf_path) { - flb_free(config->conf_path); - } - - /* conf path file (file system config path) */ - if (config->conf_path_file) { - flb_sds_destroy(config->conf_path_file); - } - - /* Working directory */ - if (config->workdir) { - flb_free(config->workdir); - } - - /* Destroy any DSO context */ - if (config->dso_plugins) { - flb_plugin_destroy(config->dso_plugins); - } - - /* Workers */ - flb_worker_exit(config); - - /* Event flush */ - if (config->evl) { - if (config->event_flush.status != MK_EVENT_NONE) { - mk_event_timeout_destroy(config->evl, &config->event_flush); - } - } - - /* Release scheduler */ - if (config->sched) { - flb_sched_destroy(config->sched); - } - -#ifdef FLB_HAVE_HTTP_SERVER - if (config->http_listen) { - flb_free(config->http_listen); - } - - if (config->http_port) { - flb_free(config->http_port); - } -#endif - -#ifdef FLB_HAVE_PARSER - /* parsers */ - flb_parser_exit(config); -#endif - - if (config->dns_mode) { - flb_free(config->dns_mode); - } - if (config->dns_resolver) { - flb_free(config->dns_resolver); - } - - if (config->storage_path) { - flb_free(config->storage_path); - } - if (config->storage_sync) { - flb_free(config->storage_sync); - } - if (config->storage_bl_mem_limit) { - flb_free(config->storage_bl_mem_limit); - } - -#ifdef FLB_HAVE_STREAM_PROCESSOR - if (config->stream_processor_file) { - flb_free(config->stream_processor_file); - } - - flb_slist_destroy(&config->stream_processor_tasks); -#endif - - flb_slist_destroy(&config->external_plugins); - - if (config->evl) { - mk_event_loop_destroy(config->evl); - } - if (config->evl_bktq) { - flb_bucket_queue_destroy(config->evl_bktq); - } - - flb_plugins_unregister(config); - - if (config->cf_main) { - flb_cf_destroy(config->cf_main); - } - - /* cf_opts' lifetime should differ from config's lifetime. - * This member should be storing just for the cf_opts reference. - * Don't destroy it here. - */ - - /* remove parsers */ - mk_list_foreach_safe(head, tmp, &config->cf_parsers_list) { - cf = mk_list_entry(head, struct flb_cf, _head); - mk_list_del(&cf->_head); - flb_cf_destroy(cf); - } - - flb_free(config); -} - -const char *flb_config_prop_get(const char *key, struct mk_list *list) -{ - return flb_kv_get_key_value(key, list); -} - -static inline int prop_key_check(const char *key, const char *kv, int k_len) -{ - size_t len; - - len = strnlen(key,256); - if (strncasecmp(key, kv, k_len) == 0 && len == k_len) { - return 0; - } - return -1; -} - -static int set_log_level(struct flb_config *config, const char *v_str) -{ - if (v_str != NULL) { - if (strcasecmp(v_str, "error") == 0) { - config->verbose = 1; - } - else if (strcasecmp(v_str, "warn") == 0 || - strcasecmp(v_str, "warning") == 0) { - config->verbose = 2; - } - else if (strcasecmp(v_str, "info") == 0) { - config->verbose = 3; - } - else if (strcasecmp(v_str, "debug") == 0) { - config->verbose = 4; - } - else if (strcasecmp(v_str, "trace") == 0) { - config->verbose = 5; - } - else if (strcasecmp(v_str, "off") == 0) { - config->verbose = FLB_LOG_OFF; - } - else { - return -1; - } - } - else if (config->log) { - config->verbose = 3; - } - return 0; -} - -int set_log_level_from_env(struct flb_config *config) -{ - const char *val = NULL; - val = flb_env_get(config->env, FLB_CONF_ENV_LOGLEVEL); - if (val) { - return set_log_level(config, val); - } - return -1; -} - -int flb_config_set_property(struct flb_config *config, - const char *k, const char *v) -{ - int i=0; - int ret = -1; - int *i_val; - double *d_val; - char **s_val; - size_t len = strnlen(k, 256); - char *key = service_configs[0].key; - flb_sds_t tmp = NULL; - - while (key != NULL) { - if (prop_key_check(key, k,len) == 0) { - if (!strncasecmp(key, FLB_CONF_STR_LOGLEVEL, 256)) { - #ifndef FLB_HAVE_STATIC_CONF - if (set_log_level_from_env(config) < 0) { - #endif - tmp = flb_env_var_translate(config->env, v); - if (tmp) { - ret = set_log_level(config, tmp); - flb_sds_destroy(tmp); - tmp = NULL; - } - else { - ret = set_log_level(config, v); - } - #ifndef FLB_HAVE_STATIC_CONF - } - #endif - } - else if (!strncasecmp(key, FLB_CONF_STR_PARSERS_FILE, 32)) { -#ifdef FLB_HAVE_PARSER - tmp = flb_env_var_translate(config->env, v); - ret = flb_parser_conf_file(tmp, config); - flb_sds_destroy(tmp); - tmp = NULL; -#endif - } - else if (!strncasecmp(key, FLB_CONF_STR_PLUGINS_FILE, 32)) { - tmp = flb_env_var_translate(config->env, v); - ret = flb_plugin_load_config_file(tmp, config); - flb_sds_destroy(tmp); - tmp = NULL; - } - else { - ret = 0; - tmp = flb_env_var_translate(config->env, v); - switch(service_configs[i].type) { - case FLB_CONF_TYPE_INT: - i_val = (int*)((char*)config + service_configs[i].offset); - *i_val = atoi(tmp); - flb_sds_destroy(tmp); - break; - case FLB_CONF_TYPE_DOUBLE: - d_val = (double*)((char*)config + service_configs[i].offset); - *d_val = atof(tmp); - flb_sds_destroy(tmp); - break; - case FLB_CONF_TYPE_BOOL: - i_val = (int*)((char*)config+service_configs[i].offset); - *i_val = flb_utils_bool(tmp); - flb_sds_destroy(tmp); - break; - case FLB_CONF_TYPE_STR: - s_val = (char**)((char*)config+service_configs[i].offset); - if ( *s_val != NULL ) { - flb_free(*s_val); /* release before overwriting */ - } - - *s_val = flb_strdup(tmp); - flb_sds_destroy(tmp); - break; - default: - ret = -1; - } - } - - if (ret < 0) { - if (tmp) { - flb_sds_destroy(tmp); - } - return -1; - } - return 0; - } - key = service_configs[++i].key; - } - return 0; -} - -int flb_config_set_program_name(struct flb_config *config, char *name) -{ - config->program_name = flb_sds_create(name); - - if (!config->program_name) { - return -1; - } - - return 0; -} - -static int configure_plugins_type(struct flb_config *config, struct flb_cf *cf, enum section_type type) -{ - int ret; - char *tmp; - char *name; - char *s_type; - struct mk_list *list; - struct mk_list *head; - struct cfl_list *h_prop; - struct cfl_kvpair *kv; - struct cfl_variant *val; - struct flb_cf_section *s; - struct flb_cf_group *processors = NULL; - int i; - void *ins; - - if (type == FLB_CF_CUSTOM) { - s_type = "custom"; - list = &cf->customs; - } - else if (type == FLB_CF_INPUT) { - s_type = "input"; - list = &cf->inputs; - } - else if (type == FLB_CF_FILTER) { - s_type = "filter"; - list = &cf->filters; - } - else if (type == FLB_CF_OUTPUT) { - s_type = "output"; - list = &cf->outputs; - } - else { - return -1; - } - - mk_list_foreach(head, list) { - s = mk_list_entry(head, struct flb_cf_section, _head_section); - name = flb_cf_section_property_get_string(cf, s, "name"); - if (!name) { - flb_error("[config] section '%s' is missing the 'name' property", - s_type); - return -1; - } - - /* translate the variable */ - tmp = flb_env_var_translate(config->env, name); - - /* create an instance of the plugin */ - ins = NULL; - if (type == FLB_CF_CUSTOM) { - ins = flb_custom_new(config, tmp, NULL); - } - else if (type == FLB_CF_INPUT) { - ins = flb_input_new(config, tmp, NULL, FLB_TRUE); - } - else if (type == FLB_CF_FILTER) { - ins = flb_filter_new(config, tmp, NULL); - } - else if (type == FLB_CF_OUTPUT) { - ins = flb_output_new(config, tmp, NULL, FLB_TRUE); - } - flb_sds_destroy(tmp); - - /* validate the instance creation */ - if (!ins) { - flb_error("[config] section '%s' tried to instance a plugin name " - "that don't exists", name); - flb_sds_destroy(name); - return -1; - } - flb_sds_destroy(name); - - /* - * iterate section properties and populate instance by using specific - * api function. - */ - cfl_list_foreach(h_prop, &s->properties->list) { - kv = cfl_list_entry(h_prop, struct cfl_kvpair, _head); - if (strcasecmp(kv->key, "name") == 0) { - continue; - } - - /* set ret to -1 to ensure that we treat any unhandled plugin or - * value types as errors. - */ - ret = -1; - - if (type == FLB_CF_CUSTOM) { - if (kv->val->type == CFL_VARIANT_STRING) { - ret = flb_custom_set_property(ins, kv->key, kv->val->data.as_string); - } else if (kv->val->type == CFL_VARIANT_ARRAY) { - for (i = 0; i < kv->val->data.as_array->entry_count; i++) { - val = kv->val->data.as_array->entries[i]; - ret = flb_custom_set_property(ins, kv->key, val->data.as_string); - } - } - } - else if (type == FLB_CF_INPUT) { - if (kv->val->type == CFL_VARIANT_STRING) { - ret = flb_input_set_property(ins, kv->key, kv->val->data.as_string); - } else if (kv->val->type == CFL_VARIANT_ARRAY) { - for (i = 0; i < kv->val->data.as_array->entry_count; i++) { - val = kv->val->data.as_array->entries[i]; - ret = flb_input_set_property(ins, kv->key, val->data.as_string); - } - } - } - else if (type == FLB_CF_FILTER) { - if (kv->val->type == CFL_VARIANT_STRING) { - ret = flb_filter_set_property(ins, kv->key, kv->val->data.as_string); - } else if (kv->val->type == CFL_VARIANT_ARRAY) { - for (i = 0; i < kv->val->data.as_array->entry_count; i++) { - val = kv->val->data.as_array->entries[i]; - ret = flb_filter_set_property(ins, kv->key, val->data.as_string); - } - } - } - else if (type == FLB_CF_OUTPUT) { - if (kv->val->type == CFL_VARIANT_STRING) { - ret = flb_output_set_property(ins, kv->key, kv->val->data.as_string); - } else if (kv->val->type == CFL_VARIANT_ARRAY) { - for (i = 0; i < kv->val->data.as_array->entry_count; i++) { - val = kv->val->data.as_array->entries[i]; - ret = flb_output_set_property(ins, kv->key, val->data.as_string); - } - } - } - - if (ret == -1) { - flb_error("[config] could not configure property '%s' on " - "%s plugin with section name '%s'", - kv->key, s_type, name); - } - } - - /* Processors */ - processors = flb_cf_group_get(cf, s, "processors"); - if (processors) { - if (type == FLB_CF_INPUT) { - flb_processors_load_from_config_format_group(((struct flb_input_instance *) ins)->processor, processors); - } - else if (type == FLB_CF_OUTPUT) { - flb_processors_load_from_config_format_group(((struct flb_output_instance *) ins)->processor, processors); - } - else { - flb_error("[config] section '%s' does not support processors", s_type); - } - } - } - - return 0; -} -/* Load a struct flb_config_format context into a flb_config instance */ -int flb_config_load_config_format(struct flb_config *config, struct flb_cf *cf) -{ - int ret; - struct flb_kv *kv; - struct mk_list *head; - struct cfl_kvpair *ckv; - struct cfl_list *chead; - struct flb_cf_section *s; - - /* Process config environment vars */ - mk_list_foreach(head, &cf->env) { - kv = mk_list_entry(head, struct flb_kv, _head); - ret = flb_env_set(config->env, kv->key, kv->val); - if (ret == -1) { - flb_error("could not set config environment variable '%s'", kv->key); - return -1; - } - } - - /* Process all meta commands */ - mk_list_foreach(head, &cf->metas) { - kv = mk_list_entry(head, struct flb_kv, _head); - flb_meta_run(config, kv->key, kv->val); - } - - /* Validate sections */ - mk_list_foreach(head, &cf->sections) { - s = mk_list_entry(head, struct flb_cf_section, _head); - - if (strcasecmp(s->name, "env") == 0 || - strcasecmp(s->name, "service") == 0 || - strcasecmp(s->name, "custom") == 0 || - strcasecmp(s->name, "input") == 0 || - strcasecmp(s->name, "filter") == 0 || - strcasecmp(s->name, "output") == 0) { - - /* continue on valid sections */ - continue; - } - - /* Extra sanity checks */ - if (strcasecmp(s->name, "parser") == 0 || - strcasecmp(s->name, "multiline_parser") == 0) { - fprintf(stderr, - "Sections 'multiline_parser' and 'parser' are not valid in " - "the main configuration file. It belongs to \n" - "the 'parsers_file' configuration files.\n"); - return -1; - } - } - - /* Read main 'service' section */ - s = cf->service; - if (s) { - /* Iterate properties */ - cfl_list_foreach(chead, &s->properties->list) { - ckv = cfl_list_entry(chead, struct cfl_kvpair, _head); - flb_config_set_property(config, ckv->key, ckv->val->data.as_string); - } - } - - ret = configure_plugins_type(config, cf, FLB_CF_CUSTOM); - if (ret == -1) { - return -1; - } - - ret = configure_plugins_type(config, cf, FLB_CF_INPUT); - if (ret == -1) { - return -1; - } - ret = configure_plugins_type(config, cf, FLB_CF_FILTER); - if (ret == -1) { - return -1; - } - ret = configure_plugins_type(config, cf, FLB_CF_OUTPUT); - if (ret == -1) { - return -1; - } - - return 0; -} diff --git a/fluent-bit/src/flb_config_map.c b/fluent-bit/src/flb_config_map.c deleted file mode 100644 index 9a3dd7ac4..000000000 --- a/fluent-bit/src/flb_config_map.c +++ /dev/null @@ -1,817 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <fluent-bit/flb_info.h> -#include <fluent-bit/flb_kv.h> -#include <fluent-bit/flb_env.h> -#include <fluent-bit/flb_mem.h> -#include <fluent-bit/flb_log.h> -#include <fluent-bit/flb_utils.h> -#include <fluent-bit/flb_slist.h> -#include <fluent-bit/flb_macros.h> -#include <fluent-bit/flb_config_map.h> - -static int check_list_size(struct mk_list *list, int type) -{ - int len; - - len = mk_list_size(list); - if (type == FLB_CONFIG_MAP_SLIST_1 || type == FLB_CONFIG_MAP_CLIST_1) { - if (len < 1) { - return -1; - } - } - else if (type == FLB_CONFIG_MAP_SLIST_2 || type == FLB_CONFIG_MAP_CLIST_2) { - if (len < 2) { - return -1; - } - } - else if (type == FLB_CONFIG_MAP_SLIST_3 || type == FLB_CONFIG_MAP_CLIST_3) { - if (len < 3) { - return -1; - } - } - else if (type == FLB_CONFIG_MAP_SLIST_4 || type == FLB_CONFIG_MAP_CLIST_4) { - if (len < 4) { - return -1; - } - } - - return 0; -} - -/* - * Given a string, split the content using it proper separator generating a linked - * list of 'slist' - */ -static struct mk_list *parse_string_map_to_list(struct flb_config_map *map, char *str) -{ - int ret = -1; - int type; - int max_split = -1; - struct mk_list *list; - - type = map->type; - - /* Allocate list head */ - list = flb_malloc(sizeof(struct mk_list)); - if (!list) { - flb_errno(); - return NULL; - } - mk_list_init(list); - - /* Determinate the max split value based on it type */ - if (map->type > FLB_CONFIG_MAP_CLIST && map->type < FLB_CONFIG_MAP_SLIST) { - type = FLB_CONFIG_MAP_CLIST; - max_split = (map->type - FLB_CONFIG_MAP_CLIST); - } - else if (map->type > FLB_CONFIG_MAP_SLIST) { - type = FLB_CONFIG_MAP_SLIST; - max_split = (map->type - FLB_CONFIG_MAP_SLIST); - } - - if (type == FLB_CONFIG_MAP_CLIST) { - ret = flb_slist_split_string(list, str, ',', max_split); - } - else if (type == FLB_CONFIG_MAP_SLIST) { - ret = flb_slist_split_tokens(list, str, max_split); - } - - if (ret == -1) { - flb_error("[config map] error reading list of options"); - flb_free(list); - return NULL; - } - - return list; -} - -static int translate_default_value(struct flb_config_map *map, char *val) -{ - int ret; - struct flb_config_map_val *entry = NULL; - struct mk_list *list = NULL; - - /* Prepare contexts if the map allows multiple entries */ - if (map->flags & FLB_CONFIG_MAP_MULT) { - entry = flb_calloc(1, sizeof(struct flb_config_map_val)); - if (!entry) { - flb_errno(); - /* - * do not worry about 'list' allocation, it will be destroyed by the caller - * when it catches this error - */ - return -1; - } - } - else { - entry = &map->value; - } - - /* Based on specific data types, populate 'value' */ - if (map->type == FLB_CONFIG_MAP_STR) { - /* Duplicate string as a flb_sds_t */ - entry->val.str = flb_sds_create(val); - - /* Validate new memory allocation */ - if (!entry->val.str) { - goto error; - } - } - else if (map->type == FLB_CONFIG_MAP_STR_PREFIX) { - /* - * For prefixed string types we don't process them, just validate - * that no default value has been set. - */ - if (val) { - flb_error("[config map] invalid default value for prefixed string '%s'", - map->name); - goto error; - } - } - else if (map->type == FLB_CONFIG_MAP_BOOL) { - ret = flb_utils_bool(val); - if (ret == -1) { - flb_error("[config map] invalid default value for boolean '%s=%s'", - map->name, val); - goto error; - } - entry->val.boolean = flb_utils_bool(val); - } - else if (map->type == FLB_CONFIG_MAP_INT) { - entry->val.i_num = atoi(val); - } - else if (map->type == FLB_CONFIG_MAP_DOUBLE) { - entry->val.d_num = atof(val); - } - else if (map->type == FLB_CONFIG_MAP_SIZE) { - entry->val.s_num = flb_utils_size_to_bytes(val); - } - else if (map->type == FLB_CONFIG_MAP_TIME) { - entry->val.i_num = flb_utils_time_to_seconds(val); - } - else if (map->type >= FLB_CONFIG_MAP_CLIST && - map->type <= FLB_CONFIG_MAP_SLIST_4) { - - list = parse_string_map_to_list(map, val); - if (!list) { - flb_error("[config map] cannot parse list of values '%s'", val); - goto error; - } - - entry->val.list = list; - list = NULL; - } - - if (map->flags & FLB_CONFIG_MAP_MULT) { - mk_list_add(&entry->_head, map->value.mult); - } - - return 0; - - error: - if (map->flags & FLB_CONFIG_MAP_MULT) { - flb_free(entry); - } - return -1; -} - -static flb_sds_t helper_map_options(struct mk_list *map) -{ - flb_sds_t buf; - flb_sds_t tmp; - struct mk_list *head; - struct flb_config_map *m; - - buf = flb_sds_create_size(256); - if (!buf) { - flb_errno(); - return NULL; - } - - tmp = flb_sds_printf(&buf, "The following properties are allowed: "); - if (!tmp) { - flb_errno(); - flb_sds_destroy(buf); - return NULL; - } - buf = tmp; - - mk_list_foreach(head, map) { - m = mk_list_entry(head, struct flb_config_map, _head); - if (head->next != map) { - tmp = flb_sds_printf(&buf, "%s, ", m->name); - } - else { - if (mk_list_size(map) == 1) { - tmp = flb_sds_printf(&buf, "%s.", m->name); - } - else { - tmp = flb_sds_printf(&buf, "and %s.", m->name); - } - } - - if (!tmp) { - flb_errno(); - flb_sds_destroy(buf); - return NULL; - } - buf = tmp; - } - - return buf; -} - -/* - * Given a static plugin configuration map, create a linked list representation. We use a - * linked list using heap memory instead of the stack since a plugin can be loaded multiple - * times. - * - * In addition, for default values, we process them and populate the 'value' field with - * proper data types. - */ -struct mk_list *flb_config_map_create(struct flb_config *config, - struct flb_config_map *map) -{ - int ret; - flb_sds_t env; - struct mk_list *tmp; - struct mk_list *list; - struct flb_config_map *new = NULL; - struct flb_config_map *m; - - list = flb_malloc(sizeof(struct mk_list)); - if (!list) { - flb_errno(); - return NULL; - } - mk_list_init(list); - - /* - * Read every property defined in the config map and create a new dynamic list - * with the same content. - * - * As an additional step, it populate the 'value' field using the given default - * value if any. Note that default values are strings so they are processed - * to fit into the proper data type of 'value'. - */ - m = map; - while (m && m->name) { - /* Allocate map node */ - new = flb_calloc(1, sizeof(struct flb_config_map)); - if (!new) { - flb_errno(); - flb_config_map_destroy(list); - return NULL; - } - - new->type = m->type; - new->name = flb_sds_create(m->name); - if (new->name == NULL) { - flb_free(new); - flb_config_map_destroy(list); - return NULL; - } - - /* Translate default value */ - if (m->def_value) { - /* - * Before to translate any value, make sure to disable the warning - * about unused variables. This might happen if a default value is an - * environment variable and the user is not using it (which is ok for - * that specific use case). - */ - flb_env_warn_unused(config->env, FLB_FALSE); - - /* Translate the value */ - env = flb_env_var_translate(config->env, m->def_value); - if (env == NULL) { - flb_errno(); - flb_sds_destroy(new->name); - flb_free(new); - flb_config_map_destroy(list); - return NULL; - } - new->def_value = env; - flb_env_warn_unused(config->env, FLB_TRUE); - } - - new->flags = m->flags; - new->set_property = m->set_property; - new->offset = m->offset; - new->value.mult = NULL; - new->desc = m->desc; - mk_list_add(&new->_head, list); - - if (new->set_property == FLB_FALSE) { - m++; - continue; - } - - /* If this is a multiple type of entries, initialize the main list */ - if (new->flags & FLB_CONFIG_MAP_MULT) { - tmp = flb_malloc(sizeof(struct mk_list)); - if (!tmp) { - flb_errno(); - flb_config_map_destroy(list); - return NULL; - } - mk_list_init(tmp); - new->value.mult = tmp; - } - - /* - * If there is no default value or the entry will not be set, just - * continue with the next map entry - */ - if (!m->def_value) { - m++; - continue; - } - - /* Assign value based on data type and multiple mode if set */ - ret = translate_default_value(new, new->def_value); - if (ret == -1) { - flb_config_map_destroy(list); - return NULL; - } - m++; - } - - return list; -} - -static void destroy_map_val(int type, struct flb_config_map_val *value) -{ - if (type == FLB_CONFIG_MAP_STR && value->val.str) { - flb_sds_destroy(value->val.str); - } - else if ((type >= FLB_CONFIG_MAP_CLIST && - type <= FLB_CONFIG_MAP_SLIST_4) && - value->val.list) { - flb_slist_destroy(value->val.list); - flb_free(value->val.list); - } -} - -/* Destroy a config map context */ -void flb_config_map_destroy(struct mk_list *list) -{ - struct mk_list *tmp; - struct mk_list *head; - struct mk_list *v_head; - struct mk_list *v_tmp; - struct flb_config_map *map; - struct flb_config_map_val *entry; - - mk_list_foreach_safe(head, tmp, list) { - map = mk_list_entry(head, struct flb_config_map, _head); - mk_list_del(&map->_head); - - if (map->flags & FLB_CONFIG_MAP_MULT && map->value.mult) { - mk_list_foreach_safe(v_head, v_tmp, map->value.mult) { - entry = mk_list_entry(v_head, struct flb_config_map_val, _head); - mk_list_del(&entry->_head); - destroy_map_val(map->type, entry); - flb_free(entry); - } - flb_free(map->value.mult); - } - else { - destroy_map_val(map->type, &map->value); - } - if (map->def_value) { - flb_sds_destroy(map->def_value); - } - flb_sds_destroy(map->name); - flb_free(map); - } - flb_free(list); -} - -/* Count the number of times a property key exists */ -int property_count(char *key, int len, struct mk_list *properties) -{ - int count = 0; - struct mk_list *head; - struct flb_kv *kv; - - mk_list_foreach(head, properties) { - kv = mk_list_entry(head, struct flb_kv, _head); - if (flb_sds_len(kv->key) != len) { - continue; - } - - if (strncmp(kv->key, key, len) == 0) { - count++; - } - } - return count; -} - -/* - * If the property starts with '_debug.', it's an internal property for - * some component of Fluent Bit, not the plugin it self. - */ -static int is_internal_debug_property(char *prop_name) -{ -#ifdef FLB_HAVE_HTTP_CLIENT_DEBUG - if (strncmp(prop_name, "_debug.http.", 12) == 0) { - return FLB_TRUE; - } -#endif - - return FLB_FALSE; -} - - -/* Validate that the incoming properties set by the caller are allowed by the plugin */ -int flb_config_map_properties_check(char *context_name, - struct mk_list *in_properties, - struct mk_list *map) -{ - int len; - int found; - int count = 0; - int ret; - flb_sds_t helper; - struct flb_kv *kv; - struct mk_list *head; - struct mk_list *m_head; - struct flb_config_map *m; - - /* Iterate all incoming property list */ - mk_list_foreach(head, in_properties) { - kv = mk_list_entry(head, struct flb_kv, _head); - found = FLB_FALSE; - - - ret = is_internal_debug_property(kv->key); - if (ret == FLB_TRUE) { - /* Skip the config map */ - continue; - } - - if (strcasecmp(kv->key, "active") == 0) { - /* Accept 'active' property ... */ - continue; - } - - /* Lookup the key into the provided map */ - mk_list_foreach(m_head, map) { - m = mk_list_entry(m_head, struct flb_config_map, _head); - - len = flb_sds_len(m->name); - if (m->type != FLB_CONFIG_MAP_STR_PREFIX) { - if (len != flb_sds_len(kv->key)) { - continue; - } - } - - if (strncasecmp(kv->key, m->name, len) == 0) { - if (m->type == FLB_CONFIG_MAP_STR_PREFIX) { - if (flb_sds_len(kv->key) <= len) { - flb_error("[config] incomplete prefixed key '%s'", kv->key); - found = FLB_FALSE; - break; - } - } - else if(m->type == FLB_CONFIG_MAP_DEPRECATED) { - flb_warn("[config] %s: '%s' is deprecated", - context_name, kv->key); - } - found = FLB_TRUE; - break; - } - } - - if (found == FLB_FALSE) { - helper = helper_map_options(map); - if (!helper) { - flb_error("[config] %s: unknown configuration property '%s'", - context_name, kv->key); - } - else { - flb_error("[config] %s: unknown configuration property '%s'. %s", - context_name, kv->key, helper); - flb_sds_destroy(helper); - } - - return -1; - } - - /* Validate number of times the property is set */ - count = property_count(kv->key, flb_sds_len(kv->key), in_properties); - if ((m->flags & FLB_CONFIG_MAP_MULT) == 0) { - if (count > 1) { - flb_error("[config] %s: configuration property '%s' is set %i times", - context_name, kv->key, count); - return -1; - } - } - } - - return 0; -} - -/* - * Returns FLB_TRUE or FLB_FALSE if a property aims to override the default value - * assigned to the map key valled 'name'. - */ -static int properties_override_default(struct mk_list *properties, char *name) -{ - int len; - struct mk_list *head; - struct flb_kv *kv; - - len = strlen(name); - - mk_list_foreach(head, properties) { - kv = mk_list_entry(head, struct flb_kv, _head); - if (flb_sds_len(kv->key) != len) { - continue; - } - - if (strcasecmp(kv->key, name) == 0) { - return FLB_TRUE; - } - } - - return FLB_FALSE; -} - -/* - * Return the number of expected values if the property type is from CLIST - * or SLIST family. - */ -int flb_config_map_expected_values(int type) -{ - if (type > FLB_CONFIG_MAP_CLIST && type < FLB_CONFIG_MAP_SLIST) { - return type - FLB_CONFIG_MAP_CLIST; - } - if (type > FLB_CONFIG_MAP_SLIST && type <= FLB_CONFIG_MAP_SLIST_4) { - return type - FLB_CONFIG_MAP_SLIST; - } - return -1; -} - - -/* - * Function used by plugins that needs to populate their context structure with the - * configuration properties already mapped. - */ -int flb_config_map_set(struct mk_list *properties, struct mk_list *map, void *context) -{ - int ret; - int len; - char *base; - char *m_bool; - int *m_i_num; - double *m_d_num; - size_t *m_s_num; - flb_sds_t *m_str; - struct flb_kv *kv; - struct mk_list *head; - struct mk_list *m_head; - struct mk_list **m_list; - struct mk_list *list; - struct flb_config_map *m = NULL; - struct flb_config_map_val *entry = NULL; - - base = context; - - /* Link 'already processed default values' into the caller context */ - mk_list_foreach(m_head, map) { - m = mk_list_entry(m_head, struct flb_config_map, _head); - - /* - * If the map type allows multiple entries, the user context is a pointer - * for a linked list. We just point their structure to our pre-processed - * list of entries. - */ - if (m->flags & FLB_CONFIG_MAP_MULT && m->set_property == FLB_TRUE) { - m_list = (struct mk_list **) (base + m->offset); - *m_list = m->value.mult; - continue; - } - - /* - * If no default value exists or the map will not write to the user - * context.. skip it. - */ - if (!m->def_value || m->set_property == FLB_FALSE) { - continue; - } - - /* - * If a property set by the user will override the default value, just - * do not put the default value into the context since it will be replaced - * later. - */ - ret = properties_override_default(properties, m->name); - if (ret == FLB_TRUE) { - continue; - } - - /* All the following steps are direct writes to the user context */ - if (m->type == FLB_CONFIG_MAP_STR) { - m_str = (char **) (base + m->offset); - *m_str = m->value.val.str; - } - else if (m->type == FLB_CONFIG_MAP_INT) { - m_i_num = (int *) (base + m->offset); - *m_i_num = m->value.val.i_num; - } - else if (m->type == FLB_CONFIG_MAP_DOUBLE) { - m_d_num = (double *) (base + m->offset); - *m_d_num = m->value.val.d_num; - } - else if (m->type == FLB_CONFIG_MAP_SIZE) { - m_s_num = (size_t *) (base + m->offset); - *m_s_num = m->value.val.s_num; - } - else if (m->type == FLB_CONFIG_MAP_TIME) { - m_i_num = (int *) (base + m->offset); - *m_i_num = m->value.val.s_num; - } - else if (m->type == FLB_CONFIG_MAP_BOOL) { - m_bool = (char *) (base + m->offset); - *m_bool = m->value.val.boolean; - } - else if (m->type >= FLB_CONFIG_MAP_CLIST || - m->type <= FLB_CONFIG_MAP_SLIST_4) { - m_list = (struct mk_list **) (base + m->offset); - *m_list = m->value.val.list; - } - } - - /* - * Iterate all properties coming from the configuration reader. If a property overrides - * a default value already set in the previous step, just link to the new value. - */ - mk_list_foreach(head, properties) { - kv = mk_list_entry(head, struct flb_kv, _head); - if (kv->val == NULL) { - continue; - } - - mk_list_foreach(m_head, map) { - m = mk_list_entry(m_head, struct flb_config_map, _head); - if (flb_sds_len(kv->key) != flb_sds_len(m->name)) { - m = NULL; - continue; - } - - if (strncasecmp(kv->key, m->name, flb_sds_len(m->name)) == 0) { - break; - } - m = NULL; - continue; - - } - - if (!m || m->set_property == FLB_FALSE) { - continue; - } - - /* Check if the map allows multiple entries */ - if (m->flags & FLB_CONFIG_MAP_MULT) { - /* Create node */ - entry = flb_calloc(1, sizeof(struct flb_config_map_val)); - if (!entry) { - flb_errno(); - return -1; - } - - /* Populate value */ - if (m->type == FLB_CONFIG_MAP_STR) { - entry->val.str = flb_sds_create(kv->val); - } - else if (m->type == FLB_CONFIG_MAP_INT) { - entry->val.i_num = atoi(kv->val); - } - else if (m->type == FLB_CONFIG_MAP_DOUBLE) { - entry->val.d_num = atof(kv->val); - } - else if (m->type == FLB_CONFIG_MAP_SIZE) { - entry->val.s_num = flb_utils_size_to_bytes(kv->val); - } - else if (m->type == FLB_CONFIG_MAP_TIME) { - entry->val.i_num = flb_utils_time_to_seconds(kv->val); - } - else if (m->type == FLB_CONFIG_MAP_BOOL) { - ret = flb_utils_bool(kv->val); - if (ret == -1) { - flb_free(entry); - flb_error("[config map] invalid value for boolean property '%s=%s'", - m->name, kv->val); - return -1; - } - entry->val.boolean = ret; - } - else if (m->type >= FLB_CONFIG_MAP_CLIST || - m->type <= FLB_CONFIG_MAP_SLIST_4) { - - list = parse_string_map_to_list(m, kv->val); - if (!list) { - flb_error("[config map] cannot parse list of values '%s'", kv->val); - flb_free(entry); - return -1; - } - entry->val.list = list; - - /* Validate the number of entries are the minimum expected */ - len = mk_list_size(list); - ret = check_list_size(list, m->type); - if (ret == -1) { - flb_error("[config map] property '%s' expects %i values " - "(only %i were found)", - kv->key, - flb_config_map_expected_values(m->type), len); - /* - * Register the entry anyways, so on exit the resources will - * be released - */ - mk_list_add(&entry->_head, m->value.mult); - return -1; - } - } - - /* Add entry to the map 'mult' list tail */ - mk_list_add(&entry->_head, m->value.mult); - - /* Override user context */ - m_list = (struct mk_list **) (base + m->offset); - *m_list = m->value.mult; - } - else if (map != NULL) { - /* Direct write to user context */ - if (m->type == FLB_CONFIG_MAP_STR) { - m_str = (char **) (base + m->offset); - *m_str = kv->val; - } - else if (m->type == FLB_CONFIG_MAP_INT) { - m_i_num = (int *) (base + m->offset); - *m_i_num = atoi(kv->val); - } - else if (m->type == FLB_CONFIG_MAP_DOUBLE) { - m_d_num = (double *) (base + m->offset); - *m_d_num = atof(kv->val); - } - else if (m->type == FLB_CONFIG_MAP_BOOL) { - m_bool = (char *) (base + m->offset); - ret = flb_utils_bool(kv->val); - if (ret == -1) { - flb_error("[config map] invalid value for boolean property '%s=%s'", - m->name, kv->val); - return -1; - } - *m_bool = ret; - } - else if (m->type == FLB_CONFIG_MAP_SIZE) { - m_s_num = (size_t *) (base + m->offset); - *m_s_num = flb_utils_size_to_bytes(kv->val); - } - else if (m->type == FLB_CONFIG_MAP_TIME) { - m_i_num = (int *) (base + m->offset); - *m_i_num = flb_utils_time_to_seconds(kv->val); - } - else if (m->type >= FLB_CONFIG_MAP_CLIST || - m->type <= FLB_CONFIG_MAP_SLIST_4) { - list = parse_string_map_to_list(m, kv->val); - if (!list) { - flb_error("[config map] cannot parse list of values '%s'", kv->val); - flb_free(entry); - return -1; - } - - if (m->value.val.list) { - destroy_map_val(m->type, &m->value); - } - - m->value.val.list = list; - m_list = (struct mk_list **) (base + m->offset); - *m_list = m->value.val.list; - } - } - } - - return 0; -} diff --git a/fluent-bit/src/flb_config_static.c b/fluent-bit/src/flb_config_static.c deleted file mode 100644 index 7005b4ada..000000000 --- a/fluent-bit/src/flb_config_static.c +++ /dev/null @@ -1,63 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <fluent-bit/flb_info.h> -#include <fluent-bit/flb_config_format.h> -#include <fluent-bit/conf/flb_static_conf.h> - -/* - * If Fluent Bit has static configuration support, this function allows to lookup, - * parse and create configuration file contexts from the entries generated by - * CMake at build-time. - * - * This routing passes a 'virtual' file name that should be registered into the - * static array 'flb_config_files'. Learn more about it at: - * - * include/fluent-bit/conf/flb_static_conf.h - * - */ -struct flb_cf *flb_config_static_open(const char *file) -{ - int i; - const char *k = NULL; - const char *v = NULL; - struct flb_cf *cf; - - /* Iterate static array and lookup the file name */ - for (i = 0; i < flb_config_files_size; i++) { - k = (const char *) flb_config_files[i][0]; - v = (const char *) flb_config_files[i][1]; - - if (strcmp(k, file) == 0) { - break; - } - k = NULL; - } - - if (!k) { - return NULL; - } - - cf = flb_cf_fluentbit_create(NULL, (char *) file, (char *) v, 0); - if (!cf) { - return NULL; - } - - return cf; -} diff --git a/fluent-bit/src/flb_connection.c b/fluent-bit/src/flb_connection.c deleted file mode 100644 index a3ed40265..000000000 --- a/fluent-bit/src/flb_connection.c +++ /dev/null @@ -1,257 +0,0 @@ -#include <assert.h> - -#include <fluent-bit/flb_connection.h> -#include <fluent-bit/flb_upstream.h> -#include <fluent-bit/flb_downstream.h> - -int flb_connection_setup(struct flb_connection *connection, - flb_sockfd_t socket, - int type, - void *stream, - struct mk_event_loop *event_loop, - struct flb_coro *coroutine) -{ - assert(connection != NULL); - - memset(connection, 0, sizeof(struct flb_connection)); - - connection->fd = socket; - connection->type = type; - connection->stream = stream; - connection->net_error = -1; - connection->evl = event_loop; - connection->coroutine = coroutine; - connection->tls_session = NULL; - connection->ts_created = time(NULL); - connection->ts_assigned = time(NULL); - connection->busy_flag = FLB_FALSE; - connection->shutdown_flag = FLB_FALSE; - - connection->net = &connection->stream->net; - - assert(connection->net != NULL); - - MK_EVENT_ZERO(&connection->event); - - flb_connection_unset_connection_timeout(connection); - flb_connection_unset_io_timeout(connection); - - return 0; -} - -struct flb_connection *flb_connection_create(flb_sockfd_t socket, - int type, - void *stream, - struct mk_event_loop *event_loop, - struct flb_coro *coroutine) -{ - struct flb_connection *connection; - int result; - - connection = flb_calloc(1, sizeof(struct flb_connection)); - - if (connection == NULL) { - flb_errno(); - } - else { - result = flb_connection_setup(connection, - socket, - type, - stream, - event_loop, - coroutine); - - if (result != 0) { - flb_connection_destroy(connection); - - connection = NULL; - } - else { - connection->dynamically_allocated = FLB_TRUE; - } - } - - return connection; -} - -void flb_connection_destroy(struct flb_connection *connection) -{ - assert(connection != NULL); - - if (connection->dynamically_allocated) { - flb_free(connection); - } -} - -static void compose_user_friendly_remote_host(struct flb_connection *connection) -{ - int connection_type; - - connection_type = connection->stream->transport; - - if (connection_type == FLB_TRANSPORT_TCP) { - snprintf(connection->user_friendly_remote_host, - sizeof(connection->user_friendly_remote_host), - "tcp://%s:%u", - connection->remote_host, - connection->remote_port); - } - else if (connection_type == FLB_TRANSPORT_UDP) { - snprintf(connection->user_friendly_remote_host, - sizeof(connection->user_friendly_remote_host), - "udp://%s:%u", - connection->remote_host, - connection->remote_port); - } - else if (connection_type == FLB_TRANSPORT_UNIX_STREAM) { - snprintf(connection->user_friendly_remote_host, - sizeof(connection->user_friendly_remote_host), - "unix://%s", - connection->remote_host); - } - else if (connection_type == FLB_TRANSPORT_UNIX_DGRAM) { - snprintf(connection->user_friendly_remote_host, - sizeof(connection->user_friendly_remote_host), - "unix://%s", - connection->remote_host); - } -} - -void flb_connection_set_remote_host(struct flb_connection *connection, - struct sockaddr *remote_host) -{ - size_t address_size; - - address_size = flb_network_address_size((struct sockaddr_storage *) remote_host); - - if (address_size > 0 && - address_size < sizeof(struct sockaddr_storage)) { - memcpy(&connection->raw_remote_host, - remote_host, - address_size); - } -} - -char *flb_connection_get_remote_address(struct flb_connection *connection) -{ - int address_refresh_required; - size_t dummy_size_receptacle; - int refresh_required; - int stream_type; - int transport; - int result; - - stream_type = connection->stream->type; - transport = connection->stream->transport; - - address_refresh_required = FLB_FALSE; - refresh_required = FLB_FALSE; - - if (stream_type == FLB_DOWNSTREAM) { - if (transport == FLB_TRANSPORT_UDP) { - if (connection->raw_remote_host.ss_family != AF_UNSPEC) { - refresh_required = FLB_TRUE; - } - } - else if (transport == FLB_TRANSPORT_TCP || - transport == FLB_TRANSPORT_UNIX_STREAM) { - if (connection->raw_remote_host.ss_family == AF_UNSPEC) { - address_refresh_required = FLB_TRUE; - } - } - } - else if (stream_type == FLB_UPSTREAM) { - if (transport == FLB_TRANSPORT_TCP || - transport == FLB_TRANSPORT_UNIX_STREAM) { - if (connection->raw_remote_host.ss_family == AF_UNSPEC) { - address_refresh_required = FLB_TRUE; - } - } - } - - if (connection->remote_port == 0) { - refresh_required = FLB_TRUE; - } - - if (refresh_required) { - if (address_refresh_required) { - result = flb_net_socket_peer_address(connection->fd, - &connection->raw_remote_host); - } - - result = flb_net_socket_address_info(connection->fd, - &connection->raw_remote_host, - &connection->remote_port, - connection->remote_host, - sizeof(connection->remote_host), - &dummy_size_receptacle); - - if (result == 0) { - compose_user_friendly_remote_host(connection); - } - } - - return connection->user_friendly_remote_host; -} - -int flb_connection_get_flags(struct flb_connection *connection) -{ - return flb_stream_get_flags(connection->stream); -} - -void flb_connection_reset_connection_timeout(struct flb_connection *connection) -{ - time_t current_time; - time_t timeout_time; - - assert(connection != NULL); - - if (connection->type == FLB_UPSTREAM_CONNECTION) { - if (connection->net->connect_timeout > 0) { - current_time = time(NULL); - timeout_time = current_time + connection->net->connect_timeout; - - connection->ts_connect_start = current_time; - connection->ts_connect_timeout = timeout_time; - } - } - else if(connection->type == FLB_DOWNSTREAM_CONNECTION) { - if (connection->net->accept_timeout > 0) { - current_time = time(NULL); - timeout_time = current_time + connection->net->accept_timeout; - - connection->ts_connect_start = current_time; - connection->ts_connect_timeout = timeout_time; - } - } -} - -void flb_connection_unset_connection_timeout(struct flb_connection *connection) -{ - assert(connection != NULL); - - connection->ts_connect_start = -1; - connection->ts_connect_timeout = -1; -} - -void flb_connection_reset_io_timeout(struct flb_connection *connection) -{ - time_t current_time; - time_t timeout_time; - - assert(connection != NULL); - - if (connection->net->io_timeout > 0) { - current_time = time(NULL); - timeout_time = current_time + connection->net->io_timeout; - - connection->ts_io_timeout = timeout_time; - } -} - -void flb_connection_unset_io_timeout(struct flb_connection *connection) -{ - assert(connection != NULL); - - connection->ts_io_timeout = -1; -}
\ No newline at end of file diff --git a/fluent-bit/src/flb_coro.c b/fluent-bit/src/flb_coro.c deleted file mode 100644 index 0d4e67f09..000000000 --- a/fluent-bit/src/flb_coro.c +++ /dev/null @@ -1,56 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <fluent-bit/flb_info.h> -#include <fluent-bit/flb_thread_storage.h> -#include <fluent-bit/flb_coro.h> - -FLB_TLS_DEFINE(struct flb_coro, flb_coro_key); - -static pthread_mutex_t coro_mutex_init; - -void flb_coro_init() -{ - FLB_TLS_INIT(flb_coro_key); - pthread_mutex_init(&coro_mutex_init, NULL); -} - -void flb_coro_thread_init() -{ - size_t s; - cothread_t th; - - pthread_mutex_lock(&coro_mutex_init); - th = co_create(256, NULL, &s); - co_delete(th); - pthread_mutex_unlock(&coro_mutex_init); -} - -struct flb_coro *flb_coro_get() -{ - struct flb_coro *coro; - - coro = FLB_TLS_GET(flb_coro_key); - return coro; -} - -void flb_coro_set(struct flb_coro *coro) -{ - FLB_TLS_SET(flb_coro_key, coro); -} diff --git a/fluent-bit/src/flb_crypto.c b/fluent-bit/src/flb_crypto.c deleted file mode 100644 index c2811039a..000000000 --- a/fluent-bit/src/flb_crypto.c +++ /dev/null @@ -1,405 +0,0 @@ -/* Fluent Bit - * ========== - * Copyright (C) 2019-2020 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <fluent-bit/flb_crypto.h> -#include <openssl/bio.h> -#include <string.h> - -static int flb_crypto_get_rsa_padding_type_by_id(int padding_type_id) -{ - int result; - - if (padding_type_id == FLB_CRYPTO_PADDING_PKCS1) { - result = RSA_PKCS1_PADDING; - } - else if (padding_type_id == FLB_CRYPTO_PADDING_PKCS1_OEAP) { - result = RSA_PKCS1_OAEP_PADDING; - } - else if (padding_type_id == FLB_CRYPTO_PADDING_PKCS1_X931) { - result = RSA_X931_PADDING; - } - else if (padding_type_id == FLB_CRYPTO_PADDING_PKCS1_PSS) { - result = RSA_PKCS1_PSS_PADDING; - } - else { - result = FLB_CRYPTO_PADDING_NONE; - } - - return result; -} - -static const EVP_MD *flb_crypto_get_digest_algorithm_instance_by_id(int algorithm_id) -{ - const EVP_MD *algorithm; - - if (algorithm_id == FLB_HASH_SHA256) { - algorithm = EVP_sha256(); - } - else if (algorithm_id == FLB_HASH_SHA512) { - algorithm = EVP_sha512(); - } - else if (algorithm_id == FLB_HASH_MD5) { - algorithm = EVP_md5(); - } - else { - algorithm = NULL; - } - - return algorithm; -} - -static int flb_crypto_import_pem_key(int key_type, - unsigned char *key, - size_t key_length, - EVP_PKEY **ingested_key) -{ - BIO *io_provider; - int result; - - if (key_type != FLB_CRYPTO_PUBLIC_KEY && - key_type != FLB_CRYPTO_PRIVATE_KEY) { - return FLB_CRYPTO_INVALID_ARGUMENT; - } - - if (ingested_key == NULL) { - return FLB_CRYPTO_INVALID_ARGUMENT; - } - - result = FLB_CRYPTO_BACKEND_ERROR; - - io_provider = BIO_new_mem_buf((void*) key, key_length); - - if (io_provider != NULL) { - if (ingested_key != NULL) { - if (key_type == FLB_CRYPTO_PRIVATE_KEY) { - *ingested_key = PEM_read_bio_PrivateKey(io_provider, - NULL, NULL, - NULL); - } - else if (key_type == FLB_CRYPTO_PUBLIC_KEY) { - *ingested_key = PEM_read_bio_PUBKEY(io_provider, - NULL, NULL, - NULL); - - // printf("\n\nFAILURE? %p\n\n", *ingested_key); - // printf("ERROR : %s\n", ERR_error_string(ERR_get_error(), NULL)); - // exit(0); - } - - if (*ingested_key != NULL) { - result = FLB_CRYPTO_SUCCESS; - } - } - - BIO_free_all(io_provider); - } - - return result; -} - -int flb_crypto_init(struct flb_crypto *context, - int padding_type, - int digest_algorithm, - int key_type, - unsigned char *key, - size_t key_length) -{ - int result; - - if (context == NULL) { - return FLB_CRYPTO_INVALID_ARGUMENT; - } - - if (key == NULL) { - return FLB_CRYPTO_INVALID_ARGUMENT; - } - - if (key_length == 0) { - return FLB_CRYPTO_INVALID_ARGUMENT; - } - - memset(context, 0, sizeof(struct flb_crypto)); - - result = flb_crypto_import_pem_key(key_type, - key, - key_length, - &context->key); - - if (result != FLB_CRYPTO_SUCCESS) { - if (result == FLB_CRYPTO_BACKEND_ERROR) { - context->last_error = ERR_get_error(); - } - - flb_crypto_cleanup(context); - - return result; - } - - context->backend_context = EVP_PKEY_CTX_new(context->key, NULL); - - if (context->backend_context == NULL) { - context->last_error = ERR_get_error(); - - flb_crypto_cleanup(context); - - return result; - } - - context->block_size = (size_t) EVP_PKEY_size(context->key); - - context->padding_type = flb_crypto_get_rsa_padding_type_by_id(padding_type); - - context->digest_algorithm = flb_crypto_get_digest_algorithm_instance_by_id(digest_algorithm); - - return FLB_CRYPTO_SUCCESS; -} - - -int flb_crypto_cleanup(struct flb_crypto *context) -{ - if (context->backend_context != NULL) { - EVP_PKEY_free(context->key); - - context->key = NULL; - } - - if (context->backend_context != NULL) { - EVP_PKEY_CTX_free(context->backend_context); - - context->backend_context = NULL; - } - - return FLB_CRYPTO_SUCCESS; -} - -int flb_crypto_transform(struct flb_crypto *context, - int operation, - unsigned char *input_buffer, - size_t input_length, - unsigned char *output_buffer, - size_t *output_length) -{ - int result = FLB_CRYPTO_BACKEND_ERROR; - - if (context == NULL) { - return FLB_CRYPTO_INVALID_ARGUMENT; - } - - if (operation != FLB_CRYPTO_OPERATION_SIGN && - operation != FLB_CRYPTO_OPERATION_ENCRYPT && - operation != FLB_CRYPTO_OPERATION_DECRYPT) { - return FLB_CRYPTO_INVALID_ARGUMENT; - } - - if (context->last_operation == FLB_CRYPTO_OPERATION_NONE) { - if (operation == FLB_CRYPTO_OPERATION_SIGN) { - result = EVP_PKEY_sign_init(context->backend_context); - } - else if (operation == FLB_CRYPTO_OPERATION_ENCRYPT) { - result = EVP_PKEY_encrypt_init(context->backend_context); - } - else if (operation == FLB_CRYPTO_OPERATION_DECRYPT) { - result = EVP_PKEY_decrypt_init(context->backend_context); - } - - if (result == 1) { - result = EVP_PKEY_CTX_set_rsa_padding(context->backend_context, - context->padding_type); - - if (result > 0) { - if (context->digest_algorithm != NULL) { - result = EVP_PKEY_CTX_set_signature_md(context->backend_context, - context->digest_algorithm); - } - } - - if (result > 0) { - result = 1; - } - } - - if (result != 1) { - context->last_error = ERR_get_error(); - - return FLB_CRYPTO_BACKEND_ERROR; - } - - context->last_operation = operation; - } - else if (context->last_operation != operation) { - return FLB_CRYPTO_INVALID_STATE; - } - - if (operation == FLB_CRYPTO_OPERATION_SIGN) { - result = EVP_PKEY_sign(context->backend_context, - output_buffer, output_length, - input_buffer, input_length); - } - else if(operation == FLB_CRYPTO_OPERATION_ENCRYPT) { - result = EVP_PKEY_encrypt(context->backend_context, - output_buffer, output_length, - input_buffer, input_length); - } - else if(operation == FLB_CRYPTO_OPERATION_DECRYPT) { - result = EVP_PKEY_decrypt(context->backend_context, - output_buffer, output_length, - input_buffer, input_length); - } - - if (result != 1) { - context->last_error = ERR_get_error(); - - return FLB_CRYPTO_BACKEND_ERROR; - } - - return FLB_CRYPTO_SUCCESS; -} - -int flb_crypto_sign(struct flb_crypto *context, - unsigned char *input_buffer, - size_t input_length, - unsigned char *output_buffer, - size_t *output_length) -{ - return flb_crypto_transform(context, - FLB_CRYPTO_OPERATION_SIGN, - input_buffer, - input_length, - output_buffer, - output_length); -} - -int flb_crypto_encrypt(struct flb_crypto *context, - unsigned char *input_buffer, - size_t input_length, - unsigned char *output_buffer, - size_t *output_length) -{ - return flb_crypto_transform(context, - FLB_CRYPTO_OPERATION_ENCRYPT, - input_buffer, - input_length, - output_buffer, - output_length); -} - -int flb_crypto_decrypt(struct flb_crypto *context, - unsigned char *input_buffer, - size_t input_length, - unsigned char *output_buffer, - size_t *output_length) -{ - return flb_crypto_transform(context, - FLB_CRYPTO_OPERATION_DECRYPT, - input_buffer, - input_length, - output_buffer, - output_length); -} - -int flb_crypto_sign_simple(int key_type, - int padding_type, - int digest_algorithm, - unsigned char *key, - size_t key_length, - unsigned char *input_buffer, - size_t input_length, - unsigned char *output_buffer, - size_t *output_length) -{ - struct flb_crypto context; - int result; - - result = flb_crypto_init(&context, - padding_type, - digest_algorithm, - key_type, - key, - key_length); - - if (result == FLB_CRYPTO_SUCCESS) { - result = flb_crypto_sign(&context, - input_buffer, input_length, - output_buffer, output_length); - - flb_crypto_cleanup(&context); - } - - return result; -} - -int flb_crypto_encrypt_simple(int padding_type, - unsigned char *key, - size_t key_length, - unsigned char *input_buffer, - size_t input_length, - unsigned char *output_buffer, - size_t *output_length) -{ - struct flb_crypto context; - int result; - - result = flb_crypto_init(&context, - padding_type, - FLB_HASH_NONE, - FLB_CRYPTO_PUBLIC_KEY, - key, - key_length); - - if (result == FLB_CRYPTO_SUCCESS) { - result = flb_crypto_encrypt(&context, - input_buffer, input_length, - output_buffer, output_length); - - - flb_crypto_cleanup(&context); - } - - return result; -} - -int flb_crypto_decrypt_simple(int padding_type, - unsigned char *key, - size_t key_length, - unsigned char *input_buffer, - size_t input_length, - unsigned char *output_buffer, - size_t *output_length) -{ - struct flb_crypto context; - int result; - - result = flb_crypto_init(&context, - padding_type, - FLB_HASH_NONE, - FLB_CRYPTO_PRIVATE_KEY, - key, - key_length); - - if (result == FLB_CRYPTO_SUCCESS) { - result = flb_crypto_decrypt(&context, - input_buffer, input_length, - output_buffer, output_length); - - flb_crypto_cleanup(&context); - } - - return result; -} - - - diff --git a/fluent-bit/src/flb_csv.c b/fluent-bit/src/flb_csv.c deleted file mode 100644 index 5ba38574e..000000000 --- a/fluent-bit/src/flb_csv.c +++ /dev/null @@ -1,313 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include <stddef.h> -#include <stdio.h> -#include <string.h> -#include <sys/types.h> -#define _GNU_SOURCE -#include <time.h> - -#include <fluent-bit/flb_mem.h> -#include <fluent-bit/flb_sds.h> -#include <fluent-bit/flb_str.h> - -#include <fluent-bit/flb_csv.h> - - -enum { - FLB_CSV_STATE_INITIAL = 0, - FLB_CSV_STATE_STARTED_SIMPLE, - FLB_CSV_STATE_STARTED_DQUOTE, - FLB_CSV_STATE_FOUND_DQUOTE, - FLB_CSV_STATE_FOUND_CR -}; - -static void reset_state(struct flb_csv_state *state) -{ - state->start = 0; - state->length = 0; - state->has_dquote = false; - state->field_parsed = false; - state->state = FLB_CSV_STATE_INITIAL; - flb_sds_len_set(state->escape_buffer, 0); - flb_sds_len_set(state->buffered_data, 0); - state->offset = 0; -} - -static int invoke_field_callback(struct flb_csv_state *state, const char *buf, size_t bufsize) -{ - size_t escpos; - size_t bufpos; - size_t bufend; - - if (!state->has_dquote) { - /* simple case, since there's no double quotes, no escaping needs - * to be done */ - state->field_callback(state->data, buf + state->start, state->length); - return 0; - } - /* ensure there's enough space in the escape buffer */ - if (flb_sds_alloc(state->escape_buffer) < state->length) { - state->escape_buffer = flb_sds_increase( - state->escape_buffer, state->length); - if (!state->escape_buffer) { - return FLB_CSV_ALLOC_FAILED; - } - } - - escpos = 0; - bufpos = state->start; - bufend = bufpos + state->length; - while (bufpos < bufend) { - if (buf[bufpos] == '"') { - /* escape double quote */ - bufpos++; - } - state->escape_buffer[escpos++] = buf[bufpos++]; - } - state->escape_buffer[escpos] = 0; - flb_sds_len_set(state->escape_buffer, escpos); - state->field_callback(state->data, state->escape_buffer, escpos); - return 0; -} - -static int parse_simple(struct flb_csv_state *state, const char *buf, size_t bufsize) -{ - char c; - - for (;;) { - if (state->offset >= bufsize) { - return FLB_CSV_EOF; - } - c = buf[state->offset]; - if (c == ',' || c == '\n' || c == '\r') { - /* end of field */ - break; - } - state->offset++; - } - - state->length = state->offset - state->start; - return 0; -} - -static int parse_quoted(struct flb_csv_state *state, const char *buf, size_t bufsize) -{ - char c; - - for (;;) { - if (state->offset >= bufsize) { - return FLB_CSV_EOF; - } - c = buf[state->offset]; - if (state->state == FLB_CSV_STATE_FOUND_DQUOTE) { - state->state = FLB_CSV_STATE_STARTED_DQUOTE; - if (c == '"') { - /* dquote inside field, skip but set flag so we can properly escape later */ - state->has_dquote = true; - } - else { - /* end of field */ - break; - } - } - else if (c == '"') { - state->state = FLB_CSV_STATE_FOUND_DQUOTE; - } - state->offset++; - } - - /* subtract 1 to remove ending double quote */ - state->length = state->offset - state->start - 1; - return 0; -} - -static int parse_csv_field(struct flb_csv_state *state, const char *data, size_t len) -{ - int ret; - const char *buf; - size_t bufsize; - bool buffered = false; - - buf = data; - bufsize = len; - - if (state->state == FLB_CSV_STATE_INITIAL) { - if (data[state->offset] == '"') { - /* advance past opening quote */ - state->offset++; - state->state = FLB_CSV_STATE_STARTED_DQUOTE; - } - else { - state->state = FLB_CSV_STATE_STARTED_SIMPLE; - } - state->start = state->offset; - } - else if (state->field_callback) { - state->buffered_data = flb_sds_cat(state->buffered_data, data, len); - if (!state->buffered_data) { - return FLB_CSV_ALLOC_FAILED; - } - buf = state->buffered_data; - bufsize = flb_sds_len(state->buffered_data); - buffered = true; - } - - switch (state->state) { - case FLB_CSV_STATE_STARTED_SIMPLE: - ret = parse_simple(state, buf, bufsize); - break; - case FLB_CSV_STATE_STARTED_DQUOTE: - case FLB_CSV_STATE_FOUND_DQUOTE: - ret = parse_quoted(state, buf, bufsize); - break; - default: - return FLB_CSV_INVALID_STATE; - } - - if (ret) { - if (!buffered && ret == FLB_CSV_EOF) { - /* not finished, we need to save data in the buffer */ - state->buffered_data = flb_sds_cat(state->buffered_data, data, len); - if (!state->buffered_data) { - return FLB_CSV_ALLOC_FAILED; - } - } - return ret; - } - - if (state->field_callback) { - ret = invoke_field_callback(state, buf, bufsize); - if (ret) { - return ret; - } - } - - return ret; -} - -void flb_csv_init(struct flb_csv_state *state, - flb_csv_field_parsed_callback field_callback, - void *data) -{ - state->buffered_data = flb_sds_create(""); - state->escape_buffer = flb_sds_create(""); - state->field_callback = field_callback; - state->data = data; - state->field_count = 0; - reset_state(state); -} - -int flb_csv_parse_record(struct flb_csv_state *state, - char **bufptr, - size_t *buflen, - size_t *field_count) -{ - char c; - int ret; - size_t initial_offset; - size_t advanced; - - for (;;) { - if (!(*buflen)) { - return FLB_CSV_EOF; - } - c = **bufptr; - if (state->state == FLB_CSV_STATE_INITIAL) { - if (c == '\r') { - state->state = FLB_CSV_STATE_FOUND_CR; - (*bufptr)++; - (*buflen)--; - continue; - } - else if (c == '\n') { - /* accept single linefeed as record terminator, even - * though the spec says to look for \r\n */ - (*bufptr)++; - (*buflen)--; - break; - } - else if (c == ',') { - (*bufptr)++; - (*buflen)--; - if (!state->field_parsed) { - state->field_count++; - if (state->field_callback) { - /* empty field, but we need to invoke the callback anyway */ - state->field_callback(state->data, "", 0); - } - } - state->field_parsed = false; - continue; - } - } - else if (state->state == FLB_CSV_STATE_FOUND_CR) { - state->state = FLB_CSV_STATE_INITIAL; - if (c == '\n') { - /* if the character following \r is \n, consume it */ - (*bufptr)++; - (*buflen)--; - } - /* in any case, accept lone \r as record separator */ - break; - } - - initial_offset = state->offset; - - ret = parse_csv_field(state, *bufptr, *buflen); - - advanced = state->offset - initial_offset; - *bufptr += advanced; - *buflen -= advanced; - - if (ret) { - if (!state->field_callback) { - /* when no field callback is set, we shouldn't keep - * offset state between calls since no data will be buffered */ - state->offset = 0; - } - return ret; - } - - /* when a field is fully parsed, we can reset state */ - reset_state(state); - /* set this flag so we can properly handle empty fields at the start - * of the loop */ - state->field_parsed = true; - state->field_count++; - } - - if (!state->field_parsed) { - state->field_count++; - if (state->field_callback) { - /* empty field, but we need to invoke the callback anyway */ - state->field_callback(state->data, "", 0); - } - } - state->field_parsed = false; - *field_count = state->field_count; - state->field_count = 0; - return FLB_CSV_SUCCESS; -} - -void flb_csv_destroy(struct flb_csv_state *state) -{ - flb_sds_destroy(state->buffered_data); - flb_sds_destroy(state->escape_buffer); -} diff --git a/fluent-bit/src/flb_custom.c b/fluent-bit/src/flb_custom.c deleted file mode 100644 index 8279bb65d..000000000 --- a/fluent-bit/src/flb_custom.c +++ /dev/null @@ -1,314 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <fluent-bit/flb_config.h> -#include <fluent-bit/flb_custom.h> -#include <fluent-bit/flb_str.h> -#include <fluent-bit/flb_env.h> -#include <fluent-bit/flb_router.h> -#include <fluent-bit/flb_mp.h> -#include <fluent-bit/flb_kv.h> -#include <fluent-bit/flb_pack.h> -#include <fluent-bit/flb_metrics.h> -#include <fluent-bit/flb_utils.h> -#include <chunkio/chunkio.h> - -static inline int instance_id(struct flb_config *config) -{ - struct flb_custom_instance *entry; - - if (mk_list_size(&config->customs) == 0) { - return 0; - } - - entry = mk_list_entry_last(&config->customs, struct flb_custom_instance, - _head); - return (entry->id + 1); -} - -static inline int prop_key_check(const char *key, const char *kv, int k_len) -{ - int len; - - len = strlen(key); - if (strncasecmp(key, kv, k_len) == 0 && len == k_len) { - return 0; - } - - return -1; -} - -int flb_custom_set_property(struct flb_custom_instance *ins, - const char *k, const char *v) -{ - int len; - int ret; - flb_sds_t tmp; - struct flb_kv *kv; - - len = strlen(k); - tmp = flb_env_var_translate(ins->config->env, v); - if (!tmp) { - return -1; - } - - if (prop_key_check("alias", k, len) == 0 && tmp) { - flb_utils_set_plugin_string_property("alias", &ins->alias, tmp); - } - else if (prop_key_check("log_level", k, len) == 0 && tmp) { - ret = flb_log_get_level_str(tmp); - flb_sds_destroy(tmp); - if (ret == -1) { - return -1; - } - ins->log_level = ret; - } - else { - /* - * Create the property, we don't pass the value since we will - * map it directly to avoid an extra memory allocation. - */ - kv = flb_kv_item_create(&ins->properties, (char *) k, NULL); - if (!kv) { - if (tmp) { - flb_sds_destroy(tmp); - } - return -1; - } - kv->val = tmp; - } - - return 0; -} - -const char *flb_custom_get_property(const char *key, - struct flb_custom_instance *ins) -{ - return flb_kv_get_key_value(key, &ins->properties); -} - -void flb_custom_instance_exit(struct flb_custom_instance *ins, - struct flb_config *config) -{ - struct flb_custom_plugin *p; - - p = ins->p; - if (p->cb_exit && ins->context) { - p->cb_exit(ins->context, config); - } -} - -/* Invoke exit call for the custom plugin */ -void flb_custom_exit(struct flb_config *config) -{ - struct mk_list *tmp; - struct mk_list *head; - struct flb_custom_instance *ins; - struct flb_custom_plugin *p; - - mk_list_foreach_safe(head, tmp, &config->customs) { - ins = mk_list_entry(head, struct flb_custom_instance, _head); - p = ins->p; - if (!p) { - continue; - } - flb_custom_instance_exit(ins, config); - flb_custom_instance_destroy(ins); - } -} - -struct flb_custom_instance *flb_custom_new(struct flb_config *config, - const char *custom, void *data) -{ - int id; - struct mk_list *head; - struct flb_custom_plugin *plugin; - struct flb_custom_instance *instance = NULL; - - if (!custom) { - return NULL; - } - - mk_list_foreach(head, &config->custom_plugins) { - plugin = mk_list_entry(head, struct flb_custom_plugin, _head); - if (strcmp(plugin->name, custom) == 0) { - break; - } - plugin = NULL; - } - - if (!plugin) { - return NULL; - } - - instance = flb_calloc(1, sizeof(struct flb_custom_instance)); - if (!instance) { - flb_errno(); - return NULL; - } - instance->config = config; - - /* Get an ID */ - id = instance_id(config); - - /* format name (with instance id) */ - snprintf(instance->name, sizeof(instance->name) - 1, - "%s.%i", plugin->name, id); - - instance->id = id; - instance->alias = NULL; - instance->p = plugin; - instance->data = data; - instance->log_level = -1; - - mk_list_init(&instance->properties); - mk_list_add(&instance->_head, &config->customs); - - return instance; -} - -/* Return an instance name or alias */ -const char *flb_custom_name(struct flb_custom_instance *ins) -{ - if (ins->alias) { - return ins->alias; - } - - return ins->name; -} - -int flb_custom_plugin_property_check(struct flb_custom_instance *ins, - struct flb_config *config) -{ - int ret = 0; - struct mk_list *config_map; - struct flb_custom_plugin *p = ins->p; - - if (p->config_map) { - /* - * Create a dynamic version of the configmap that will be used by the specific - * instance in question. - */ - config_map = flb_config_map_create(config, p->config_map); - if (!config_map) { - flb_error("[custom] error loading config map for '%s' plugin", - p->name); - return -1; - } - ins->config_map = config_map; - - /* Validate incoming properties against config map */ - ret = flb_config_map_properties_check(ins->p->name, - &ins->properties, ins->config_map); - if (ret == -1) { - if (config->program_name) { - flb_helper("try the command: %s -F %s -h\n", - config->program_name, ins->p->name); - } - return -1; - } - } - - return 0; -} - -/* Initialize all custom plugins */ -int flb_custom_init_all(struct flb_config *config) -{ - int ret; - struct mk_list *tmp; - struct mk_list *head; - struct flb_custom_plugin *p; - struct flb_custom_instance *ins; - - /* Iterate all active custom instance plugins */ - mk_list_foreach_safe(head, tmp, &config->customs) { - ins = mk_list_entry(head, struct flb_custom_instance, _head); - - if (ins->log_level == -1) { - ins->log_level = config->log->level; - } - - p = ins->p; - -#ifdef FLB_HAVE_METRICS - /* CMetrics */ - ins->cmt = cmt_create(); - if (!ins->cmt) { - flb_error("[custom] could not create cmetrics context: %s", - flb_custom_name(ins)); - return -1; - } -#endif - - /* - * Before to call the initialization callback, make sure that the received - * configuration parameters are valid if the plugin is registering a config map. - */ - if (flb_custom_plugin_property_check(ins, config) == -1) { - flb_custom_instance_destroy(ins); - return -1; - } - - /* Initialize the input */ - if (p->cb_init) { - ret = p->cb_init(ins, config, ins->data); - if (ret != 0) { - flb_error("Failed initialize custom %s", ins->name); - flb_custom_instance_destroy(ins); - return -1; - } - } - } - - return 0; -} - -void flb_custom_instance_destroy(struct flb_custom_instance *ins) -{ - if (!ins) { - return; - } - - /* destroy config map */ - if (ins->config_map) { - flb_config_map_destroy(ins->config_map); - } - - /* release properties */ - flb_kv_release(&ins->properties); - - if (ins->alias) { - flb_sds_destroy(ins->alias); - } - -#ifdef FLB_HAVE_METRICS - if (ins->cmt) { - cmt_destroy(ins->cmt); - } -#endif - - mk_list_del(&ins->_head); - flb_free(ins); -} - -void flb_custom_set_context(struct flb_custom_instance *ins, void *context) -{ - ins->context = context; -} diff --git a/fluent-bit/src/flb_dlfcn_win32.c b/fluent-bit/src/flb_dlfcn_win32.c deleted file mode 100644 index 1ca398f51..000000000 --- a/fluent-bit/src/flb_dlfcn_win32.c +++ /dev/null @@ -1,101 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <fluent-bit/flb_compat.h> - -static CHAR dlerrorbuf[512]; -static BOOL has_error_message = FALSE; - -static void store_error(void) -{ - DWORD err = GetLastError(); - if (err == NO_ERROR) { - return; - } - - if (!FormatMessageA(FORMAT_MESSAGE_FROM_SYSTEM | - FORMAT_MESSAGE_IGNORE_INSERTS, - NULL, - err, - MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), - dlerrorbuf, - _countof(dlerrorbuf), NULL)) - dlerrorbuf[0] = '\0'; - - has_error_message = TRUE; -} - -__declspec(noinline) -void *dlopen(const char *filename, int _flag) -{ - HMODULE handle; - - handle = LoadLibrary(filename); - if (handle == NULL) { - store_error(); - return NULL; - } - return (void *)handle; -} - -char *dlerror(void) -{ - char *errorptr = dlerrorbuf; - - /* POSIX requests that the second consective dlerror() calling should - * be return NULL.*/ - if (!has_error_message) - { - return NULL; - } - - has_error_message = FALSE; - - return errorptr; -} - -__declspec(noinline) -void *dlsym(void *handle, const char *name) -{ - FARPROC *symbol; - symbol = NULL; - - symbol = GetProcAddress((HMODULE) handle, name); - if (symbol == NULL) { - store_error(); - return NULL; - } - - return (void *)symbol; -} - -int dlclose(void *handle) -{ - BOOL result; - - result = FreeLibrary((HMODULE) handle); - if (!result) - store_error(); - - /* dlcose(3) returns 0 on success, and nonzero on error. */ - /* FreeLibrary returns nonzero on success, and 0 on error. */ - /* ref: - * https://docs.microsoft.com/en-us/windows/desktop/api/libloaderapi/nf-libloaderapi-freelibrary */ - return !result; -} diff --git a/fluent-bit/src/flb_downstream.c b/fluent-bit/src/flb_downstream.c deleted file mode 100644 index e4b7e19ef..000000000 --- a/fluent-bit/src/flb_downstream.c +++ /dev/null @@ -1,514 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <monkey/mk_core.h> -#include <fluent-bit/flb_info.h> -#include <fluent-bit/flb_mem.h> -#include <fluent-bit/flb_kv.h> -#include <fluent-bit/flb_io.h> -#include <fluent-bit/flb_str.h> -#include <fluent-bit/flb_slist.h> -#include <fluent-bit/flb_utils.h> -#include <fluent-bit/flb_engine.h> -#include <fluent-bit/tls/flb_tls.h> -#include <fluent-bit/flb_downstream.h> -#include <fluent-bit/flb_connection.h> -#include <fluent-bit/flb_config_map.h> -#include <fluent-bit/flb_thread_storage.h> - -/* Config map for Downstream networking setup */ -struct flb_config_map downstream_net[] = { - { - FLB_CONFIG_MAP_TIME, "net.io_timeout", "0s", - 0, FLB_TRUE, offsetof(struct flb_net_setup, io_timeout), - "Set maximum time a connection can stay idle" - }, - - { - FLB_CONFIG_MAP_TIME, "net.accept_timeout", "10s", - 0, FLB_TRUE, offsetof(struct flb_net_setup, accept_timeout), - "Set maximum time allowed to establish an incoming connection, this time " - "includes the TLS handshake" - }, - - { - FLB_CONFIG_MAP_BOOL, "net.accept_timeout_log_error", "true", - 0, FLB_TRUE, offsetof(struct flb_net_setup, accept_timeout_log_error), - "On client accept timeout, specify if it should log an error. When " - "disabled, the timeout is logged as a debug message" - }, - - /* EOF */ - {0} -}; - -/* Enable thread-safe mode for downstream connection */ -void flb_downstream_thread_safe(struct flb_downstream *stream) -{ - flb_stream_enable_thread_safety(&stream->base); -} - -struct mk_list *flb_downstream_get_config_map(struct flb_config *config) -{ - return flb_config_map_create(config, downstream_net); -} - -/* Initialize any downstream environment context */ -void flb_downstream_init() -{ - /* There's nothing to do here yet */ -} - -int flb_downstream_setup(struct flb_downstream *stream, - int transport, int flags, - const char *host, - unsigned short int port, - struct flb_tls *tls, - struct flb_config *config, - struct flb_net_setup *net_setup) -{ - char port_string[8]; - - flb_stream_setup(&stream->base, - FLB_DOWNSTREAM, - transport, - flags, - tls, - config, - net_setup); - - stream->server_fd = FLB_INVALID_SOCKET; - stream->host = flb_strdup(host); - stream->port = port; - - if (stream->host == NULL) { - return -1; - } - - mk_list_init(&stream->busy_queue); - mk_list_init(&stream->destroy_queue); - - snprintf(port_string, sizeof(port_string), "%u", port); - - if (transport == FLB_TRANSPORT_TCP) { - stream->server_fd = flb_net_server(port_string, host); - } - else if (transport == FLB_TRANSPORT_UDP) { - stream->server_fd = flb_net_server_udp(port_string, host); - } - else if (transport == FLB_TRANSPORT_UNIX_STREAM) { - stream->server_fd = flb_net_server_unix(host, - FLB_TRUE, - FLB_NETWORK_DEFAULT_BACKLOG_SIZE); - } - else if (transport == FLB_TRANSPORT_UNIX_DGRAM) { - stream->server_fd = flb_net_server_unix(host, - FLB_FALSE, - FLB_NETWORK_DEFAULT_BACKLOG_SIZE); - } - - if (stream->server_fd != -1) { - flb_debug("[downstream] listening on %s:%s", host, port_string); - } - else { - flb_error("[downstream] could not bind address %s:%s. Aborting", - host, port_string); - - return -2; - } - - mk_list_add(&stream->base._head, &config->downstreams); - - return 0; -} - -/* Creates a new downstream context */ -struct flb_downstream *flb_downstream_create(int transport, int flags, - const char *host, - unsigned short int port, - struct flb_tls *tls, - struct flb_config *config, - struct flb_net_setup *net_setup) -{ - struct flb_downstream *stream; - int result; - - stream = flb_calloc(1, sizeof(struct flb_downstream)); - - if (stream == NULL) { - flb_errno(); - } - else { - stream->base.dynamically_allocated = FLB_TRUE; - - result = flb_downstream_setup(stream, - transport, flags, - host, port, - tls, - config, - net_setup); - - if (result != 0) { - flb_downstream_destroy(stream); - - stream = NULL; - } - } - - return stream; -} - -/* - * This function moves the 'downstream connection' into the queue to be - * destroyed. Note that the caller is responsible to validate and check - * required mutex if this is being used in multi-worker mode. - */ -static int prepare_destroy_conn(struct flb_connection *connection) -{ - flb_trace("[downstream] destroy connection #%i to %s", - connection->fd, flb_connection_get_remote_address(connection)); - - if (MK_EVENT_IS_REGISTERED((&connection->event))) { - mk_event_del(connection->evl, &connection->event); - } - - /* This should be != -1 to cover those use cases where stdin, stdout - * and stderr are closed. - */ - - if (connection->fd != FLB_INVALID_SOCKET) { - flb_socket_close(connection->fd); - - connection->fd = FLB_INVALID_SOCKET; - connection->event.fd = FLB_INVALID_SOCKET; - } - - /* remove connection from the queue */ - mk_list_del(&connection->_head); - - /* Add node to destroy queue */ - mk_list_add(&connection->_head, &connection->downstream->destroy_queue); - - /* - * note: the connection context is destroyed by the engine once all events - * have been processed. - */ - return 0; -} - -/* 'safe' version of prepare_destroy_conn. It set locks if necessary */ -static inline int prepare_destroy_conn_safe(struct flb_connection *connection) -{ - int result; - - /* This used to not wait for the lock in thread safe mode but it makes - * no sense so I'm changing it (08/28/22) leo - */ - - flb_stream_acquire_lock(connection->stream, FLB_TRUE); - - result = prepare_destroy_conn(connection); - - flb_stream_release_lock(connection->stream); - - return result; -} - -static int destroy_conn(struct flb_connection *connection) -{ - /* Delay the destruction of busy connections */ - if (connection->busy_flag) { - return 0; - } - - if (connection->tls_session != NULL) { - flb_tls_session_destroy(connection->tls_session); - } - - mk_list_del(&connection->_head); - - flb_connection_destroy(connection); - - return 0; -} - -struct flb_connection *flb_downstream_conn_get(struct flb_downstream *stream) -{ - flb_sockfd_t connection_fd; - struct flb_connection *connection; - int transport; - struct flb_coro *coroutine; - int result; - - transport = stream->base.transport; - - if (transport == FLB_TRANSPORT_UDP || - transport == FLB_TRANSPORT_UNIX_DGRAM ) { - if (stream->dgram_connection != NULL) { - return stream->dgram_connection; - } - - connection_fd = stream->server_fd; - } - else { - connection_fd = FLB_INVALID_SOCKET; - } - - if (flb_downstream_is_async(stream)) { - coroutine = flb_coro_get(); - } - else { - coroutine = NULL; - } - - connection = flb_connection_create(connection_fd, - FLB_DOWNSTREAM_CONNECTION, - (void *) stream, - flb_engine_evl_get(), - coroutine); - - if (connection == NULL) { - return NULL; - } - - connection->busy_flag = FLB_TRUE; - - flb_stream_acquire_lock(&stream->base, FLB_TRUE); - - /* Link new connection to the busy queue */ - mk_list_add(&connection->_head, &stream->busy_queue); - - flb_stream_release_lock(&stream->base); - - if (transport != FLB_TRANSPORT_UDP && - transport != FLB_TRANSPORT_UNIX_DGRAM ) { - flb_connection_reset_connection_timeout(connection); - - result = flb_io_net_accept(connection, coroutine); - - if (result != 0) { - flb_connection_reset_connection_timeout(connection); - - flb_debug("[downstream] connection #%i failed", - connection->fd); - - prepare_destroy_conn_safe(connection); - - connection->busy_flag = FLB_FALSE; - - return NULL; - } - - flb_connection_unset_connection_timeout(connection); - } - - connection->busy_flag = FLB_FALSE; - - flb_connection_reset_io_timeout(connection); - - if (transport == FLB_TRANSPORT_UDP || - transport == FLB_TRANSPORT_UNIX_DGRAM) { - if (stream->dgram_connection == NULL) { - stream->dgram_connection = connection; - } - } - - return connection; -} - -void flb_downstream_destroy(struct flb_downstream *stream) -{ - struct flb_connection *connection; - struct mk_list *head; - struct mk_list *tmp; - - if (stream != NULL) { - mk_list_foreach_safe(head, tmp, &stream->busy_queue) { - connection = mk_list_entry(head, struct flb_connection, _head); - - prepare_destroy_conn(connection); - } - - mk_list_foreach_safe(head, tmp, &stream->destroy_queue) { - connection = mk_list_entry(head, struct flb_connection, _head); - - destroy_conn(connection); - } - - /* If the simulated UDP connection reference is set then - * it means that connection was already cleaned up by the - * preceding code which means server_fd holds a socket - * reference that has already been closed and we need to - * honor that. - */ - - if (stream->dgram_connection != NULL) { - stream->dgram_connection = NULL; - stream->server_fd = FLB_INVALID_SOCKET; - } - - if (stream->host != NULL) { - flb_free(stream->host); - } - - if (stream->server_fd != FLB_INVALID_SOCKET) { - flb_socket_close(stream->server_fd); - } - - if (mk_list_entry_orphan(&stream->base._head) == 0) { - mk_list_del(&stream->base._head); - } - - if (stream->base.dynamically_allocated) { - flb_free(stream); - } - } -} - -int flb_downstream_conn_release(struct flb_connection *connection) -{ - return prepare_destroy_conn_safe(connection); -} - -int flb_downstream_conn_timeouts(struct mk_list *list) -{ - int elapsed_time; - struct flb_connection *connection; - const char *reason; - struct flb_downstream *stream; - struct mk_list *s_head; - struct mk_list *head; - int drop; - int inject; - struct mk_list *tmp; - time_t now; - - now = time(NULL); - - /* Iterate all downstream contexts */ - mk_list_foreach(head, list) { - stream = mk_list_entry(head, struct flb_downstream, base._head); - - if (stream->base.transport == FLB_TRANSPORT_UDP) { - continue; - } - - flb_stream_acquire_lock(&stream->base, FLB_TRUE); - - /* Iterate every busy connection */ - mk_list_foreach_safe(s_head, tmp, &stream->busy_queue) { - connection = mk_list_entry(s_head, struct flb_connection, _head); - - drop = FLB_FALSE; - - /* Connect timeouts */ - if (connection->net->connect_timeout > 0 && - connection->ts_connect_timeout > 0 && - connection->ts_connect_timeout <= now) { - drop = FLB_TRUE; - reason = "connection timeout"; - elapsed_time = connection->net->accept_timeout; - } - else if (connection->net->io_timeout > 0 && - connection->ts_io_timeout > 0 && - connection->ts_io_timeout <= now) { - drop = FLB_TRUE; - reason = "IO timeout"; - elapsed_time = connection->net->io_timeout; - } - - if (drop) { - if (!flb_downstream_is_shutting_down(stream)) { - if (connection->net->accept_timeout_log_error) { - flb_error("[downstream] connection #%i from %s timed " - "out after %i seconds (%s)", - connection->fd, - connection->user_friendly_remote_host, - elapsed_time, - reason); - } - else { - flb_debug("[downstream] connection #%i from %s timed " - "out after %i seconds (%s)", - connection->fd, - connection->user_friendly_remote_host, - elapsed_time, - reason); - } - } - - inject = FLB_FALSE; - if (connection->event.status != MK_EVENT_NONE) { - inject = FLB_TRUE; - } - connection->net_error = ETIMEDOUT; - prepare_destroy_conn(connection); - if (inject == FLB_TRUE) { - mk_event_inject(connection->evl, - &connection->event, - connection->event.mask, - FLB_TRUE); - } - } - } - - flb_stream_release_lock(&stream->base); - } - - return 0; -} - -int flb_downstream_conn_pending_destroy(struct flb_downstream *stream) -{ - struct flb_connection *connection; - struct mk_list *head; - struct mk_list *tmp; - - flb_stream_acquire_lock(&stream->base, FLB_TRUE); - - mk_list_foreach_safe(head, tmp, &stream->destroy_queue) { - connection = mk_list_entry(head, struct flb_connection, _head); - - destroy_conn(connection); - } - - flb_stream_release_lock(&stream->base); - - return 0; -} - -int flb_downstream_conn_pending_destroy_list(struct mk_list *list) -{ - struct flb_downstream *stream; - struct mk_list *head; - - /* Iterate all downstream contexts */ - mk_list_foreach(head, list) { - stream = mk_list_entry(head, struct flb_downstream, base._head); - - flb_downstream_conn_pending_destroy(stream); - } - - return 0; -} - -int flb_downstream_is_async(struct flb_downstream *stream) -{ - return flb_stream_is_async(&stream->base); -} diff --git a/fluent-bit/src/flb_dump.c b/fluent-bit/src/flb_dump.c deleted file mode 100644 index fe1d93a59..000000000 --- a/fluent-bit/src/flb_dump.c +++ /dev/null @@ -1,260 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <fluent-bit/flb_info.h> -#include <fluent-bit/flb_input.h> -#include <fluent-bit/flb_input_chunk.h> -#include <fluent-bit/flb_task.h> -#include <fluent-bit/flb_config.h> -#include <fluent-bit/flb_storage.h> -#include <fluent-bit/flb_utils.h> -#include <fluent-bit/flb_event.h> - -#ifdef FLB_DUMP_STACKTRACE -#include <fluent-bit/flb_stacktrace.h> -#endif - -#include <stdio.h> -#include <time.h> - -/* - * Input Chunks - * ============ - * Every input plugin instance has it own Chunk I/O stream. The stream is used to - * associate data from the specific origin. - * - * This dump prints out information about current status of chunks registered by - * the input plugin interface and resources usage. - */ -static void dump_input_chunks(struct flb_config *ctx) -{ - /* general */ - int ret; - ssize_t size; - - /* tasks */ - int task_new; - int task_running; - - /* chunks */ - int up; - int down; - int busy; - int busy_size_err; - ssize_t busy_size; - char tmp[32]; - - struct mk_list *head; - struct mk_list *h_chunks; - struct mk_list *h_task; - struct flb_input_instance *i; - struct flb_input_chunk *ic; - struct flb_task *task; - - fprintf(stderr, "\n===== Input =====\n"); - - mk_list_foreach(head, &ctx->inputs) { - i = mk_list_entry(head, struct flb_input_instance, _head); - fprintf(stderr, "%s (%s)\n", flb_input_name(i), i->p->name); - - fprintf(stderr, "│\n"); - fprintf(stderr, "├─ status\n"); - - /* Overlimit checks */ - ret = FLB_FALSE; - if (i->mem_buf_limit > 0) { - if (i->mem_chunks_size >= i->mem_buf_limit) { - ret = FLB_TRUE; - } - } - fprintf(stderr, "│ └─ overlimit : %s\n", - ret ? "yes" : "no"); - - /* Current memory size used based on last ingestion */ - flb_utils_bytes_to_human_readable_size(i->mem_chunks_size, - tmp, sizeof(tmp) - 1); - fprintf(stderr, "│ ├─ mem size : %s (%lu bytes)\n", - tmp, i->mem_chunks_size); - - /* Mem buf limit set */ - flb_utils_bytes_to_human_readable_size(i->mem_buf_limit, - tmp, sizeof(tmp) - 1); - fprintf(stderr, "│ └─ mem limit : %s (%lu bytes)\n", - tmp, i->mem_buf_limit); - - /* - * Tasks - * ===== - * Upon flush time, the engine look for 'chunks' ready to be flushed. - * For each one, it creates a Task, this task can be routed and - * referenced by different output destinations. - * - * For short: every task is a chunk. But it's a different structure - * handled by the engine to coordinate the flush process. - */ - fprintf(stderr, "│\n"); - fprintf(stderr, "├─ tasks\n"); - fprintf(stderr, "│ ├─ total tasks : %i\n", mk_list_size(&i->tasks)); - - size = 0; - task_new = 0; - task_running = 0; - /* Iterate tasks and print a summary */ - mk_list_foreach(h_task, &i->tasks) { - task = mk_list_entry(h_task, struct flb_task, _head); - size += task->event_chunk->size; - if (task->status == FLB_TASK_NEW) { - task_new++; - } - else if (task->status == FLB_TASK_RUNNING) { - task_running++; - } - } - - flb_utils_bytes_to_human_readable_size(size, tmp, sizeof(tmp) - 1); - - fprintf(stderr, "│ ├─ new : %i\n", task_new); - fprintf(stderr, "│ ├─ running : %i\n", task_running); - fprintf(stderr, "│ └─ size : %s (%lu bytes)\n", tmp, size); - - /* - * Chunks - * ====== - * Input plugins ingest record into a 'chunk'. If the storage layer type - * for the instance is memory, all chunks are considered 'up' (meaning: - * up in memory), for filesystem based chunks they can be 'up' or 'down'. - * - * We avoid to have all of them 'up' at the same time since this can - * lead to a high memory consumption. When filesystem mode is used, some - * of them are 'down' and only get 'up' when they are going to be - * processed. - */ - fprintf(stderr, "│\n"); - fprintf(stderr, "└─ chunks\n"); - - /* Number of chunks registered */ - fprintf(stderr, " └─ total chunks : %i\n", mk_list_size(&i->chunks)); - - /* Busy chunks - * ----------- - * Chunks marked as 'busy' are 'locked' since they are in a 'flush' state. - * No more data can be appended to a busy chunk. - */ - busy = 0; - busy_size = 0; - busy_size_err = 0; - - /* up/down */ - up = 0; - down = 0; - - /* Iterate chunks for the input instance in question */ - mk_list_foreach(h_chunks, &i->chunks) { - ic = mk_list_entry(h_chunks, struct flb_input_chunk, _head); - if (ic->busy == FLB_TRUE) { - busy++; - size = cio_chunk_get_content_size(ic->chunk); - if (size >= 0) { - busy_size += size; - } - else { - busy_size_err++; - } - } - - if (cio_chunk_is_up(ic->chunk) == CIO_TRUE) { - up++; - } - else { - down++; - } - } - - fprintf(stderr, " ├─ up chunks : %i\n", up); - fprintf(stderr, " ├─ down chunks: %i\n", down); - flb_utils_bytes_to_human_readable_size(busy_size, tmp, sizeof(tmp) - 1); - - fprintf(stderr, " └─ busy chunks: %i\n", busy); - fprintf(stderr, " ├─ size : %s (%lu bytes)\n", tmp, busy_size); - fprintf(stderr, " └─ size err: %i\n", busy_size_err); - fprintf(stderr, "\n"); - } -} - -/* - * Storage - * ======= - * Dump Chunk I/O statistics, basic counters - */ -static void dump_storage(struct flb_config *ctx) -{ - struct cio_stats storage_st; - - fprintf(stderr, "\n===== Storage Layer =====\n"); - cio_stats_get(ctx->cio, &storage_st); - - fprintf(stderr, "total chunks : %i\n", storage_st.chunks_total); - fprintf(stderr, "├─ mem chunks : %i\n", storage_st.chunks_mem); - fprintf(stderr, "└─ fs chunks : %i\n", storage_st.chunks_fs); - fprintf(stderr, " ├─ up : %i\n", storage_st.chunks_fs_up); - fprintf(stderr, " └─ down : %i\n", storage_st.chunks_fs_down); -} - -void flb_dump(struct flb_config *ctx) -{ - time_t now; - struct tm *current; - - now = time(NULL); - current = localtime(&now); - - fprintf(stderr, - "[%i/%02i/%02i %02i:%02i:%02i] Fluent Bit Dump\n", - current->tm_year + 1900, - current->tm_mon + 1, - current->tm_mday, - current->tm_hour, - current->tm_min, - current->tm_sec); - - /* Stacktrace */ -#ifdef FLB_DUMP_STACKTRACE - /* - * Sorry, I had to disable the stacktrace as part of the dump - * since if backtrace_full() is called while Fluent Bit is - * inside a co-routine (output flush), it might crash. - * - * If we are in a co-routine likely we need a different libbacktrace - * context, but it's just a guess, not tested. - */ - //fprintf(stderr, "\n===== Stacktrace =====\n"); - //flb_stacktrace_print(); -#endif - - /* Input Plugins + Storage */ - dump_input_chunks(ctx); - - /* Storage Layer */ - dump_storage(ctx); - - /* Make sure to flush the stdout buffer in case output - * has been redirected to a file - */ - fflush(stderr); -} diff --git a/fluent-bit/src/flb_engine.c b/fluent-bit/src/flb_engine.c deleted file mode 100644 index 78be8d5ec..000000000 --- a/fluent-bit/src/flb_engine.c +++ /dev/null @@ -1,1124 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <math.h> -#include <stdio.h> -#include <stdlib.h> - -#include <monkey/mk_core.h> -#include <fluent-bit/flb_bucket_queue.h> -#include <fluent-bit/flb_event_loop.h> -#include <fluent-bit/flb_time.h> -#include <fluent-bit/flb_info.h> -#include <fluent-bit/flb_bits.h> - -#include <fluent-bit/flb_macros.h> -#include <fluent-bit/flb_pipe.h> -#include <fluent-bit/flb_custom.h> -#include <fluent-bit/flb_input.h> -#include <fluent-bit/flb_output.h> -#include <fluent-bit/flb_error.h> -#include <fluent-bit/flb_utils.h> -#include <fluent-bit/flb_config.h> -#include <fluent-bit/flb_engine.h> -#include <fluent-bit/flb_engine_dispatch.h> -#include <fluent-bit/flb_network.h> -#include <fluent-bit/flb_task.h> -#include <fluent-bit/flb_router.h> -#include <fluent-bit/flb_http_server.h> -#include <fluent-bit/flb_scheduler.h> -#include <fluent-bit/flb_parser.h> -#include <fluent-bit/flb_sosreport.h> -#include <fluent-bit/flb_storage.h> -#include <fluent-bit/flb_http_server.h> -#include <fluent-bit/flb_pack.h> -#include <fluent-bit/flb_metrics.h> -#include <fluent-bit/flb_version.h> -#include <fluent-bit/flb_upstream.h> -#include <fluent-bit/flb_downstream.h> -#include <fluent-bit/flb_ring_buffer.h> - -#ifdef FLB_HAVE_METRICS -#include <fluent-bit/flb_metrics_exporter.h> -#endif - -#ifdef FLB_HAVE_STREAM_PROCESSOR -#include <fluent-bit/stream_processor/flb_sp.h> -#endif - -#ifdef FLB_HAVE_AWS_ERROR_REPORTER -#include <fluent-bit/aws/flb_aws_error_reporter.h> - -extern struct flb_aws_error_reporter *error_reporter; -#endif - -#include <ctraces/ctr_version.h> - -static pthread_once_t local_thread_engine_evl_init = PTHREAD_ONCE_INIT; -FLB_TLS_DEFINE(struct mk_event_loop, flb_engine_evl); - -static void flb_engine_evl_init_private() -{ - FLB_TLS_INIT(flb_engine_evl); -} - -void flb_engine_evl_init() -{ - pthread_once(&local_thread_engine_evl_init, flb_engine_evl_init_private); -} - -struct mk_event_loop *flb_engine_evl_get() -{ - struct mk_event_loop *evl; - - evl = FLB_TLS_GET(flb_engine_evl); - return evl; -} - -void flb_engine_evl_set(struct mk_event_loop *evl) -{ - FLB_TLS_SET(flb_engine_evl, evl); -} - -int flb_engine_destroy_tasks(struct mk_list *tasks) -{ - int c = 0; - struct mk_list *tmp; - struct mk_list *head; - struct flb_task *task; - - mk_list_foreach_safe(head, tmp, tasks) { - task = mk_list_entry(head, struct flb_task, _head); - flb_task_destroy(task, FLB_FALSE); - c++; - } - - return c; -} - -void flb_engine_reschedule_retries(struct flb_config *config) -{ - int ret; - struct mk_list *head; - struct mk_list *t_head; - struct mk_list *rt_head; - struct mk_list *tmp_task; - struct mk_list *tmp_retry_task; - struct flb_task *task; - struct flb_input_instance *ins; - struct flb_task_retry *retry; - - /* Invalidate and reschedule all retry tasks to be retried immediately */ - mk_list_foreach(head, &config->inputs) { - ins = mk_list_entry(head, struct flb_input_instance, _head); - mk_list_foreach_safe(t_head, tmp_task, &ins->tasks) { - task = mk_list_entry(t_head, struct flb_task, _head); - mk_list_foreach_safe(rt_head, tmp_retry_task, &task->retries) { - retry = mk_list_entry(rt_head, struct flb_task_retry, _head); - flb_sched_request_invalidate(config, retry); - ret = flb_sched_retry_now(config, retry); - if (ret == -1) { - /* Can't do much here, just continue on */ - flb_warn("[engine] failed to immediately re-schedule retry=%p " - "for task %i. Err: %d", retry, task->id, flb_errno()); - } else { - flb_debug("[engine] re-scheduled retry=%p for task %i", - retry, task->id); - } - } - } - } -} - -int flb_engine_flush(struct flb_config *config, - struct flb_input_plugin *in_force) -{ - struct flb_input_instance *in; - struct flb_input_plugin *p; - struct mk_list *head; - - mk_list_foreach(head, &config->inputs) { - in = mk_list_entry(head, struct flb_input_instance, _head); - p = in->p; - - if (in_force != NULL && p != in_force) { - continue; - } - flb_engine_dispatch(0, in, config); - } - - return 0; -} - -/* Cleanup function that runs every 1.5 second */ -static void cb_engine_sched_timer(struct flb_config *ctx, void *data) -{ - (void) data; - - /* Upstream timeout handling */ - flb_upstream_conn_timeouts(&ctx->upstreams); - - /* Downstream timeout handling */ - flb_downstream_conn_timeouts(&ctx->downstreams); -} - -static inline int handle_input_event(flb_pipefd_t fd, uint64_t ts, - struct flb_config *config) -{ - int bytes; - uint32_t type; - uint32_t ins_id; - uint64_t val; - - bytes = flb_pipe_r(fd, &val, sizeof(val)); - if (bytes == -1) { - flb_errno(); - return -1; - } - - /* Get type and key */ - type = FLB_BITS_U64_HIGH(val); - ins_id = FLB_BITS_U64_LOW(val); - - /* At the moment we only support events coming from an input coroutine */ - if (type != FLB_ENGINE_IN_CORO) { - flb_error("[engine] invalid event type %i for input handler", - type); - return -1; - } - - flb_input_coro_finished(config, ins_id); - return 0; -} - -static inline int handle_output_event(uint64_t ts, - struct flb_config *config, - uint64_t val) -{ - int ret; - int task_id; - int out_id; - int retries; - int retry_seconds; - uint32_t type; - uint32_t key; - char *name; - struct flb_task *task; - struct flb_task_retry *retry; - struct flb_output_instance *ins; - - /* Get type and key */ - type = FLB_BITS_U64_HIGH(val); - key = FLB_BITS_U64_LOW(val); - - if (type != FLB_ENGINE_TASK) { - flb_error("[engine] invalid event type %i for output handler", - type); - return -1; - } - - /* - * The notion of ENGINE_TASK is associated to outputs. All thread - * references below belongs to flb_output_coro's. - */ - ret = FLB_TASK_RET(key); - task_id = FLB_TASK_ID(key); - out_id = FLB_TASK_OUT(key); - -#ifdef FLB_HAVE_TRACE - char *trace_st = NULL; - - if (ret == FLB_OK) { - trace_st = "OK"; - } - else if (ret == FLB_ERROR) { - trace_st = "ERROR"; - } - else if (ret == FLB_RETRY) { - trace_st = "RETRY"; - } - - flb_trace("%s[engine] [task event]%s task_id=%i out_id=%i return=%s", - ANSI_YELLOW, ANSI_RESET, - task_id, out_id, trace_st); -#endif - - task = config->tasks_map[task_id].task; - ins = flb_output_get_instance(config, out_id); - if (flb_output_is_threaded(ins) == FLB_FALSE) { - flb_output_flush_finished(config, out_id); - } - name = (char *) flb_output_name(ins); - - /* If we are in synchronous mode, flush the next waiting task */ - if (ins->flags & FLB_OUTPUT_SYNCHRONOUS) { - if (ret == FLB_OK || ret == FLB_RETRY || ret == FLB_ERROR) { - flb_output_task_singleplex_flush_next(ins->singleplex_queue); - } - } - - /* A task has finished, delete it */ - if (ret == FLB_OK) { - /* cmetrics */ - cmt_counter_add(ins->cmt_proc_records, ts, task->event_chunk->total_events, - 1, (char *[]) {name}); - - cmt_counter_add(ins->cmt_proc_bytes, ts, task->event_chunk->size, - 1, (char *[]) {name}); - - /* [OLD API] Update metrics */ -#ifdef FLB_HAVE_METRICS - if (ins->metrics) { - flb_metrics_sum(FLB_METRIC_OUT_OK_RECORDS, - task->event_chunk->total_events, ins->metrics); - flb_metrics_sum(FLB_METRIC_OUT_OK_BYTES, - task->event_chunk->size, ins->metrics); - } -#endif - /* Inform the user if a 'retry' succedeed */ - if (mk_list_size(&task->retries) > 0) { - retries = flb_task_retry_count(task, ins); - if (retries > 0) { - flb_info("[engine] flush chunk '%s' succeeded at retry %i: " - "task_id=%i, input=%s > output=%s (out_id=%i)", - flb_input_chunk_get_name(task->ic), - retries, task_id, - flb_input_name(task->i_ins), - flb_output_name(ins), out_id); - } - } - else if (flb_task_from_fs_storage(task) == FLB_TRUE) { - flb_info("[engine] flush backlog chunk '%s' succeeded: " - "task_id=%i, input=%s > output=%s (out_id=%i)", - flb_input_chunk_get_name(task->ic), - task_id, - flb_input_name(task->i_ins), - flb_output_name(ins), out_id); - } - - flb_task_retry_clean(task, ins); - flb_task_users_dec(task, FLB_TRUE); - } - else if (ret == FLB_RETRY) { - if (ins->retry_limit == FLB_OUT_RETRY_NONE) { - /* cmetrics: output_dropped_records_total */ - cmt_counter_add(ins->cmt_dropped_records, ts, task->records, - 1, (char *[]) {name}); - - /* OLD metrics API */ -#ifdef FLB_HAVE_METRICS - flb_metrics_sum(FLB_METRIC_OUT_DROPPED_RECORDS, task->records, ins->metrics); -#endif - flb_info("[engine] chunk '%s' is not retried (no retry config): " - "task_id=%i, input=%s > output=%s (out_id=%i)", - flb_input_chunk_get_name(task->ic), - task_id, - flb_input_name(task->i_ins), - flb_output_name(ins), out_id); - - flb_task_retry_clean(task, ins); - flb_task_users_dec(task, FLB_TRUE); - - return 0; - } - - /* Create a Task-Retry */ - retry = flb_task_retry_create(task, ins); - if (!retry) { - /* - * It can fail in two situations: - * - * - No enough memory (unlikely) - * - It reached the maximum number of re-tries - */ - - /* cmetrics */ - cmt_counter_inc(ins->cmt_retries_failed, ts, 1, (char *[]) {name}); - cmt_counter_add(ins->cmt_dropped_records, ts, task->records, - 1, (char *[]) {name}); - - /* OLD metrics API */ -#ifdef FLB_HAVE_METRICS - flb_metrics_sum(FLB_METRIC_OUT_RETRY_FAILED, 1, ins->metrics); - flb_metrics_sum(FLB_METRIC_OUT_DROPPED_RECORDS, task->records, ins->metrics); -#endif - /* Notify about this failed retry */ - flb_error("[engine] chunk '%s' cannot be retried: " - "task_id=%i, input=%s > output=%s", - flb_input_chunk_get_name(task->ic), - task_id, - flb_input_name(task->i_ins), - flb_output_name(ins)); - - flb_task_retry_clean(task, ins); - flb_task_users_dec(task, FLB_TRUE); - - return 0; - } - - /* Always destroy the old coroutine */ - flb_task_users_dec(task, FLB_FALSE); - - /* Let the scheduler to retry the failed task/thread */ - retry_seconds = flb_sched_request_create(config, - retry, retry->attempts); - - /* - * If for some reason the Scheduler could not include this retry, - * we need to get rid of it, likely this is because of not enough - * memory available or we ran out of file descriptors. - */ - if (retry_seconds == -1) { - flb_warn("[engine] retry for chunk '%s' could not be scheduled: " - "input=%s > output=%s", - flb_input_chunk_get_name(task->ic), - flb_input_name(task->i_ins), - flb_output_name(ins)); - - flb_task_retry_destroy(retry); - flb_task_users_release(task); - } - else { - /* Inform the user 'retry' has been scheduled */ - flb_warn("[engine] failed to flush chunk '%s', retry in %i seconds: " - "task_id=%i, input=%s > output=%s (out_id=%i)", - flb_input_chunk_get_name(task->ic), - retry_seconds, - task->id, - flb_input_name(task->i_ins), - flb_output_name(ins), out_id); - - /* cmetrics */ - cmt_counter_inc(ins->cmt_retries, ts, 1, (char *[]) {name}); - cmt_counter_add(ins->cmt_retried_records, ts, task->records, - 1, (char *[]) {name}); - - /* OLD metrics API: update the metrics since a new retry is coming */ -#ifdef FLB_HAVE_METRICS - flb_metrics_sum(FLB_METRIC_OUT_RETRY, 1, ins->metrics); - flb_metrics_sum(FLB_METRIC_OUT_RETRIED_RECORDS, task->records, ins->metrics); -#endif - } - } - else if (ret == FLB_ERROR) { - /* cmetrics */ - cmt_counter_inc(ins->cmt_errors, ts, 1, (char *[]) {name}); - cmt_counter_add(ins->cmt_dropped_records, ts, task->records, - 1, (char *[]) {name}); - - /* OLD API */ -#ifdef FLB_HAVE_METRICS - flb_metrics_sum(FLB_METRIC_OUT_ERROR, 1, ins->metrics); - flb_metrics_sum(FLB_METRIC_OUT_DROPPED_RECORDS, task->records, ins->metrics); -#endif - - flb_task_retry_clean(task, ins); - flb_task_users_dec(task, FLB_TRUE); - } - - return 0; -} - -static inline int handle_output_events(flb_pipefd_t fd, - struct flb_config *config) -{ - uint64_t values[FLB_ENGINE_OUTPUT_EVENT_BATCH_SIZE]; - int result; - int bytes; - size_t limit; - size_t index; - uint64_t ts; - - memset(&values, 0, sizeof(values)); - - bytes = flb_pipe_r(fd, &values, sizeof(values)); - - if (bytes == -1) { - flb_errno(); - return -1; - } - - limit = floor(bytes / sizeof(uint64_t)); - - ts = cfl_time_now(); - - for (index = 0 ; - index < limit && - index < (sizeof(values) / sizeof(values[0])) ; - index++) { - if (values[index] == 0) { - break; - } - - result = handle_output_event(ts, config, values[index]); - } - - /* This is wrong, in one hand, if handle_output_event_ fails we should - * stop, on the other, we have already consumed the signals from the pipe - * so we have to do whatever we can with them. - * - * And a side effect is that since we have N results but we are not aborting - * as soon as we get an error there could be N results to this function which - * not only are we not ready to handle but is not even checked at the moment. - */ - - return result; -} - -static inline int flb_engine_manager(flb_pipefd_t fd, struct flb_config *config) -{ - int bytes; - uint32_t type; - uint32_t key; - uint64_t val; - - /* read the event */ - bytes = flb_pipe_r(fd, &val, sizeof(val)); - if (bytes == -1) { - flb_errno(); - return -1; - } - - /* Get type and key */ - type = FLB_BITS_U64_HIGH(val); - key = FLB_BITS_U64_LOW(val); - - /* Flush all remaining data */ - if (type == 1) { /* Engine type */ - if (key == FLB_ENGINE_STOP) { - flb_trace("[engine] flush enqueued data"); - flb_engine_flush(config, NULL); - return FLB_ENGINE_STOP; - } - } - - return 0; -} - -static FLB_INLINE int flb_engine_handle_event(flb_pipefd_t fd, int mask, - struct flb_config *config) -{ - int ret; - - /* flb_engine_shutdown was already initiated */ - if (config->is_running == FLB_FALSE) { - return 0; - } - - if (mask & MK_EVENT_READ) { - /* Check if we need to flush */ - if (config->flush_fd == fd) { - flb_utils_timer_consume(fd); - flb_engine_flush(config, NULL); - return 0; - } - else if (config->shutdown_fd == fd) { - flb_utils_pipe_byte_consume(fd); - return FLB_ENGINE_SHUTDOWN; - } - else if (config->ch_manager[0] == fd) { - ret = flb_engine_manager(fd, config); - if (ret == FLB_ENGINE_STOP || ret == FLB_ENGINE_EV_STOP) { - return FLB_ENGINE_STOP; - } - } - - /* Try to match the file descriptor with a collector event */ - ret = flb_input_collector_fd(fd, config); - if (ret != -1) { - return ret; - } - - /* Metrics exporter event ? */ -#ifdef FLB_HAVE_METRICS - ret = flb_me_fd_event(fd, config->metrics); - if (ret != -1) { - return ret; - } -#endif - - /* Stream processor event ? */ -#ifdef FLB_HAVE_STREAM_PROCESSOR - if (config->stream_processor_ctx) { - ret = flb_sp_fd_event(fd, config->stream_processor_ctx); - if (ret != -1) { - return ret; - } - } -#endif - } - - return 0; -} - -static int flb_engine_started(struct flb_config *config) -{ - uint64_t val; - - /* Check the channel is valid (enabled by library mode) */ - if (config->ch_notif[1] <= 0) { - return -1; - } - - val = FLB_ENGINE_STARTED; - return flb_pipe_w(config->ch_notif[1], &val, sizeof(uint64_t)); -} - -int flb_engine_failed(struct flb_config *config) -{ - int ret; - uint64_t val; - - /* Check the channel is valid (enabled by library mode) */ - if (config->ch_notif[1] <= 0) { - flb_error("[engine] no channel to notify FAILED message"); - return -1; - } - - val = FLB_ENGINE_FAILED; - ret = flb_pipe_w(config->ch_notif[1], &val, sizeof(uint64_t)); - if (ret == -1) { - flb_error("[engine] fail to dispatch FAILED message"); - } - - /* Waiting flushing log */ - sleep(1); - - return ret; -} - -static int flb_engine_log_start(struct flb_config *config) -{ - int type; - int level; - - /* Log Level */ - if (config->verbose != FLB_LOG_INFO) { - level = config->verbose; - } - else { - level = FLB_LOG_INFO; - } - - /* Destination based on type */ - if (config->log_file) { - type = FLB_LOG_FILE; - } - else { - type = FLB_LOG_STDERR; - } - - if (flb_log_create(config, type, level, config->log_file) == NULL) { - return -1; - } - - return 0; -} - -static void flb_engine_drain_ring_buffer_signal_channel(flb_pipefd_t fd) -{ - static char signal_buffer[512]; - - flb_pipe_r(fd, signal_buffer, sizeof(signal_buffer)); -} - - -#ifdef FLB_HAVE_IN_STORAGE_BACKLOG -extern int sb_segregate_chunks(struct flb_config *config); -#else -int sb_segregate_chunks(struct flb_config *config) -{ - return 0; -} -#endif - -int flb_engine_start(struct flb_config *config) -{ - int ret; - uint64_t ts; - char tmp[16]; - int rb_flush_flag; - struct flb_time t_flush; - struct mk_event *event; - struct mk_event_loop *evl; - struct flb_bucket_queue *evl_bktq; - struct flb_sched *sched; - struct flb_net_dns dns_ctx; - - /* Initialize the networking layer */ - flb_net_lib_init(); - flb_net_ctx_init(&dns_ctx); - flb_net_dns_ctx_init(); - flb_net_dns_ctx_set(&dns_ctx); - - flb_pack_init(config); - - /* Create the event loop and set it in the global configuration */ - evl = mk_event_loop_create(256); - if (!evl) { - fprintf(stderr, "[log] could not create event loop\n"); - return -1; - } - config->evl = evl; - - /* Create the bucket queue (FLB_ENGINE_PRIORITY_COUNT priorities) */ - evl_bktq = flb_bucket_queue_create(FLB_ENGINE_PRIORITY_COUNT); - if (!evl_bktq) { - return -1; - } - config->evl_bktq = evl_bktq; - - /* - * Event loop channel to ingest flush events from flb_engine_flush() - * - * - FLB engine uses 'ch_self_events[1]' to dispatch tasks to self - * - Self to receive message on ch_parent_events[0] - * - * The mk_event_channel_create() will attach the pipe read end ch_self_events[0] - * to the local event loop 'evl'. - */ - ret = mk_event_channel_create(config->evl, - &config->ch_self_events[0], - &config->ch_self_events[1], - &config->event_thread_init); - if (ret == -1) { - flb_error("[engine] could not create engine thread channel"); - return -1; - } - /* Signal type to indicate a "flush" request */ - config->event_thread_init.type = FLB_ENGINE_EV_THREAD_ENGINE; - config->event_thread_init.priority = FLB_ENGINE_PRIORITY_THREAD; - - /* Register the event loop on this thread */ - flb_engine_evl_init(); - flb_engine_evl_set(evl); - - /* Start the Logging service */ - ret = flb_engine_log_start(config); - if (ret == -1) { - fprintf(stderr, "[engine] log start failed\n"); - return -1; - } - - flb_info("[fluent bit] version=%s, commit=%.10s, pid=%i", - FLB_VERSION_STR, FLB_GIT_HASH, getpid()); - - /* Debug coroutine stack size */ - flb_utils_bytes_to_human_readable_size(config->coro_stack_size, - tmp, sizeof(tmp)); - flb_debug("[engine] coroutine stack size: %u bytes (%s)", - config->coro_stack_size, tmp); - - /* - * Create a communication channel: this routine creates a channel to - * signal the Engine event loop. It's useful to stop the event loop - * or to instruct anything else without break. - */ - ret = mk_event_channel_create(config->evl, - &config->ch_manager[0], - &config->ch_manager[1], - &config->ch_event); - if (ret != 0) { - flb_error("[engine] could not create manager channels"); - return -1; - } - - /* Initialize custom plugins */ - ret = flb_custom_init_all(config); - if (ret == -1) { - return -1; - } - - /* Start the Storage engine */ - ret = flb_storage_create(config); - if (ret == -1) { - flb_error("[engine] storage creation failed"); - return -1; - } - - /* Init Metrics engine */ - cmt_initialize(); - flb_info("[cmetrics] version=%s", cmt_version()); - flb_info("[ctraces ] version=%s", ctr_version()); - - /* Initialize the scheduler */ - sched = flb_sched_create(config, config->evl); - if (!sched) { - flb_error("[engine] scheduler could not start"); - return -1; - } - config->sched = sched; - - /* Register the scheduler context */ - flb_sched_ctx_init(); - flb_sched_ctx_set(sched); - - /* Initialize input plugins */ - ret = flb_input_init_all(config); - if (ret == -1) { - flb_error("[engine] input initialization failed"); - return -1; - } - - /* Initialize filter plugins */ - ret = flb_filter_init_all(config); - if (ret == -1) { - flb_error("[engine] filter initialization failed"); - return -1; - } - - /* Inputs pre-run */ - flb_input_pre_run_all(config); - - /* Initialize output plugins */ - ret = flb_output_init_all(config); - if (ret == -1) { - flb_error("[engine] output initialization failed"); - return -1; - } - - /* Outputs pre-run */ - flb_output_pre_run(config); - - /* Create and register the timer fd for flush procedure */ - event = &config->event_flush; - event->mask = MK_EVENT_EMPTY; - event->status = MK_EVENT_NONE; - - flb_time_from_double(&t_flush, config->flush); - config->flush_fd = mk_event_timeout_create(evl, - t_flush.tm.tv_sec, - t_flush.tm.tv_nsec, - event); - event->priority = FLB_ENGINE_PRIORITY_FLUSH; - if (config->flush_fd == -1) { - flb_utils_error(FLB_ERR_CFG_FLUSH_CREATE); - } - - -#ifdef FLB_HAVE_METRICS - if (config->storage_metrics == FLB_TRUE) { - config->storage_metrics_ctx = flb_storage_metrics_create(config); - } -#endif - - /* Prepare routing paths */ - ret = flb_router_io_set(config); - if (ret == -1) { - flb_error("[engine] router failed"); - return -1; - } - - /* Support mode only */ - if (config->support_mode == FLB_TRUE) { - sleep(1); - flb_sosreport(config); - exit(1); - } - - /* Initialize Metrics exporter */ -#ifdef FLB_HAVE_METRICS - config->metrics = flb_me_create(config); -#endif - - /* Initialize HTTP Server */ -#ifdef FLB_HAVE_HTTP_SERVER - if (config->http_server == FLB_TRUE) { - config->http_ctx = flb_hs_create(config->http_listen, config->http_port, - config); - flb_hs_start(config->http_ctx); - } -#endif - -#ifdef FLB_HAVE_STREAM_PROCESSOR - config->stream_processor_ctx = flb_sp_create(config); - if (!config->stream_processor_ctx) { - flb_error("[engine] could not initialize stream processor"); - } -#endif - - /* Initialize collectors */ - flb_input_collectors_start(config); - - /* - * Sched a permanent callback triggered every 1.5 second to let other - * Fluent Bit components run tasks at that interval. - */ - ret = flb_sched_timer_cb_create(config->sched, - FLB_SCHED_TIMER_CB_PERM, - 1500, cb_engine_sched_timer, config, NULL); - if (ret == -1) { - flb_error("[engine] could not schedule permanent callback"); - return -1; - } - - /* DEV/TEST change only */ - int rb_ms; - char *rb_env; - - rb_env = getenv("FLB_DEV_RB_MS"); - if (!rb_env) { - rb_ms = 250; - } - else { - rb_ms = atoi(rb_env); - } - - /* Input instance / Ring buffer collector */ - ret = flb_sched_timer_cb_create(config->sched, - FLB_SCHED_TIMER_CB_PERM, - rb_ms, flb_input_chunk_ring_buffer_collector, - config, NULL); - if (ret == -1) { - flb_error("[engine] could not schedule permanent callback"); - return -1; - } - - /* Signal that we have started */ - flb_engine_started(config); - - ret = sb_segregate_chunks(config); - - if (ret) { - flb_error("[engine] could not segregate backlog chunks"); - return -2; - } - - while (1) { - rb_flush_flag = FLB_FALSE; - - mk_event_wait(evl); /* potentially conditional mk_event_wait or mk_event_wait_2 based on bucket queue capacity for one shot events */ - flb_event_priority_live_foreach(event, evl_bktq, evl, FLB_ENGINE_LOOP_MAX_ITER) { - if (event->type == FLB_ENGINE_EV_CORE) { - ret = flb_engine_handle_event(event->fd, event->mask, config); - if (ret == FLB_ENGINE_STOP) { - if (config->grace_count == 0) { - if (config->grace >= 0) { - flb_warn("[engine] service will shutdown in max %u seconds", - config->grace); - } else { - flb_warn("[engine] service will shutdown when all remaining tasks are flushed"); - } - - /* Reschedule retry tasks to be retried immediately */ - flb_engine_reschedule_retries(config); - } - - /* mark the runtime as the ingestion is not active and that we are in shutting down mode */ - config->is_ingestion_active = FLB_FALSE; - config->is_shutting_down = FLB_TRUE; - - /* pause all input plugin instances */ - flb_input_pause_all(config); - - /* - * We are preparing to shutdown, we give a graceful time - * of 'config->grace' seconds to process any pending event. - */ - event = &config->event_shutdown; - event->mask = MK_EVENT_EMPTY; - event->status = MK_EVENT_NONE; - - /* - * Configure a timer of 1 second, on expiration the code will - * jump into the FLB_ENGINE_SHUTDOWN condition where it will - * check if the grace period has finished, or if there are - * any remaining tasks. - * - * If no tasks exists, there is no need to wait for the maximum - * grace period. - */ - config->shutdown_fd = mk_event_timeout_create(evl, - 1, - 0, - event); - event->priority = FLB_ENGINE_PRIORITY_SHUTDOWN; - } - else if (ret == FLB_ENGINE_SHUTDOWN) { - if (config->shutdown_fd > 0) { - mk_event_timeout_destroy(config->evl, - &config->event_shutdown); - } - - /* Increase the grace counter */ - config->grace_count++; - - /* - * Grace timeout has finished, but we need to check if there is - * any pending running task. A running task is associated to an - * output co-routine, since we don't know what's the state or - * resources allocated by that co-routine, the best thing is to - * wait again for the grace period and re-check again. - * If grace period is set to -1, keep trying to shut down until all - * tasks and retries get flushed. - */ - ret = flb_task_running_count(config); - if (ret > 0 && (config->grace_count < config->grace || config->grace == -1)) { - if (config->grace_count == 1) { - flb_task_running_print(config); - } - flb_engine_exit(config); - } - else { - if (ret > 0) { - flb_task_running_print(config); - } - flb_info("[engine] service has stopped (%i pending tasks)", - ret); - ret = config->exit_status_code; - flb_engine_shutdown(config); - config = NULL; - return ret; - } - } - } - else if (event->type & FLB_ENGINE_EV_SCHED) { - /* Event type registered by the Scheduler */ - flb_sched_event_handler(config, event); - } - else if (event->type == FLB_ENGINE_EV_THREAD_ENGINE) { - struct flb_output_flush *output_flush; - - /* Read the coroutine reference */ - ret = flb_pipe_r(event->fd, &output_flush, sizeof(struct flb_output_flush *)); - if (ret <= 0 || output_flush == 0) { - flb_errno(); - continue; - } - - /* Init coroutine */ - flb_coro_resume(output_flush->coro); - } - else if (event->type == FLB_ENGINE_EV_CUSTOM) { - event->handler(event); - } - else if (event->type == FLB_ENGINE_EV_THREAD) { - struct flb_connection *connection; - - /* - * Check if we have some co-routine associated to this event, - * if so, resume the co-routine - */ - - connection = (struct flb_connection *) event; - - if (connection->coroutine) { - flb_trace("[engine] resuming coroutine=%p", connection->coroutine); - - flb_coro_resume(connection->coroutine); - } - } - else if (event->type == FLB_ENGINE_EV_OUTPUT) { - /* - * Event originated by an output plugin. likely a Task return - * status. - */ - handle_output_events(event->fd, config); - } - else if (event->type == FLB_ENGINE_EV_INPUT) { - ts = cfl_time_now(); - handle_input_event(event->fd, ts, config); - } - else if(event->type == FLB_ENGINE_EV_THREAD_INPUT) { - flb_engine_drain_ring_buffer_signal_channel(event->fd); - - rb_flush_flag = FLB_TRUE; - } - } - - if (rb_flush_flag) { - flb_input_chunk_ring_buffer_collector(config, NULL); - } - - /* Cleanup functions associated to events and timers */ - if (config->is_running == FLB_TRUE) { - flb_net_dns_lookup_context_cleanup(&dns_ctx); - flb_sched_timer_cleanup(config->sched); - flb_upstream_conn_pending_destroy_list(&config->upstreams); - flb_downstream_conn_pending_destroy_list(&config->downstreams); - - /* - * depend on main thread to clean up expired message - * in aws error reporting message queue - */ - #ifdef FLB_HAVE_AWS_ERROR_REPORTER - if (is_error_reporting_enabled()) { - flb_aws_error_reporter_clean(error_reporter); - } - #endif - } - } -} - -/* Release all resources associated to the engine */ -int flb_engine_shutdown(struct flb_config *config) -{ - - config->is_running = FLB_FALSE; - flb_input_pause_all(config); - -#ifdef FLB_HAVE_STREAM_PROCESSOR - if (config->stream_processor_ctx) { - flb_sp_destroy(config->stream_processor_ctx); - } -#endif - - /* router */ - flb_router_exit(config); - - /* cleanup plugins */ - flb_filter_exit(config); - flb_output_exit(config); - flb_custom_exit(config); - flb_input_exit_all(config); - - /* Destroy the storage context */ - flb_storage_destroy(config); - - /* metrics */ -#ifdef FLB_HAVE_METRICS - if (config->metrics) { - flb_me_destroy(config->metrics); - } -#endif - -#ifdef FLB_HAVE_HTTP_SERVER - if (config->http_server == FLB_TRUE) { - flb_hs_destroy(config->http_ctx); - } -#endif - - return 0; -} - -int flb_engine_exit(struct flb_config *config) -{ - int ret; - uint64_t val; - - val = FLB_ENGINE_EV_STOP; - ret = flb_pipe_w(config->ch_manager[1], &val, sizeof(uint64_t)); - return ret; -} - -int flb_engine_exit_status(struct flb_config *config, int status) -{ - config->exit_status_code = status; - return flb_engine_exit(config); -} diff --git a/fluent-bit/src/flb_engine_dispatch.c b/fluent-bit/src/flb_engine_dispatch.c deleted file mode 100644 index 5b30c28b4..000000000 --- a/fluent-bit/src/flb_engine_dispatch.c +++ /dev/null @@ -1,339 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <stdlib.h> - -#include <fluent-bit/flb_info.h> -#include <fluent-bit/flb_input.h> -#include <fluent-bit/flb_input_chunk.h> -#include <fluent-bit/flb_output.h> -#include <fluent-bit/flb_router.h> -#include <fluent-bit/flb_config.h> -#include <fluent-bit/flb_coro.h> -#include <fluent-bit/flb_engine.h> -#include <fluent-bit/flb_task.h> -#include <fluent-bit/flb_event.h> - - -/* It creates a new output thread using a 'Retry' context */ -int flb_engine_dispatch_retry(struct flb_task_retry *retry, - struct flb_config *config) -{ - int ret; - char *buf_data; - size_t buf_size; - struct flb_task *task; - - task = retry->parent; - - /* Set file up/down based on restrictions */ - ret = flb_input_chunk_set_up(task->ic); - if (ret == -1) { - /* - * The re-try is not possible. The chunk is not in memory and trying to bringing it - * up was not possible. - * - * A common cause for this is that the Chunk I/O system is not draining fast - * enough like errors on delivering data. So if we cannot put the chunk in memory - * it cannot be retried. - */ - ret = flb_task_retry_reschedule(retry, config); - if (ret == -1) { - return -1; - } - - /* Just return because it has been re-scheduled */ - return 0; - } - - /* There is a match, get the buffer */ - buf_data = (char *) flb_input_chunk_flush(task->ic, &buf_size); - if (!buf_data) { - /* Could not retrieve chunk content */ - flb_error("[engine_dispatch] could not retrieve chunk content, removing retry"); - flb_task_retry_destroy(retry); - return -1; - } - - /* Update the buffer reference */ - flb_event_chunk_update(task->event_chunk, buf_data, buf_size); - - /* flush the task */ - if (retry->o_ins->flags & FLB_OUTPUT_SYNCHRONOUS) { - /* - * If the plugin doesn't allow for multiplexing. - * singleplex_enqueue deletes retry context on flush or delayed flush failure - */ - ret = flb_output_task_singleplex_enqueue(retry->o_ins->singleplex_queue, retry, - task, retry->o_ins, config); - if (ret == -1) { - return -1; - } - } - else { - ret = flb_output_task_flush(task, retry->o_ins, config); - if (ret == -1) { - flb_task_retry_destroy(retry); - return -1; - } - } - - return 0; -} - -static void test_run_formatter(struct flb_config *config, - struct flb_input_instance *i_ins, - struct flb_output_instance *o_ins, - struct flb_task *task, - void *flush_ctx) -{ - int ret; - void *out_buf = NULL; - size_t out_size = 0; - struct flb_test_out_formatter *otf; - struct flb_event_chunk *evc; - - otf = &o_ins->test_formatter; - evc = task->event_chunk; - - /* Invoke the output plugin formatter test callback */ - ret = otf->callback(config, - i_ins, - o_ins->context, - flush_ctx, - evc->type, - evc->tag, flb_sds_len(evc->tag), - evc->data, evc->size, - &out_buf, &out_size); - - /* Call the runtime test callback checker */ - if (otf->rt_out_callback) { - otf->rt_out_callback(otf->rt_ctx, - otf->rt_ffd, - ret, - out_buf, out_size, - otf->rt_data); - } - else { - flb_free(out_buf); - } -} - -static int tasks_start(struct flb_input_instance *in, - struct flb_config *config) -{ - int hits = 0; - int retry = 0; - struct mk_list *tmp; - struct mk_list *head; - struct mk_list *r_head; - struct mk_list *r_tmp; - struct flb_task *task; - struct flb_task_route *route; - struct flb_output_instance *out; - - /* At this point the input instance should have some tasks linked */ - mk_list_foreach_safe(head, tmp, &in->tasks) { - task = mk_list_entry(head, struct flb_task, _head); - - if (mk_list_is_empty(&task->retries) != 0) { - retry++; - } - - /* Only process recently created tasks */ - if (task->status != FLB_TASK_NEW) { - continue; - } - task->status = FLB_TASK_RUNNING; - - /* A task contain one or more routes */ - mk_list_foreach_safe(r_head, r_tmp, &task->routes) { - route = mk_list_entry(r_head, struct flb_task_route, _head); - - /* - * Test mode: if the output plugin is in test mode, just invoke - * the proper test function and continue; - */ - out = route->out; - if (out->test_mode == FLB_TRUE && - out->test_formatter.callback != NULL) { - - /* Run the formatter test */ - test_run_formatter(config, in, out, - task, - out->test_formatter.flush_ctx); - - /* Remove the route */ - mk_list_del(&route->_head); - flb_free(route); - continue; - } - - /* - * If the plugin don't allow multiplexing Tasks, check if it's - * running something. - */ - if (out->flags & FLB_OUTPUT_NO_MULTIPLEX) { - if (flb_output_coros_size(route->out) > 0 || retry > 0) { - continue; - } - } - - hits++; - - /* - * If the plugin is in synchronous mode, enqueue the task and flush - * when appropriate. - */ - if (out->flags & FLB_OUTPUT_SYNCHRONOUS) { - flb_output_task_singleplex_enqueue(route->out->singleplex_queue, NULL, - task, route->out, config); - } - else { - /* - * We have the Task and the Route, created a thread context for the - * data handling. - */ - flb_output_task_flush(task, route->out, config); - } - - /* - th = flb_output_thread(task, - in, - route->out, - config, - task->buf, task->size, - task->tag, - task->tag_len); - flb_task_add_thread(th, task); - flb_thread_resume(th); - */ - } - - if (hits == 0) { - task->status = FLB_TASK_NEW; - } - - hits = 0; - } - - return 0; -} - -/* - * The engine dispatch is responsible for: - * - * - Get chunks generated by input plugins. - * - For each set of records under the same tag, create a Task. A Task set - * a reference to the records and routes through output instances. - */ -int flb_engine_dispatch(uint64_t id, struct flb_input_instance *in, - struct flb_config *config) -{ - int ret; - int t_err; - const char *buf_data; - size_t buf_size = 0; - const char *tag_buf; - int tag_len; - struct mk_list *tmp; - struct mk_list *head; - struct flb_input_plugin *p; - struct flb_input_chunk *ic; - struct flb_task *task = NULL; - - p = in->p; - if (!p) { - return 0; - } - - /* Look for chunks ready to go */ - mk_list_foreach_safe(head, tmp, &in->chunks) { - ic = mk_list_entry(head, struct flb_input_chunk, _head); - if (ic->busy == FLB_TRUE) { - continue; - } - - /* There is a match, get the buffer */ - buf_data = flb_input_chunk_flush(ic, &buf_size); - if (buf_size == 0) { - /* - * Do not release the buffer since if allocated, it will be - * released when the task is destroyed. - */ - flb_input_chunk_release_lock(ic); - continue; - } - if (!buf_data) { - flb_input_chunk_release_lock(ic); - continue; - } - - /* Get the the tag reference (chunk metadata) */ - ret = flb_input_chunk_get_tag(ic, &tag_buf, &tag_len); - if (ret == -1) { - flb_input_chunk_release_lock(ic); - continue; - } - - /* Validate outgoing Tag information */ - if (!tag_buf || tag_len <= 0) { - flb_input_chunk_release_lock(ic); - continue; - } - - /* Create a task */ - task = flb_task_create(id, buf_data, buf_size, - ic->in, ic, - tag_buf, tag_len, - config, &t_err); - if (!task) { - /* - * If task creation failed, check the error status flag. An error - * is associated with memory allocation or exhaustion of tasks_id, - * on that case the input chunk must be preserved and retried - * later. So we just release it busy lock. - */ - if (t_err == FLB_TRUE) { - flb_input_chunk_release_lock(ic); - } - continue; - } - } - - /* Start the new enqueued Tasks */ - tasks_start(in, config); - - /* - * Tasks cleanup: if some tasks are associated to output plugins running - * in test mode, they must be cleaned up since they do not longer contains - * an outgoing route. - */ - mk_list_foreach_safe(head, tmp, &in->tasks) { - task = mk_list_entry(head, struct flb_task, _head); - if (task->users == 0 && - mk_list_size(&task->retries) == 0 && - mk_list_size(&task->routes) == 0) { - flb_info("[task] cleanup test task"); - flb_task_destroy(task, FLB_TRUE); - } - } - - return 0; -} diff --git a/fluent-bit/src/flb_env.c b/fluent-bit/src/flb_env.c deleted file mode 100644 index 3b9158095..000000000 --- a/fluent-bit/src/flb_env.c +++ /dev/null @@ -1,273 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <fluent-bit/flb_info.h> -#include <fluent-bit/flb_hash_table.h> -#include <fluent-bit/flb_mem.h> -#include <fluent-bit/flb_log.h> -#include <fluent-bit/flb_str.h> -#include <fluent-bit/flb_env.h> - -#include <stdlib.h> - -static inline flb_sds_t buf_append(flb_sds_t buf, const char *str, int len) -{ - flb_sds_t tmp; - - tmp = flb_sds_cat(buf, str, len); - if (!tmp) { - return NULL; - } - - return tmp; -} - -/* Preset some useful variables */ -static int env_preset(struct flb_env *env) -{ - int ret; - char *buf; - char tmp[512]; - - /* - * ${HOSTNAME} this variable is very useful to identify records, - * despite this variable is recognized by the Shell, that does not - * means that is exposed as a real environment variable, e.g: - * - * 1. $ echo $HOSTNAME - * monotop - * 2. $ env | grep HOSTNAME - * (nothing) - */ - buf = getenv("HOSTNAME"); - if (!buf) { - ret = gethostname(tmp, sizeof(tmp) - 1); - if (ret == 0) { - flb_env_set(env, "HOSTNAME", tmp); - } - } - - return 0; -} - -struct flb_env *flb_env_create() -{ - struct flb_env *env; - struct flb_hash_table *ht; - - env = flb_malloc(sizeof(struct flb_env)); - if (!env) { - flb_errno(); - return NULL; - } - - /* Create the hash-table */ - ht = flb_hash_table_create(FLB_HASH_TABLE_EVICT_NONE, FLB_ENV_SIZE, -1); - if (!ht) { - flb_free(env); - return NULL; - } - - env->warn_unused = FLB_TRUE; - env->ht = ht; - env_preset(env); - - return env; -} - -void flb_env_destroy(struct flb_env *env) -{ - flb_hash_table_destroy(env->ht); - flb_free(env); -} - -int flb_env_set(struct flb_env *env, const char *key, const char *val) -{ - int id; - int klen; - int vlen; - void *out_buf; - size_t out_size; - - /* Get lengths */ - klen = strlen(key); - vlen = strlen(val); - - /* Check if the key is already set */ - id = flb_hash_table_get(env->ht, key, klen, &out_buf, &out_size); - if (id >= 0) { - /* Remove the old entry */ - flb_hash_table_del(env->ht, key); - } - - /* Register the new key */ - id = flb_hash_table_add(env->ht, key, klen, (void *) val, vlen); - return id; -} - -const char *flb_env_get(struct flb_env *env, const char *key) -{ - int len; - int ret; - void *out_buf; - size_t out_size; - - if (!key) { - return NULL; - } - - len = strlen(key); - - /* Try to get the value from the hash table */ - ret = flb_hash_table_get(env->ht, key, len, &out_buf, &out_size); - if (ret >= 0) { - return (char *) out_buf; - } - - /* If it was not found, try to get it from the real environment */ - out_buf = getenv(key); - if (!out_buf) { - return NULL; - } - - if (strlen(out_buf) == 0) { - return NULL; - } - - return (char *) out_buf; -} - -/* - * Given a 'value', lookup for variables, if found, return a new composed - * sds string. - */ -flb_sds_t flb_env_var_translate(struct flb_env *env, const char *value) -{ - int i; - int len; - int v_len; - int e_len; - int pre_var; - int have_var = FLB_FALSE; - const char *env_var = NULL; - char *v_start = NULL; - char *v_end = NULL; - char tmp[4096]; - flb_sds_t buf; - flb_sds_t s; - - if (!value) { - return NULL; - } - - len = strlen(value); - buf = flb_sds_create_size(len); - if (!buf) { - return NULL; - } - - for (i = 0; i < len; i++) { - v_start = strstr(value + i, "${"); - if (!v_start) { - break; - } - - v_end = strstr(value + i, "}"); - if (!v_end) { - break; - } - - v_start += 2; - v_len = v_end - v_start; - if (v_len <= 0 || v_len >= sizeof(tmp)) { - break; - } - - /* variable */ - strncpy(tmp, v_start, v_len); - tmp[v_len] = '\0'; - have_var = FLB_TRUE; - - /* Append pre-variable content */ - pre_var = (v_start - 2) - (value + i); - if (pre_var > 0) { - s = buf_append(buf, value + i, (v_start - 2) - (value + i)); - if (!s) { - flb_sds_destroy(buf); - return NULL; - } - if (s != buf) { - buf = s; - } - } - - /* Lookup the variable in our env-hash */ - env_var = flb_env_get(env, tmp); - if (env_var) { - e_len = strlen(env_var); - s = buf_append(buf, env_var, e_len); - if (!s) { - flb_sds_destroy(buf); - return NULL; - } - if (s != buf) { - buf = s; - } - } - else if (env->warn_unused == FLB_TRUE) { - flb_warn("[env] variable ${%s} is used but not set", tmp); - } - i += (v_start - (value + i)) + v_len; - } - - /* Copy the remaining value into our buffer */ - if (v_end) { - if (have_var == FLB_TRUE && (value + len) - (v_end + 1) > 0) { - s = buf_append(buf, v_end + 1, (value + len) - (v_end + 1)); - if (!s) { - flb_sds_destroy(buf); - return NULL; - } - if (s != buf) { - buf = s; - } - } - } - - if (flb_sds_len(buf) == 0) { - /* - * If the output length buffer is zero, it could mean: - * - * - just one variable was given and it don't have any value - * - no variables given (keep original value) - * - * In order to avoid problems in the caller, if a variable is null - * and is the only one content available, return a new empty memory - * string. - */ - if (have_var == FLB_TRUE) { - return flb_sds_copy(buf, "", 0); - } - else { - return flb_sds_copy(buf, value, len); - } - } - - return buf; -} diff --git a/fluent-bit/src/flb_event.c b/fluent-bit/src/flb_event.c deleted file mode 100644 index d18bbe32a..000000000 --- a/fluent-bit/src/flb_event.c +++ /dev/null @@ -1,76 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <fluent-bit/flb_info.h> -#include <fluent-bit/flb_mem.h> -#include <fluent-bit/flb_log.h> - -#include <fluent-bit/flb_event.h> -#include <fluent-bit/flb_sds.h> - -struct flb_event_chunk *flb_event_chunk_create(int type, - int total_events, - char *tag_buf, int tag_len, - char *buf_data, size_t buf_size) -{ - struct flb_event_chunk *evc; - - /* event chunk context */ - evc = flb_malloc(sizeof(struct flb_event_chunk)); - if (!evc) { - flb_errno(); - return NULL; - } - - /* create a copy of the tag */ - evc->tag = flb_sds_create_len(tag_buf, tag_len); - if (!evc->tag) { - flb_free(evc); - return NULL; - } - - evc->type = type; - evc->data = buf_data; - evc->size = buf_size; - evc->total_events = total_events; - - return evc; -} - -/* Update the buffer reference */ -int flb_event_chunk_update(struct flb_event_chunk *evc, - char *buf_data, size_t buf_size) -{ - evc->data = buf_data; - evc->size = buf_size; - - return 0; -} - -void flb_event_chunk_destroy(struct flb_event_chunk *evc) -{ - if (!evc) { - return; - } - - if (evc->tag) { - flb_sds_destroy(evc->tag); - } - flb_free(evc); -} diff --git a/fluent-bit/src/flb_file.c b/fluent-bit/src/flb_file.c deleted file mode 100644 index 2225bc3c9..000000000 --- a/fluent-bit/src/flb_file.c +++ /dev/null @@ -1,73 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2019-2021 The Fluent Bit Authors - * Copyright (C) 2015-2018 Treasure Data Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <fluent-bit/flb_file.h> -#include <fluent-bit/flb_log.h> -#include <fluent-bit/flb_mem.h> -#include <fluent-bit/flb_sds.h> - -#include <stdio.h> - -flb_sds_t flb_file_read(const char *path) -{ - long flen; - FILE *f = NULL; - flb_sds_t result = NULL; - - f = fopen(path, "rb"); - if (!f) { - return NULL; - } - - if (fseek(f, 0, SEEK_END) == -1) { - goto err; - } - - flen = ftell(f); - if (flen < 0) { - goto err; - } - - if (fseek(f, 0, SEEK_SET) == -1) { - goto err; - } - - result = flb_sds_create_size(flen); - if (!result) { - goto err; - } - - if (flen > 0 && fread(result, flen, 1, f) != 1) { - goto err; - } - - result[flen] = 0; - flb_sds_len_set(result, flen); - fclose(f); - return result; - -err: - flb_errno(); - fclose(f); - if (result) { - flb_sds_destroy(result); - } - return NULL; -} diff --git a/fluent-bit/src/flb_filter.c b/fluent-bit/src/flb_filter.c deleted file mode 100644 index 389709a9a..000000000 --- a/fluent-bit/src/flb_filter.c +++ /dev/null @@ -1,669 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <fluent-bit/flb_config.h> -#include <fluent-bit/flb_filter.h> -#include <fluent-bit/flb_str.h> -#include <fluent-bit/flb_env.h> -#include <fluent-bit/flb_router.h> -#include <fluent-bit/flb_mp.h> -#include <fluent-bit/flb_kv.h> -#include <fluent-bit/flb_pack.h> -#include <fluent-bit/flb_metrics.h> -#include <fluent-bit/flb_utils.h> -#include <chunkio/chunkio.h> - -#ifdef FLB_HAVE_CHUNK_TRACE -#include <fluent-bit/flb_chunk_trace.h> -#endif /* FLB_HAVE_CHUNK_TRACE */ - -static inline int instance_id(struct flb_config *config) -{ - struct flb_filter_instance *entry; - - if (mk_list_size(&config->filters) == 0) { - return 0; - } - - entry = mk_list_entry_last(&config->filters, struct flb_filter_instance, - _head); - return (entry->id + 1); -} - -static int is_active(struct mk_list *in_properties) -{ - struct mk_list *head; - struct flb_kv *kv; - - mk_list_foreach(head, in_properties) { - kv = mk_list_entry(head, struct flb_kv, _head); - if (strcasecmp(kv->key, "active") == 0) { - /* Skip checking deactivation ... */ - if (strcasecmp(kv->val, "FALSE") == 0 || strcmp(kv->val, "0") == 0) { - return FLB_FALSE; - } - } - } - return FLB_TRUE; -} - -static inline int prop_key_check(const char *key, const char *kv, int k_len) -{ - int len; - - len = strlen(key); - if (strncasecmp(key, kv, k_len) == 0 && len == k_len) { - return 0; - } - - return -1; -} - -void flb_filter_do(struct flb_input_chunk *ic, - const void *data, size_t bytes, - const char *tag, int tag_len, - struct flb_config *config) -{ - int ret; -#ifdef FLB_HAVE_METRICS - int in_records = 0; - int out_records = 0; - int diff = 0; - int pre_records = 0; - uint64_t ts; - char *name; -#endif - char *ntag; - const char *work_data; - size_t work_size; - void *out_buf; - size_t cur_size; - size_t out_size; - ssize_t content_size; - ssize_t write_at; - struct mk_list *head; - struct flb_filter_instance *f_ins; - struct flb_input_instance *i_ins = ic->in; -/* measure time between filters for chunk traces. */ -#ifdef FLB_HAVE_CHUNK_TRACE - struct flb_time tm_start; - struct flb_time tm_finish; -#endif /* FLB_HAVE_CHUNK_TRACE */ - - /* For the incoming Tag make sure to create a NULL terminated reference */ - ntag = flb_malloc(tag_len + 1); - if (!ntag) { - flb_errno(); - flb_error("[filter] could not filter record due to memory problems"); - return; - } - memcpy(ntag, tag, tag_len); - ntag[tag_len] = '\0'; - - work_data = (const char *) data; - work_size = bytes; - -#ifdef FLB_HAVE_METRICS - /* timestamp */ - ts = cfl_time_now(); - - /* Count number of incoming records */ - in_records = ic->added_records; - pre_records = ic->total_records - in_records; -#endif - - /* Iterate filters */ - mk_list_foreach(head, &config->filters) { - f_ins = mk_list_entry(head, struct flb_filter_instance, _head); - if (is_active(&f_ins->properties) == FLB_FALSE) { - continue; - } - if (flb_router_match(ntag, tag_len, f_ins->match -#ifdef FLB_HAVE_REGEX - , f_ins->match_regex -#else - , NULL -#endif - )) { - /* Reset filtered buffer */ - out_buf = NULL; - out_size = 0; - - content_size = cio_chunk_get_content_size(ic->chunk); - - /* where to position the new content if modified ? */ - write_at = (content_size - work_size); - -#ifdef FLB_HAVE_CHUNK_TRACE - if (ic->trace) { - flb_time_get(&tm_start); - } -#endif /* FLB_HAVE_CHUNK_TRACE */ - /* Invoke the filter callback */ - ret = f_ins->p->cb_filter(work_data, /* msgpack buffer */ - work_size, /* msgpack size */ - ntag, tag_len, /* input tag */ - &out_buf, /* new data */ - &out_size, /* new data size */ - f_ins, /* filter instance */ - i_ins, /* input instance */ - f_ins->context, /* filter priv data */ - config); -#ifdef FLB_HAVE_CHUNK_TRACE - if (ic->trace) { - flb_time_get(&tm_finish); - } -#endif /* FLB_HAVE_CHUNK_TRACE */ - -#ifdef FLB_HAVE_METRICS - name = (char *) flb_filter_name(f_ins); - - cmt_counter_add(f_ins->cmt_records, ts, in_records, - 1, (char *[]) {name}); - cmt_counter_add(f_ins->cmt_bytes, ts, content_size, - 1, (char *[]) {name}); - - flb_metrics_sum(FLB_METRIC_N_RECORDS, in_records, f_ins->metrics); - flb_metrics_sum(FLB_METRIC_N_BYTES, content_size, f_ins->metrics); -#endif - - /* Override buffer just if it was modified */ - if (ret == FLB_FILTER_MODIFIED) { - /* all records removed, no data to continue processing */ - if (out_size == 0) { - /* reset data content length */ - flb_input_chunk_write_at(ic, write_at, "", 0); -#ifdef FLB_HAVE_CHUNK_TRACE - if (ic->trace) { - flb_chunk_trace_filter(ic->trace, (void *)f_ins, &tm_start, &tm_finish, "", 0); - } -#endif /* FLB_HAVE_CHUNK_TRACE */ - - -#ifdef FLB_HAVE_METRICS - ic->total_records = pre_records; - - /* cmetrics */ - cmt_counter_add(f_ins->cmt_drop_records, ts, in_records, - 1, (char *[]) {name}); - - /* [OLD] Summarize all records removed */ - flb_metrics_sum(FLB_METRIC_N_DROPPED, - in_records, f_ins->metrics); -#endif - break; - } - else { -#ifdef FLB_HAVE_METRICS - out_records = flb_mp_count(out_buf, out_size); - if (out_records > in_records) { - diff = (out_records - in_records); - - /* cmetrics */ - cmt_counter_add(f_ins->cmt_add_records, ts, diff, - 1, (char *[]) {name}); - - /* [OLD] Summarize new records */ - flb_metrics_sum(FLB_METRIC_N_ADDED, - diff, f_ins->metrics); - } - else if (out_records < in_records) { - diff = (in_records - out_records); - - /* cmetrics */ - cmt_counter_add(f_ins->cmt_drop_records, ts, diff, - 1, (char *[]) {name}); - - /* [OLD] Summarize dropped records */ - flb_metrics_sum(FLB_METRIC_N_DROPPED, - diff, f_ins->metrics); - } - - /* set number of records in new chunk */ - in_records = out_records; - ic->total_records = pre_records + in_records; -#endif - } - ret = flb_input_chunk_write_at(ic, write_at, - out_buf, out_size); - if (ret == -1) { - flb_error("[filter] could not write data to storage. " - "Skipping filtering."); - flb_free(out_buf); - continue; - } - -#ifdef FLB_HAVE_CHUNK_TRACE - if (ic->trace) { - flb_chunk_trace_filter(ic->trace, (void *)f_ins, &tm_start, &tm_finish, out_buf, out_size); - } -#endif /* FLB_HAVE_CHUNK_TRACE */ - - /* Point back the 'data' pointer to the new address */ - ret = cio_chunk_get_content(ic->chunk, - (char **) &work_data, &cur_size); - if (ret != CIO_OK) { - flb_error("[filter] error retrieving data chunk"); - } - else { - work_data += (cur_size - out_size); - work_size = out_size; - } - flb_free(out_buf); - } - } - } - - flb_free(ntag); -} - -int flb_filter_set_property(struct flb_filter_instance *ins, - const char *k, const char *v) -{ - int len; - int ret; - flb_sds_t tmp; - struct flb_kv *kv; - - len = strlen(k); - tmp = flb_env_var_translate(ins->config->env, v); - if (!tmp) { - return -1; - } - - /* Check if the key is a known/shared property */ -#ifdef FLB_HAVE_REGEX - if (prop_key_check("match_regex", k, len) == 0) { - ins->match_regex = flb_regex_create(tmp); - flb_sds_destroy(tmp); - } - else -#endif - if (prop_key_check("match", k, len) == 0) { - flb_utils_set_plugin_string_property("match", &ins->match, tmp); - } - else if (prop_key_check("alias", k, len) == 0 && tmp) { - flb_utils_set_plugin_string_property("alias", &ins->alias, tmp); - } - else if (prop_key_check("log_level", k, len) == 0 && tmp) { - ret = flb_log_get_level_str(tmp); - flb_sds_destroy(tmp); - if (ret == -1) { - return -1; - } - ins->log_level = ret; - } - else if (prop_key_check("log_suppress_interval", k, len) == 0 && tmp) { - ret = flb_utils_time_to_seconds(tmp); - flb_sds_destroy(tmp); - if (ret == -1) { - return -1; - } - ins->log_suppress_interval = ret; - } - else { - /* - * Create the property, we don't pass the value since we will - * map it directly to avoid an extra memory allocation. - */ - kv = flb_kv_item_create(&ins->properties, (char *) k, NULL); - if (!kv) { - if (tmp) { - flb_sds_destroy(tmp); - } - return -1; - } - kv->val = tmp; - } - - return 0; -} - -const char *flb_filter_get_property(const char *key, - struct flb_filter_instance *ins) -{ - return flb_kv_get_key_value(key, &ins->properties); -} - -void flb_filter_instance_exit(struct flb_filter_instance *ins, - struct flb_config *config) -{ - struct flb_filter_plugin *p; - - p = ins->p; - if (p->cb_exit && ins->context) { - p->cb_exit(ins->context, config); - } -} - -/* Invoke exit call for the filter plugin */ -void flb_filter_exit(struct flb_config *config) -{ - struct mk_list *tmp; - struct mk_list *head; - struct flb_filter_instance *ins; - struct flb_filter_plugin *p; - - mk_list_foreach_safe(head, tmp, &config->filters) { - ins = mk_list_entry(head, struct flb_filter_instance, _head); - p = ins->p; - if (!p) { - continue; - } - flb_filter_instance_exit(ins, config); - flb_filter_instance_destroy(ins); - } -} - -struct flb_filter_instance *flb_filter_new(struct flb_config *config, - const char *filter, void *data) -{ - int id; - struct mk_list *head; - struct flb_filter_plugin *plugin; - struct flb_filter_instance *instance = NULL; - - if (!filter) { - return NULL; - } - - mk_list_foreach(head, &config->filter_plugins) { - plugin = mk_list_entry(head, struct flb_filter_plugin, _head); - if (strcasecmp(plugin->name, filter) == 0) { - break; - } - plugin = NULL; - } - - if (!plugin) { - return NULL; - } - - instance = flb_calloc(1, sizeof(struct flb_filter_instance)); - if (!instance) { - flb_errno(); - return NULL; - } - instance->config = config; - - /* - * Initialize event type, if not set, default to FLB_FILTER_LOGS. Note that a - * zero value means it's undefined. - */ - if (plugin->event_type == 0) { - instance->event_type = FLB_FILTER_LOGS; - } - else { - instance->event_type = plugin->event_type; - } - - /* Get an ID */ - id = instance_id(config); - - /* format name (with instance id) */ - snprintf(instance->name, sizeof(instance->name) - 1, - "%s.%i", plugin->name, id); - - instance->id = id; - instance->alias = NULL; - instance->p = plugin; - instance->data = data; - instance->match = NULL; -#ifdef FLB_HAVE_REGEX - instance->match_regex = NULL; -#endif - instance->log_level = -1; - instance->log_suppress_interval = -1; - - mk_list_init(&instance->properties); - mk_list_add(&instance->_head, &config->filters); - - return instance; -} - -/* Return an instance name or alias */ -const char *flb_filter_name(struct flb_filter_instance *ins) -{ - if (ins->alias) { - return ins->alias; - } - - return ins->name; -} - -int flb_filter_plugin_property_check(struct flb_filter_instance *ins, - struct flb_config *config) -{ - int ret = 0; - struct mk_list *config_map; - struct flb_filter_plugin *p = ins->p; - - if (p->config_map) { - /* - * Create a dynamic version of the configmap that will be used by the specific - * instance in question. - */ - config_map = flb_config_map_create(config, p->config_map); - if (!config_map) { - flb_error("[filter] error loading config map for '%s' plugin", - p->name); - return -1; - } - ins->config_map = config_map; - - /* Validate incoming properties against config map */ - ret = flb_config_map_properties_check(ins->p->name, - &ins->properties, ins->config_map); - if (ret == -1) { - if (config->program_name) { - flb_helper("try the command: %s -F %s -h\n", - config->program_name, ins->p->name); - } - return -1; - } - } - - return 0; -} - -int flb_filter_match_property_existence(struct flb_filter_instance *ins) -{ - if (!ins->match -#ifdef FLB_HAVE_REGEX - && !ins->match_regex -#endif - ) { - return FLB_FALSE; - } - - return FLB_TRUE; -} - -int flb_filter_init(struct flb_config *config, struct flb_filter_instance *ins) -{ - int ret; - uint64_t ts; - char *name; - struct flb_filter_plugin *p; - - if (flb_filter_match_property_existence(ins) == FLB_FALSE) { - flb_warn("[filter] NO match rule for %s filter instance, unloading.", - ins->name); - return -1; - } - - if (ins->log_level == -1 && config->log) { - ins->log_level = config->log->level; - } - - p = ins->p; - - /* Get name or alias for the instance */ - name = (char *) flb_filter_name(ins); - ts = cfl_time_now(); - - /* CMetrics */ - ins->cmt = cmt_create(); - if (!ins->cmt) { - flb_error("[filter] could not create cmetrics context: %s", - flb_filter_name(ins)); - return -1; - } - - /* Register generic filter plugin metrics */ - ins->cmt_records = cmt_counter_create(ins->cmt, - "fluentbit", "filter", - "records_total", - "Total number of new records processed.", - 1, (char *[]) {"name"}); - cmt_counter_set(ins->cmt_records, ts, 0, 1, (char *[]) {name}); - - /* Register generic filter plugin metrics */ - ins->cmt_bytes = cmt_counter_create(ins->cmt, - "fluentbit", "filter", - "bytes_total", - "Total number of new bytes processed.", - 1, (char *[]) {"name"}); - cmt_counter_set(ins->cmt_bytes, ts, 0, 1, (char *[]) {name}); - - /* Register generic filter plugin metrics */ - ins->cmt_add_records = cmt_counter_create(ins->cmt, - "fluentbit", "filter", - "add_records_total", - "Total number of new added records.", - 1, (char *[]) {"name"}); - cmt_counter_set(ins->cmt_add_records, ts, 0, 1, (char *[]) {name}); - - /* Register generic filter plugin metrics */ - ins->cmt_drop_records = cmt_counter_create(ins->cmt, - "fluentbit", "filter", - "drop_records_total", - "Total number of dropped records.", - 1, (char *[]) {"name"}); - cmt_counter_set(ins->cmt_drop_records, ts, 0, 1, (char *[]) {name}); - - /* OLD Metrics API */ -#ifdef FLB_HAVE_METRICS - - /* Create the metrics context */ - ins->metrics = flb_metrics_create(name); - if (!ins->metrics) { - flb_warn("[filter] cannot initialize metrics for %s filter, " - "unloading.", name); - return -1; - } - - /* Register filter metrics */ - flb_metrics_add(FLB_METRIC_N_DROPPED, "drop_records", ins->metrics); - flb_metrics_add(FLB_METRIC_N_ADDED, "add_records", ins->metrics); - flb_metrics_add(FLB_METRIC_N_RECORDS, "records", ins->metrics); - flb_metrics_add(FLB_METRIC_N_BYTES, "bytes", ins->metrics); -#endif - - /* - * Before to call the initialization callback, make sure that the received - * configuration parameters are valid if the plugin is registering a config map. - */ - if (flb_filter_plugin_property_check(ins, config) == -1) { - return -1; - } - - if (is_active(&ins->properties) == FLB_FALSE) { - return 0; - } - - /* Initialize the input */ - if (p->cb_init) { - ret = p->cb_init(ins, config, ins->data); - if (ret != 0) { - flb_error("Failed initialize filter %s", ins->name); - return -1; - } - } - - return 0; -} - -/* Initialize all filter plugins */ -int flb_filter_init_all(struct flb_config *config) -{ - int ret; - struct mk_list *tmp; - struct mk_list *head; - struct flb_filter_instance *ins; - - /* Iterate all active filter instance plugins */ - mk_list_foreach_safe(head, tmp, &config->filters) { - ins = mk_list_entry(head, struct flb_filter_instance, _head); - ret = flb_filter_init(config, ins); - if (ret == -1) { - flb_filter_instance_destroy(ins); - return -1; - } - } - - return 0; -} - -void flb_filter_instance_destroy(struct flb_filter_instance *ins) -{ - if (!ins) { - return; - } - - /* destroy config map */ - if (ins->config_map) { - flb_config_map_destroy(ins->config_map); - } - - /* release properties */ - flb_kv_release(&ins->properties); - - if (ins->match != NULL) { - flb_sds_destroy(ins->match); - } - -#ifdef FLB_HAVE_REGEX - if (ins->match_regex) { - flb_regex_destroy(ins->match_regex); - } -#endif - - /* Remove metrics */ -#ifdef FLB_HAVE_METRICS - if (ins->cmt) { - cmt_destroy(ins->cmt); - } - - if (ins->metrics) { - flb_metrics_destroy(ins->metrics); - } -#endif - if (ins->alias) { - flb_sds_destroy(ins->alias); - } - - mk_list_del(&ins->_head); - flb_free(ins); -} - -void flb_filter_set_context(struct flb_filter_instance *ins, void *context) -{ - ins->context = context; -} diff --git a/fluent-bit/src/flb_fstore.c b/fluent-bit/src/flb_fstore.c deleted file mode 100644 index 03bcc99db..000000000 --- a/fluent-bit/src/flb_fstore.c +++ /dev/null @@ -1,558 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <fluent-bit/flb_info.h> -#include <fluent-bit/flb_fstore.h> -#include <fluent-bit/flb_log.h> -#include <fluent-bit/flb_mem.h> -#include <fluent-bit/flb_sds.h> -#include <chunkio/chunkio.h> - -static int log_cb(struct cio_ctx *ctx, int level, const char *file, int line, - char *str) -{ - if (level == CIO_LOG_ERROR) { - flb_error("[fstore] %s", str); - } - else if (level == CIO_LOG_WARN) { - flb_warn("[fstore] %s", str); - } - else if (level == CIO_LOG_INFO) { - flb_info("[fstore] %s", str); - } - else if (level == CIO_LOG_DEBUG) { - flb_debug("[fstore] %s", str); - } - - return 0; -} - -/* - * this function sets metadata into a fstore_file structure, note that it makes - * it own copy of the data to set a NULL byte at the end. - */ -static int meta_set(struct flb_fstore_file *fsf, void *meta, size_t size) -{ - - char *p; - - p = flb_calloc(1, size + 1); - if (!p) { - flb_errno(); - flb_error("[fstore] could not cache metadata in file: %s:%s", - fsf->stream->name, fsf->chunk->name); - return -1; - } - - if (fsf->meta_buf) { - flb_free(fsf->meta_buf); - } - fsf->meta_buf = p; - memcpy(fsf->meta_buf, meta, size); - fsf->meta_size = size; - - return 0; -} - -/* Set a file metadata */ -int flb_fstore_file_meta_set(struct flb_fstore *fs, - struct flb_fstore_file *fsf, - void *meta, size_t size) -{ - int ret; - int set_down = FLB_FALSE; - - /* Check if the chunk is up */ - if (cio_chunk_is_up(fsf->chunk) == CIO_FALSE) { - ret = cio_chunk_up_force(fsf->chunk); - if (ret != CIO_OK) { - flb_error("[fstore] error loading up file chunk"); - return -1; - } - set_down = FLB_TRUE; - } - - ret = cio_meta_write(fsf->chunk, meta, size); - if (ret == -1) { - flb_error("[fstore] could not write metadata to file: %s:%s", - fsf->stream->name, fsf->chunk->name); - - if (set_down == FLB_TRUE) { - cio_chunk_down(fsf->chunk); - } - - return -1; - } - - if (set_down == FLB_TRUE) { - cio_chunk_down(fsf->chunk); - } - - return meta_set(fsf, meta, size); -} - -/* Re-read Chunk I/O metadata into fstore file */ -int flb_fstore_file_meta_get(struct flb_fstore *fs, - struct flb_fstore_file *fsf) -{ - int ret; - int set_down = FLB_FALSE; - char *meta_buf = NULL; - int meta_size = 0; - - /* Check if the chunk is up */ - if (cio_chunk_is_up(fsf->chunk) == CIO_FALSE) { - ret = cio_chunk_up_force(fsf->chunk); - if (ret != CIO_OK) { - flb_error("[fstore] error loading up file chunk"); - return -1; - } - set_down = FLB_TRUE; - } - - ret = cio_meta_read(fsf->chunk, &meta_buf, &meta_size); - if (ret == -1) { - flb_error("[fstore] error reading file chunk metadata"); - if (set_down == FLB_TRUE) { - cio_chunk_down(fsf->chunk); - } - } - - ret = meta_set(fsf, meta_buf, meta_size); - if (ret == -1) { - flb_free(meta_buf); - if (set_down == FLB_TRUE) { - cio_chunk_down(fsf->chunk); - } - return -1; - } - - if (set_down == FLB_TRUE) { - cio_chunk_down(fsf->chunk); - } - return 0; -} - -/* Create a new file */ -struct flb_fstore_file *flb_fstore_file_create(struct flb_fstore *fs, - struct flb_fstore_stream *fs_stream, - char *name, size_t size) -{ - int err; - struct cio_chunk *chunk; - struct flb_fstore_file *fsf; - - fsf = flb_calloc(1, sizeof(struct flb_fstore_file)); - if (!fsf) { - flb_errno(); - return NULL; - } - fsf->stream = fs_stream->stream; - - fsf->name = flb_sds_create(name); - if (!fsf->name) { - flb_error("[fstore] could not create file: %s:%s", - fsf->stream->name, name); - flb_free(fsf); - return NULL; - } - - chunk = cio_chunk_open(fs->cio, fs_stream->stream, name, - CIO_OPEN, size, &err); - if (!chunk) { - flb_error("[fstore] could not create file: %s:%s", - fsf->stream->name, name); - flb_sds_destroy(fsf->name); - flb_free(fsf); - return NULL; - } - - fsf->chunk = chunk; - mk_list_add(&fsf->_head, &fs_stream->files); - - return fsf; -} - -/* Lookup file on stream by using it name */ -struct flb_fstore_file *flb_fstore_file_get(struct flb_fstore *fs, - struct flb_fstore_stream *fs_stream, - char *name, size_t size) -{ - struct mk_list *head; - struct flb_fstore_file *fsf; - - mk_list_foreach(head, &fs_stream->files) { - fsf = mk_list_entry(head, struct flb_fstore_file, _head); - if (flb_sds_len(fsf->name) != size) { - continue; - } - - if (strncmp(fsf->name, name, size) == 0) { - return fsf; - } - } - - return NULL; -} - -/* - * Set a file to inactive mode. Inactive means just to remove the reference - * from the list. - */ -int flb_fstore_file_inactive(struct flb_fstore *fs, - struct flb_fstore_file *fsf) -{ - /* close the Chunk I/O reference, but don't delete the real file */ - if (fsf->chunk) { - cio_chunk_close(fsf->chunk, CIO_FALSE); - } - - /* release */ - mk_list_del(&fsf->_head); - flb_sds_destroy(fsf->name); - if (fsf->meta_buf) { - flb_free(fsf->meta_buf); - } - flb_free(fsf); - - return 0; -} - -/* Delete a file (permantent deletion) */ -int flb_fstore_file_delete(struct flb_fstore *fs, - struct flb_fstore_file *fsf) -{ - /* close the Chunk I/O reference, but don't delete it the real file */ - cio_chunk_close(fsf->chunk, CIO_TRUE); - - /* release */ - mk_list_del(&fsf->_head); - if (fsf->meta_buf) { - flb_free(fsf->meta_buf); - } - flb_sds_destroy(fsf->name); - flb_free(fsf); - - return 0; -} - -/* - * Set an output buffer that contains a copy of the file. Note that this buffer - * needs to be freed by the caller (heap memory). - */ -int flb_fstore_file_content_copy(struct flb_fstore *fs, - struct flb_fstore_file *fsf, - void **out_buf, size_t *out_size) -{ - int ret; - - ret = cio_chunk_get_content_copy(fsf->chunk, out_buf, out_size); - if (ret == CIO_OK) { - return 0; - } - - return -1; -} - -/* Append data to an existing file */ -int flb_fstore_file_append(struct flb_fstore_file *fsf, void *data, size_t size) -{ - int ret; - int set_down = FLB_FALSE; - - /* Check if the chunk is up */ - if (cio_chunk_is_up(fsf->chunk) == CIO_FALSE) { - ret = cio_chunk_up_force(fsf->chunk); - if (ret != CIO_OK) { - flb_error("[fstore] error loading up file chunk"); - return -1; - } - set_down = FLB_TRUE; - } - - ret = cio_chunk_write(fsf->chunk, data, size); - if (ret != CIO_OK) { - flb_error("[fstore] could not write data to file %s", fsf->name); - - if (set_down == FLB_TRUE) { - cio_chunk_down(fsf->chunk); - } - - return -1; - } - - if (set_down == FLB_TRUE) { - cio_chunk_down(fsf->chunk); - } - - return 0; -} - -/* - * Create a new stream, if it already exists, it returns the stream - * reference. - */ -struct flb_fstore_stream *flb_fstore_stream_create(struct flb_fstore *fs, - char *stream_name) -{ - flb_sds_t path = NULL; - struct mk_list *head; - struct cio_ctx *ctx = NULL; - struct cio_stream *stream = NULL; - struct flb_fstore_stream *fs_stream = NULL; - - ctx = fs->cio; - - /* Check if the stream already exists in Chunk I/O */ - mk_list_foreach(head, &ctx->streams) { - stream = mk_list_entry(head, struct cio_stream, _head); - if (strcmp(stream->name, stream_name) == 0) { - break; - } - stream = NULL; - } - - /* If the stream exists, check if we have a fstore_stream reference */ - if (stream) { - mk_list_foreach(head, &fs->streams) { - fs_stream = mk_list_entry(head, struct flb_fstore_stream, _head); - if (fs_stream->stream == stream) { - break; - } - fs_stream = NULL; - } - - /* The stream was found, just return the reference */ - if (fs_stream) { - return fs_stream; - } - } - - if (!stream) { - /* create file-system based stream */ - stream = cio_stream_create(fs->cio, stream_name, fs->store_type); - if (!stream) { - flb_error("[fstore] cannot create stream %s", stream_name); - return NULL; - } - } - - fs_stream = flb_calloc(1, sizeof(struct flb_fstore_stream)); - if (!fs_stream) { - flb_errno(); - cio_stream_destroy(stream); - return NULL; - } - fs_stream->stream = stream; - - path = flb_sds_create_size(256); - if (!path) { - cio_stream_destroy(stream); - flb_free(fs_stream); - return NULL; - } - path = flb_sds_printf(&path, "%s/%s", fs->root_path, stream->name); - fs_stream->path = path; - fs_stream->name = stream->name; - - mk_list_init(&fs_stream->files); - mk_list_add(&fs_stream->_head, &fs->streams); - - return fs_stream; -} - -void flb_fstore_stream_destroy(struct flb_fstore_stream *stream, int delete) -{ - if (delete == FLB_TRUE) { - cio_stream_delete(stream->stream); - } - - /* - * FYI: in this function we just release the fstore_stream context, the - * underlaying cio_stream is closed when the main Chunk I/O is destroyed. - */ - mk_list_del(&stream->_head); - flb_sds_destroy(stream->path); - flb_free(stream); -} - -static int map_chunks(struct flb_fstore *ctx, struct flb_fstore_stream *fs_stream, - struct cio_stream *stream) -{ - struct mk_list *head; - struct cio_chunk *chunk; - struct flb_fstore_file *fsf; - - mk_list_foreach(head, &stream->chunks) { - chunk = mk_list_entry(head, struct cio_chunk, _head); - - fsf = flb_calloc(1, sizeof(struct flb_fstore_file)); - if (!fsf) { - flb_errno(); - return -1; - } - fsf->name = flb_sds_create(chunk->name); - if (!fsf->name) { - flb_free(fsf); - flb_error("[fstore] could not create file: %s:%s", - stream->name, chunk->name); - return -1; - } - - fsf->chunk = chunk; - - /* load metadata */ - flb_fstore_file_meta_get(ctx, fsf); - mk_list_add(&fsf->_head, &fs_stream->files); - } - - return 0; -} - -static int load_references(struct flb_fstore *fs) -{ - int ret; - struct mk_list *head; - struct cio_stream *stream; - struct flb_fstore_stream *fs_stream; - - mk_list_foreach(head, &fs->cio->streams) { - stream = mk_list_entry(head, struct cio_stream, _head); - fs_stream = flb_fstore_stream_create(fs, stream->name); - if (!fs_stream) { - flb_error("[fstore] error loading stream reference: %s", - stream->name); - return -1; - } - - /* Map chunks */ - ret = map_chunks(fs, fs_stream, stream); - if (ret == -1) { - return -1; - } - } - - return 0; -} - -struct flb_fstore *flb_fstore_create(char *path, int store_type) -{ - int ret; - int flags; - struct cio_ctx *cio; - struct flb_fstore *fs; - struct cio_options opts = {0}; - flags = CIO_OPEN; - - /* Create Chunk I/O context */ - cio_options_init(&opts); - - opts.root_path = path; - opts.log_cb = log_cb; - opts.flags = flags; - opts.log_level = CIO_LOG_INFO; - - cio = cio_create(&opts); - if (!cio) { - flb_error("[fstore] error initializing on path '%s'", path); - return NULL; - } - - /* Load content from the file system if any */ - ret = cio_load(cio, NULL); - if (ret == -1) { - flb_error("[fstore] error scanning root path content: %s", path); - cio_destroy(cio); - return NULL; - } - - fs = flb_calloc(1, sizeof(struct flb_fstore)); - if (!fs) { - flb_errno(); - cio_destroy(cio); - return NULL; - } - fs->cio = cio; - fs->root_path = cio->options.root_path; - fs->store_type = store_type; - mk_list_init(&fs->streams); - - /* Map Chunk I/O streams and chunks into fstore context */ - load_references(fs); - - return fs; -} - -int flb_fstore_destroy(struct flb_fstore *fs) -{ - int files = 0; - int delete; - struct mk_list *head; - struct mk_list *f_head; - struct mk_list *tmp; - struct mk_list *f_tmp; - struct flb_fstore_stream *fs_stream; - struct flb_fstore_file *fsf; - - mk_list_foreach_safe(head, tmp, &fs->streams) { - fs_stream = mk_list_entry(head, struct flb_fstore_stream, _head); - - /* delete file references */ - files = 0; - mk_list_foreach_safe(f_head, f_tmp, &fs_stream->files) { - fsf = mk_list_entry(f_head, struct flb_fstore_file, _head); - flb_fstore_file_inactive(fs, fsf); - files++; - } - - if (files == 0) { - delete = FLB_TRUE; - } - else { - delete = FLB_FALSE; - } - - flb_fstore_stream_destroy(fs_stream, delete); - } - - if (fs->cio) { - cio_destroy(fs->cio); - } - flb_free(fs); - return 0; -} - -void flb_fstore_dump(struct flb_fstore *fs) -{ - struct mk_list *head; - struct mk_list *f_head; - struct flb_fstore_stream *fs_stream; - struct flb_fstore_file *fsf; - - printf("===== FSTORE DUMP =====\n"); - mk_list_foreach(head, &fs->streams) { - fs_stream = mk_list_entry(head, struct flb_fstore_stream, _head); - printf("- stream: %s\n", fs_stream->name); - mk_list_foreach(f_head, &fs_stream->files) { - fsf = mk_list_entry(f_head, struct flb_fstore_file, _head); - printf(" %s/%s\n", fsf->stream->name, fsf->name); - } - } - printf("\n"); -} diff --git a/fluent-bit/src/flb_gzip.c b/fluent-bit/src/flb_gzip.c deleted file mode 100644 index 4b9b761fe..000000000 --- a/fluent-bit/src/flb_gzip.c +++ /dev/null @@ -1,747 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <fluent-bit/flb_info.h> -#include <fluent-bit/flb_mem.h> -#include <fluent-bit/flb_log.h> -#include <fluent-bit/flb_gzip.h> -#include <fluent-bit/flb_compression.h> -#include <miniz/miniz.h> - -#define FLB_GZIP_HEADER_OFFSET 10 -#define FLB_GZIP_HEADER_SIZE FLB_GZIP_HEADER_OFFSET - -#define FLB_GZIP_MAGIC_NUMBER 0x8B1F - -typedef enum { - FTEXT = 1, - FHCRC = 2, - FEXTRA = 4, - FNAME = 8, - FCOMMENT = 16 -} flb_tinf_gzip_flag; - -#pragma pack(push, 1) -struct flb_gzip_header { - uint16_t magic_number; - uint8_t compression_method; - uint8_t header_flags; - uint32_t timestamp; - uint8_t compression_flags; - uint8_t operating_system_id; -}; -#pragma pack(pop) - -struct flb_gzip_decompression_context { - struct flb_gzip_header gzip_header; - mz_stream miniz_stream; -}; - -static unsigned int read_le16(const unsigned char *p) -{ - return ((unsigned int) p[0]) | ((unsigned int) p[1] << 8); -} - -static unsigned int read_le32(const unsigned char *p) -{ - return ((unsigned int) p[0]) - | ((unsigned int) p[1] << 8) - | ((unsigned int) p[2] << 16) - | ((unsigned int) p[3] << 24); -} - -static inline void gzip_header(void *buf) -{ - uint8_t *p; - - /* GZip Magic bytes */ - p = buf; - *p++ = 0x1F; - *p++ = 0x8B; - *p++ = 8; - *p++ = 0; - *p++ = 0; - *p++ = 0; - *p++ = 0; - *p++ = 0; - *p++ = 0; - *p++ = 0xFF; -} - - -#include <ctype.h> - -static inline void flb_hex_dump(uint8_t *buffer, size_t buffer_length, size_t line_length) { - char *printable_line; - size_t buffer_index; - size_t filler_index; - - if (40 < line_length) - { - line_length = 40; - } - - printable_line = alloca(line_length + 1); - - if (NULL == printable_line) - { - printf("Alloca returned NULL\n"); - - return; - } - - memset(printable_line, '\0', line_length + 1); - - for (buffer_index = 0 ; buffer_index < buffer_length ; buffer_index++) { - if (0 != buffer_index && - 0 == (buffer_index % line_length)) { - - printf("%s\n", printable_line); - - memset(printable_line, '\0', line_length + 1); - } - - if (0 != isprint(buffer[buffer_index])) { - printable_line[(buffer_index % line_length)] = buffer[buffer_index]; - } - else { - printable_line[(buffer_index % line_length)] = '.'; - } - - printf("%02X ", buffer[buffer_index]); - } - - if (0 != buffer_index && - 0 != (buffer_index % line_length)) { - - for (filler_index = 0 ; - filler_index < (line_length - (buffer_index % line_length)) ; - filler_index++) { - printf(" "); - } - - printf("%s\n", printable_line); - - memset(printable_line, '.', line_length); - } -} - - -int flb_gzip_compress(void *in_data, size_t in_len, - void **out_data, size_t *out_len) -{ - int flush; - int status; - int footer_start; - uint8_t *pb; - size_t out_size; - void *out_buf; - z_stream strm; - mz_ulong crc; - - /* - * Calculating the upper bound for a gzip compression is - * non-trivial, so we rely on miniz's own calculation - * to guarantee memory safety. - */ - out_size = compressBound(in_len); - out_buf = flb_malloc(out_size); - - if (!out_buf) { - flb_errno(); - flb_error("[gzip] could not allocate outgoing buffer"); - return -1; - } - - /* Initialize streaming buffer context */ - memset(&strm, '\0', sizeof(strm)); - strm.zalloc = Z_NULL; - strm.zfree = Z_NULL; - strm.opaque = Z_NULL; - strm.next_in = in_data; - strm.avail_in = in_len; - strm.total_out = 0; - - /* Deflate mode */ - deflateInit2(&strm, Z_DEFAULT_COMPRESSION, - Z_DEFLATED, -Z_DEFAULT_WINDOW_BITS, 9, Z_DEFAULT_STRATEGY); - - /* - * Miniz don't support GZip format directly, instead we will: - * - * - append manual GZip magic bytes - * - deflate raw content - * - append manual CRC32 data - */ - gzip_header(out_buf); - - /* Header offset */ - pb = (uint8_t *) out_buf + FLB_GZIP_HEADER_OFFSET; - - flush = Z_NO_FLUSH; - while (1) { - strm.next_out = pb + strm.total_out; - strm.avail_out = out_size - (pb - (uint8_t *) out_buf); - - if (strm.avail_in == 0) { - flush = Z_FINISH; - } - - status = deflate(&strm, flush); - if (status == Z_STREAM_END) { - break; - } - else if (status != Z_OK) { - deflateEnd(&strm); - return -1; - } - } - - if (deflateEnd(&strm) != Z_OK) { - flb_free(out_buf); - return -1; - } - *out_len = strm.total_out; - - /* Construct the gzip checksum (CRC32 footer) */ - footer_start = FLB_GZIP_HEADER_OFFSET + *out_len; - pb = (uint8_t *) out_buf + footer_start; - - crc = mz_crc32(MZ_CRC32_INIT, in_data, in_len); - *pb++ = crc & 0xFF; - *pb++ = (crc >> 8) & 0xFF; - *pb++ = (crc >> 16) & 0xFF; - *pb++ = (crc >> 24) & 0xFF; - *pb++ = in_len & 0xFF; - *pb++ = (in_len >> 8) & 0xFF; - *pb++ = (in_len >> 16) & 0xFF; - *pb++ = (in_len >> 24) & 0xFF; - - /* Set the real buffer size for the caller */ - *out_len += FLB_GZIP_HEADER_OFFSET + 8; - *out_data = out_buf; - - return 0; -} - -/* Uncompress (inflate) GZip data */ -int flb_gzip_uncompress(void *in_data, size_t in_len, - void **out_data, size_t *out_len) -{ - int status; - uint8_t *p; - void *out_buf; - size_t out_size = 0; - void *zip_data; - size_t zip_len; - unsigned char flg; - unsigned int xlen, hcrc; - unsigned int dlen, crc; - mz_ulong crc_out; - mz_stream stream; - const unsigned char *start; - - /* Minimal length: header + crc32 */ - if (in_len < 18) { - flb_error("[gzip] unexpected content length"); - return -1; - } - - /* Magic bytes */ - p = in_data; - if (p[0] != 0x1F || p[1] != 0x8B) { - flb_error("[gzip] invalid magic bytes"); - return -1; - } - - if (p[2] != 8) { - flb_error("[gzip] invalid method"); - return -1; - } - - /* Flag byte */ - flg = p[3]; - - /* Reserved bits */ - if (flg & 0xE0) { - flb_error("[gzip] invalid flag"); - return -1; - } - - /* Skip base header of 10 bytes */ - start = p + FLB_GZIP_HEADER_OFFSET; - - /* Skip extra data if present */ - if (flg & FEXTRA) { - xlen = read_le16(start); - if (xlen > in_len - 12) { - flb_error("[gzip] invalid gzip data"); - return -1; - } - start += xlen + 2; - } - - /* Skip file name if present */ - if (flg & FNAME) { - do { - if (start - p >= in_len) { - flb_error("[gzip] invalid gzip data (FNAME)"); - return -1; - } - } while (*start++); - } - - /* Skip file comment if present */ - if (flg & FCOMMENT) { - do { - if (start - p >= in_len) { - flb_error("[gzip] invalid gzip data (FCOMMENT)"); - return -1; - } - } while (*start++); - } - - /* Check header crc if present */ - if (flg & FHCRC) { - if (start - p > in_len - 2) { - flb_error("[gzip] invalid gzip data (FHRC)"); - return -1; - } - - hcrc = read_le16(start); - crc = mz_crc32(MZ_CRC32_INIT, p, start - p) & 0x0000FFFF; - if (hcrc != crc) { - flb_error("[gzip] invalid gzip header CRC"); - return -1; - } - start += 2; - } - - /* Get decompressed length */ - dlen = read_le32(&p[in_len - 4]); - - /* Limit decompressed length to 100MB */ - if (dlen > 100000000) { - flb_error("[gzip] maximum decompression size is 100MB"); - return -1; - } - - /* Get CRC32 checksum of original data */ - crc = read_le32(&p[in_len - 8]); - - /* Decompress data */ - if ((p + in_len) - p < 8) { - flb_error("[gzip] invalid gzip CRC32 checksum"); - return -1; - } - - /* Allocate outgoing buffer */ - out_buf = flb_malloc(dlen); - if (!out_buf) { - flb_errno(); - return -1; - } - out_size = dlen; - - /* Ensure size is above 0 */ - if (((p + in_len) - start - 8) <= 0) { - flb_free(out_buf); - return -1; - } - - /* Map zip content */ - zip_data = (uint8_t *) start; - zip_len = (p + in_len) - start - 8; - - memset(&stream, 0, sizeof(stream)); - stream.next_in = zip_data; - stream.avail_in = zip_len; - stream.next_out = out_buf; - stream.avail_out = out_size; - - status = mz_inflateInit2(&stream, -Z_DEFAULT_WINDOW_BITS); - if (status != MZ_OK) { - flb_free(out_buf); - return -1; - } - - status = mz_inflate(&stream, MZ_FINISH); - if (status != MZ_STREAM_END) { - mz_inflateEnd(&stream); - flb_free(out_buf); - return -1; - } - - if (stream.total_out != dlen) { - mz_inflateEnd(&stream); - flb_free(out_buf); - flb_error("[gzip] invalid gzip data size"); - return -1; - } - - /* terminate the stream, it's not longer required */ - mz_inflateEnd(&stream); - - /* Validate message CRC vs inflated data CRC */ - crc_out = mz_crc32(MZ_CRC32_INIT, out_buf, dlen); - if (crc_out != crc) { - flb_free(out_buf); - flb_error("[gzip] invalid GZip checksum (CRC32)"); - return -1; - } - - /* set the uncompressed data */ - *out_len = dlen; - *out_data = out_buf; - - return 0; -} - - -/* Stateful gzip decompressor */ - -static int flb_gzip_decompressor_process_header( - struct flb_decompression_context *context) -{ - struct flb_gzip_decompression_context *inner_context; - - inner_context = (struct flb_gzip_decompression_context *) \ - context->inner_context; - - /* Minimal length: header + crc32 */ - if (context->input_buffer_length < FLB_GZIP_HEADER_SIZE) { - flb_error("[gzip] unexpected content length"); - - return FLB_DECOMPRESSOR_FAILURE; - } - - memcpy(&inner_context->gzip_header, - context->read_buffer, - FLB_GZIP_HEADER_SIZE); - - context->read_buffer = &context->read_buffer[FLB_GZIP_HEADER_SIZE]; - context->input_buffer_length -= FLB_GZIP_HEADER_SIZE; - - /* Magic bytes */ - if (inner_context->gzip_header.magic_number != FLB_GZIP_MAGIC_NUMBER) { - context->state = FLB_DECOMPRESSOR_STATE_FAILED; - - flb_error("[gzip] invalid magic bytes : %04x", - inner_context->gzip_header.magic_number); - - return FLB_DECOMPRESSOR_FAILURE; - } - - if (inner_context->gzip_header.compression_method != MZ_DEFLATED) { - context->state = FLB_DECOMPRESSOR_STATE_FAILED; - - flb_error("[gzip] invalid method : %u", - inner_context->gzip_header.compression_method); - - return FLB_DECOMPRESSOR_FAILURE; - } - - /* Flag processing */ - /* Reserved bits */ - if (inner_context->gzip_header.header_flags & 0xE0) { - context->state = FLB_DECOMPRESSOR_STATE_FAILED; - - flb_error("[gzip] invalid flag mask : %x", - inner_context->gzip_header.header_flags); - - return FLB_DECOMPRESSOR_FAILURE; - } - - context->state = FLB_DECOMPRESSOR_STATE_EXPECTING_OPTIONAL_HEADERS; - - return FLB_DECOMPRESSOR_SUCCESS; -} - -static int flb_gzip_decompressor_process_optional_headers( - struct flb_decompression_context *context) -{ - struct flb_gzip_decompression_context *inner_context; - int status; - uint16_t hcrc; - uint16_t xlen; - uint16_t crc; - - inner_context = (struct flb_gzip_decompression_context *) \ - context->inner_context; - - /* Skip extra data if present */ - if (inner_context->gzip_header.header_flags & FEXTRA) { - if (context->input_buffer_length <= sizeof(uint16_t)) { - return FLB_DECOMPRESSOR_INSUFFICIENT_DATA; - } - - xlen = sizeof(uint16_t) + read_le16(context->read_buffer); - - if (context->input_buffer_length < xlen) { - return FLB_DECOMPRESSOR_INSUFFICIENT_DATA; - } - - context->read_buffer = &context->read_buffer[xlen]; - context->input_buffer_length -= xlen; - - inner_context->gzip_header.header_flags &= (~FEXTRA); - } - - if (inner_context->gzip_header.header_flags != 0 && - context->input_buffer_length == 0) { - return FLB_DECOMPRESSOR_INSUFFICIENT_DATA; - } - - /* Skip file name if present */ - if (inner_context->gzip_header.header_flags & FNAME) { - xlen = strnlen((char *) context->read_buffer, - context->input_buffer_length); - - if (xlen == 0 || - xlen == context->input_buffer_length) { - return FLB_DECOMPRESSOR_INSUFFICIENT_DATA; - } - - xlen++; - - context->read_buffer = &context->read_buffer[xlen]; - context->input_buffer_length -= xlen; - - inner_context->gzip_header.header_flags &= (~FNAME); - } - - if (inner_context->gzip_header.header_flags != 0 && - context->input_buffer_length == 0) { - return FLB_DECOMPRESSOR_INSUFFICIENT_DATA; - } - - /* Skip file comment if present */ - if (inner_context->gzip_header.header_flags & FCOMMENT) { - xlen = strnlen((char *) context->read_buffer, - context->input_buffer_length); - - if (xlen == 0 || - xlen == context->input_buffer_length) { - return FLB_DECOMPRESSOR_INSUFFICIENT_DATA; - } - - context->read_buffer = &context->read_buffer[xlen]; - context->input_buffer_length -= xlen; - - inner_context->gzip_header.header_flags &= (~FCOMMENT); - } - - if (inner_context->gzip_header.header_flags != 0 && - context->input_buffer_length == 0) { - return FLB_DECOMPRESSOR_INSUFFICIENT_DATA; - } - - /* Check header crc if present (lower 16 bits of the checksum)*/ - if (inner_context->gzip_header.header_flags & FHCRC) { - if (context->input_buffer_length <= sizeof(uint16_t)) { - return FLB_DECOMPRESSOR_INSUFFICIENT_DATA; - } - - hcrc = read_le16(context->read_buffer); - - crc = mz_crc32(MZ_CRC32_INIT, - (const unsigned char *) &inner_context->gzip_header, - FLB_GZIP_HEADER_SIZE); - - crc &= 0x0000FFFF; - - if (hcrc != crc) { - context->state = FLB_DECOMPRESSOR_STATE_FAILED; - - return FLB_DECOMPRESSOR_CORRUPTED_HEADER; - } - - xlen = sizeof(uint16_t); - - context->read_buffer = &context->read_buffer[xlen]; - context->input_buffer_length -= xlen; - - inner_context->gzip_header.header_flags &= (~FHCRC); - } - - status = mz_inflateInit2(&inner_context->miniz_stream, - -Z_DEFAULT_WINDOW_BITS); - - if (status != MZ_OK) { - context->state = FLB_DECOMPRESSOR_STATE_FAILED; - - return FLB_DECOMPRESSOR_FAILURE; - } - - context->state = FLB_DECOMPRESSOR_STATE_EXPECTING_BODY; - - return FLB_DECOMPRESSOR_SUCCESS; -} - -static int flb_gzip_decompressor_process_body_chunk( - struct flb_decompression_context *context, - void *output_buffer, - size_t *output_length) -{ - size_t processed_bytes; - struct flb_gzip_decompression_context *inner_context; - int status; - - if (*output_length == 0) { - return FLB_DECOMPRESSOR_SUCCESS; - } - - inner_context = (struct flb_gzip_decompression_context *) \ - context->inner_context; - - inner_context->miniz_stream.next_in = context->read_buffer; - inner_context->miniz_stream.avail_in = context->input_buffer_length; - inner_context->miniz_stream.next_out = output_buffer; - inner_context->miniz_stream.avail_out = *output_length; - - status = mz_inflate(&inner_context->miniz_stream, MZ_PARTIAL_FLUSH); - - if (status != MZ_OK && status != MZ_STREAM_END) { - context->state = FLB_DECOMPRESSOR_STATE_FAILED; - - mz_inflateEnd(&inner_context->miniz_stream); - - *output_length = 0; - - return FLB_DECOMPRESSOR_FAILURE; - } - - processed_bytes = context->input_buffer_length;; - processed_bytes -= inner_context->miniz_stream.avail_in; - - *output_length -= inner_context->miniz_stream.avail_out; - -#ifdef FLB_DECOMPRESSOR_ERASE_DECOMPRESSED_DATA - if (processed_bytes > 0) { - memset(context->read_buffer, processed_bytes); - } -#endif - - context->read_buffer = &context->read_buffer[processed_bytes]; - context->input_buffer_length = inner_context->miniz_stream.avail_in; - - if (status == MZ_STREAM_END) { - mz_inflateEnd(&inner_context->miniz_stream); - - context->state = FLB_DECOMPRESSOR_STATE_EXPECTING_FOOTER; - - memset(&inner_context->miniz_stream, 0, sizeof(mz_stream)); - } - - return FLB_DECOMPRESSOR_SUCCESS; -} - - -static int flb_gzip_decompressor_process_footer( - struct flb_decompression_context *context) -{ - if (context->input_buffer_length < (sizeof(uint32_t) * 2)) { - return FLB_DECOMPRESSOR_INSUFFICIENT_DATA; - } - - context->input_buffer_length -= (sizeof(uint32_t) * 2); - - if (context->input_buffer_length > 0) { - context->read_buffer = &context->read_buffer[sizeof(uint32_t) * 2]; - } - else { - context->read_buffer = context->input_buffer; - } - - context->state = FLB_DECOMPRESSOR_STATE_EXPECTING_HEADER; - - return FLB_DECOMPRESSOR_SUCCESS; -} - -int flb_gzip_decompressor_dispatch(struct flb_decompression_context *context, - void *output_buffer, - size_t *output_length) -{ - size_t output_buffer_size; - int status; - - output_buffer_size = *output_length; - - *output_length = 0; - - status = FLB_DECOMPRESSOR_SUCCESS; - - if (context == NULL || - context->inner_context == NULL) { - status = FLB_DECOMPRESSOR_FAILURE; - } - - if (context->input_buffer_length == 0) { - flb_debug("[gzip] unexpected call with an empty input buffer"); - - status = FLB_DECOMPRESSOR_INSUFFICIENT_DATA; - } - - if (status == FLB_DECOMPRESSOR_SUCCESS && - context->state == FLB_DECOMPRESSOR_STATE_EXPECTING_HEADER) { - status = flb_gzip_decompressor_process_header(context); - } - - if (status == FLB_DECOMPRESSOR_SUCCESS && - context->state == FLB_DECOMPRESSOR_STATE_EXPECTING_OPTIONAL_HEADERS) { - status = flb_gzip_decompressor_process_optional_headers(context); - } - - if (status == FLB_DECOMPRESSOR_SUCCESS && - context->state == FLB_DECOMPRESSOR_STATE_EXPECTING_BODY) { - *output_length = output_buffer_size; - - status = flb_gzip_decompressor_process_body_chunk( - context, - output_buffer, - output_length); - } - - if (status == FLB_DECOMPRESSOR_SUCCESS && - context->state == FLB_DECOMPRESSOR_STATE_EXPECTING_FOOTER) { - status = flb_gzip_decompressor_process_footer(context); - } - - return status; -} - -void *flb_gzip_decompression_context_create() -{ - struct flb_gzip_decompression_context *context; - - context = flb_calloc(1, sizeof(struct flb_gzip_decompression_context)); - - if (context == NULL) { - flb_errno(); - } - - return (void *) context; -} - -void flb_gzip_decompression_context_destroy(void *context) -{ - if (context != NULL) { - flb_free(context); - } -} diff --git a/fluent-bit/src/flb_hash.c b/fluent-bit/src/flb_hash.c deleted file mode 100644 index eef18d868..000000000 --- a/fluent-bit/src/flb_hash.c +++ /dev/null @@ -1,205 +0,0 @@ -/* Fluent Bit - * ========== - * Copyright (C) 2019-2020 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <fluent-bit/flb_hash.h> -#include <openssl/bio.h> - -static const EVP_MD *flb_crypto_get_digest_algorithm_instance_by_id(int algorithm_id) -{ - const EVP_MD *algorithm; - - if (algorithm_id == FLB_HASH_SHA256) { - algorithm = EVP_sha256(); - } - else if (algorithm_id == FLB_HASH_SHA512) { - algorithm = EVP_sha512(); - } - else if (algorithm_id == FLB_HASH_MD5) { - algorithm = EVP_md5(); - } - else { - algorithm = NULL; - } - - return algorithm; -} - -int flb_hash_init(struct flb_hash *context, int hash_type) -{ - const EVP_MD *digest_algorithm; - int result; - - if (context == NULL) { - return FLB_CRYPTO_INVALID_ARGUMENT; - } - - digest_algorithm = flb_crypto_get_digest_algorithm_instance_by_id(hash_type); - - if (digest_algorithm == NULL) { - return FLB_CRYPTO_INVALID_ARGUMENT; - } - - context->backend_context = EVP_MD_CTX_create(); - - if (context->backend_context == NULL) { - context->last_error = ERR_get_error(); - - return FLB_CRYPTO_BACKEND_ERROR; - } - - result = EVP_DigestInit_ex(context->backend_context, digest_algorithm, 0); - - if (result == 0) { - context->last_error = ERR_get_error(); - - return FLB_CRYPTO_BACKEND_ERROR; - } - - context->digest_size = EVP_MD_CTX_size(context->backend_context); - - return FLB_CRYPTO_SUCCESS; -} - -int flb_hash_finalize(struct flb_hash *context, - unsigned char *digest_buffer, - size_t digest_buffer_size) -{ - unsigned int digest_length; - int result; - - if (context->backend_context == NULL) { - return FLB_CRYPTO_INVALID_ARGUMENT; - } - - if (digest_buffer == NULL) { - return FLB_CRYPTO_INVALID_ARGUMENT; - } - - if (digest_buffer_size < context->digest_size) { - return FLB_CRYPTO_INVALID_ARGUMENT; - } - - result = EVP_DigestFinal_ex(context->backend_context, - digest_buffer, &digest_length); - - if (result == 0) { - context->last_error = ERR_get_error(); - - return FLB_CRYPTO_BACKEND_ERROR; - } - - (void) digest_length; - - return FLB_CRYPTO_SUCCESS; -} - -int flb_hash_update(struct flb_hash *context, - unsigned char *data, - size_t data_length) -{ - int result; - - if (context->backend_context == NULL) { - return FLB_CRYPTO_INVALID_ARGUMENT; - } - - if (data == NULL) { - return FLB_CRYPTO_INVALID_ARGUMENT; - } - - result = EVP_DigestUpdate(context->backend_context, - data, - data_length); - - if (result == 0) { - context->last_error = ERR_get_error(); - - return FLB_CRYPTO_BACKEND_ERROR; - } - - return FLB_CRYPTO_SUCCESS; -} - -int flb_hash_cleanup(struct flb_hash *context) -{ - if (context->backend_context == NULL) { - return FLB_CRYPTO_INVALID_ARGUMENT; - } - - EVP_MD_CTX_destroy(context->backend_context); - - context->backend_context = NULL; - - return FLB_CRYPTO_SUCCESS; -} - -int flb_hash_simple_batch(int hash_type, - size_t entry_count, - unsigned char **data_entries, - size_t *length_entries, - unsigned char *digest_buffer, - size_t digest_buffer_size) -{ - struct flb_hash digest_context; - size_t entry_index; - int result; - - result = flb_hash_init(&digest_context, hash_type); - - if (result == FLB_CRYPTO_SUCCESS) { - for (entry_index = 0 ; - entry_index < entry_count && result == FLB_CRYPTO_SUCCESS; - entry_index++) { - if (data_entries[entry_index] != NULL && - length_entries[entry_index] > 0) { - result = flb_hash_update(&digest_context, - data_entries[entry_index], - length_entries[entry_index]); - } - } - - if (result == FLB_CRYPTO_SUCCESS) { - result = flb_hash_finalize(&digest_context, - digest_buffer, - digest_buffer_size); - } - - flb_hash_cleanup(&digest_context); - } - - return result; -} - -int flb_hash_simple(int hash_type, - unsigned char *data, - size_t data_length, - unsigned char *digest_buffer, - size_t digest_buffer_size) -{ - size_t length_entries[1]; - unsigned char *data_entries[1]; - - data_entries[0] = data; - length_entries[0] = data_length; - - return flb_hash_simple_batch(hash_type, - 1, - data_entries, - length_entries, - digest_buffer, - digest_buffer_size); -} diff --git a/fluent-bit/src/flb_hash_table.c b/fluent-bit/src/flb_hash_table.c deleted file mode 100644 index d31c42591..000000000 --- a/fluent-bit/src/flb_hash_table.c +++ /dev/null @@ -1,539 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <fluent-bit/flb_info.h> -#include <fluent-bit/flb_mem.h> -#include <fluent-bit/flb_hash_table.h> -#include <fluent-bit/flb_log.h> -#include <fluent-bit/flb_str.h> - -#include <cfl/cfl.h> - -static inline void flb_hash_table_entry_free(struct flb_hash_table *ht, - struct flb_hash_table_entry *entry) -{ - mk_list_del(&entry->_head); - mk_list_del(&entry->_head_parent); - entry->table->count--; - ht->total_count--; - flb_free(entry->key); - if (entry->val && entry->val_size > 0) { - flb_free(entry->val); - } - flb_free(entry); -} - -struct flb_hash_table *flb_hash_table_create(int evict_mode, size_t size, int max_entries) -{ - int i; - struct flb_hash_table_chain *tmp; - struct flb_hash_table *ht; - - if (size <= 0) { - return NULL; - } - - ht = flb_malloc(sizeof(struct flb_hash_table)); - if (!ht) { - flb_errno(); - return NULL; - } - - mk_list_init(&ht->entries); - ht->evict_mode = evict_mode; - ht->max_entries = max_entries; - ht->size = size; - ht->total_count = 0; - ht->cache_ttl = 0; - ht->table = flb_calloc(1, sizeof(struct flb_hash_table_chain) * size); - if (!ht->table) { - flb_errno(); - flb_free(ht); - return NULL; - } - - /* Initialize chains list head */ - for (i = 0; i < size; i++) { - tmp = &ht->table[i]; - tmp->count = 0; - mk_list_init(&tmp->chains); - } - - return ht; -} - -struct flb_hash_table *flb_hash_table_create_with_ttl(int cache_ttl, int evict_mode, - size_t size, int max_entries) -{ - struct flb_hash_table *ht; - - ht = flb_hash_table_create(evict_mode, size, max_entries); - if (!ht) { - flb_errno(); - return NULL; - } - - ht->cache_ttl = cache_ttl; - return ht; -} - -int flb_hash_table_del_ptr(struct flb_hash_table *ht, const char *key, int key_len, - void *ptr) -{ - int id; - uint64_t hash; - struct mk_list *head; - struct flb_hash_table_entry *entry = NULL; - struct flb_hash_table_chain *table; - - /* Generate hash number */ - hash = cfl_hash_64bits(key, key_len); - id = (hash % ht->size); - - /* Link the new entry in our table at the end of the list */ - table = &ht->table[id]; - - mk_list_foreach(head, &table->chains) { - entry = mk_list_entry(head, struct flb_hash_table_entry, _head); - if (strncmp(entry->key, key, key_len) == 0 && entry->val == ptr) { - break; - } - entry = NULL; - } - - if (!entry) { - return -1; - } - - /* delete the entry */ - flb_hash_table_entry_free(ht, entry); - return 0; -} - - -void flb_hash_table_destroy(struct flb_hash_table *ht) -{ - int i; - struct mk_list *tmp; - struct mk_list *head; - struct flb_hash_table_entry *entry; - struct flb_hash_table_chain *table; - - for (i = 0; i < ht->size; i++) { - table = &ht->table[i]; - mk_list_foreach_safe(head, tmp, &table->chains) { - entry = mk_list_entry(head, struct flb_hash_table_entry, _head); - flb_hash_table_entry_free(ht, entry); - } - } - - flb_free(ht->table); - flb_free(ht); -} - -static void flb_hash_table_evict_random(struct flb_hash_table *ht) -{ - int id; - int count = 0; - struct mk_list *tmp; - struct mk_list *head; - struct flb_hash_table_entry *entry; - - id = random() % ht->total_count; - mk_list_foreach_safe(head, tmp, &ht->entries) { - if (id == count) { - entry = mk_list_entry(head, struct flb_hash_table_entry, _head_parent); - flb_hash_table_entry_free(ht, entry); - break; - } - count++; - } -} - -static void flb_hash_table_evict_less_used(struct flb_hash_table *ht) -{ - struct mk_list *head; - struct flb_hash_table_entry *entry; - struct flb_hash_table_entry *entry_less_used = NULL; - - mk_list_foreach(head, &ht->entries) { - entry = mk_list_entry(head, struct flb_hash_table_entry, _head_parent); - if (!entry_less_used) { - entry_less_used = entry; - } - else if (entry->hits < entry_less_used->hits) { - entry_less_used = entry; - } - } - - flb_hash_table_entry_free(ht, entry_less_used); -} - -static void flb_hash_table_evict_older(struct flb_hash_table *ht) -{ - struct flb_hash_table_entry *entry; - - entry = mk_list_entry_first(&ht->entries, struct flb_hash_table_entry, _head_parent); - flb_hash_table_entry_free(ht, entry); -} - -static struct flb_hash_table_entry *hash_get_entry(struct flb_hash_table *ht, - const char *key, int key_len, int *out_id) -{ - int id; - uint64_t hash; - struct mk_list *head; - struct flb_hash_table_chain *table; - struct flb_hash_table_entry *entry; - - if (!key || key_len <= 0) { - return NULL; - } - - hash = cfl_hash_64bits(key, key_len); - id = (hash % ht->size); - - table = &ht->table[id]; - if (table->count == 0) { - return NULL; - } - - if (table->count == 1) { - entry = mk_list_entry_first(&table->chains, - struct flb_hash_table_entry, _head); - - if (entry->key_len != key_len - || strncmp(entry->key, key, key_len) != 0) { - entry = NULL; - } - } - else { - /* Iterate entries */ - mk_list_foreach(head, &table->chains) { - entry = mk_list_entry(head, struct flb_hash_table_entry, _head); - if (entry->key_len != key_len) { - entry = NULL; - continue; - } - - if (strncmp(entry->key, key, key_len) == 0) { - break; - } - - entry = NULL; - } - } - - if (entry) { - *out_id = id; - } - - return entry; -} - -static int entry_set_value(struct flb_hash_table_entry *entry, void *val, size_t val_size) -{ - char *ptr; - - /* - * If the entry already contains a previous value in the heap, just remove - * the previously assigned memory. - */ - if (entry->val_size > 0) { - flb_free(entry->val); - } - - /* - * Now set the new value. If val_size > 0, we create a new memory area, otherwise - * it means the caller just wants to store a pointer address, no allocation - * is required. - */ - if (val_size > 0) { - entry->val = flb_malloc(val_size + 1); - if (!entry->val) { - flb_errno(); - return -1; - } - - /* - * Copy the buffer and append a NULL byte in case the caller set and - * expects a string. - */ - memcpy(entry->val, val, val_size); - ptr = (char *) entry->val; - ptr[val_size] = '\0'; - entry->val_size = val_size; - } - else { - /* just do a reference */ - entry->val = val; - entry->val_size = -1; - } - - entry->created = time(NULL); - - return 0; -} - -int flb_hash_table_add(struct flb_hash_table *ht, const char *key, int key_len, - void *val, ssize_t val_size) -{ - int id; - int ret; - uint64_t hash; - struct flb_hash_table_entry *entry; - struct flb_hash_table_chain *table; - - if (!key || key_len <= 0) { - return -1; - } - - /* Check capacity */ - if (ht->max_entries > 0 && ht->total_count >= ht->max_entries) { - if (ht->evict_mode == FLB_HASH_TABLE_EVICT_NONE) { - /* Do nothing */ - } - else if (ht->evict_mode == FLB_HASH_TABLE_EVICT_OLDER) { - flb_hash_table_evict_older(ht); - } - else if (ht->evict_mode == FLB_HASH_TABLE_EVICT_LESS_USED) { - flb_hash_table_evict_less_used(ht); - } - else if (ht->evict_mode == FLB_HASH_TABLE_EVICT_RANDOM) { - flb_hash_table_evict_random(ht); - } - } - - /* Check if this is a replacement */ - entry = hash_get_entry(ht, key, key_len, &id); - if (entry) { - /* - * The key already exists, just perform a value replacement, check if the - * value refers to our own previous allocation. - */ - ret = entry_set_value(entry, val, val_size); - if (ret == -1) { - return -1; - } - - return id; - } - - /* - * Below is just code to handle the creation of a new entry in the table - */ - - /* Generate hash number */ - hash = cfl_hash_64bits(key, key_len); - id = (hash % ht->size); - - /* Allocate the entry */ - entry = flb_calloc(1, sizeof(struct flb_hash_table_entry)); - if (!entry) { - flb_errno(); - return -1; - } - entry->created = time(NULL); - entry->hash = hash; - entry->hits = 0; - - /* Store the key and value as a new memory region */ - entry->key = flb_strndup(key, key_len); - entry->key_len = key_len; - entry->val_size = 0; - - /* store or reference the value */ - ret = entry_set_value(entry, val, val_size); - if (ret == -1) { - flb_free(entry); - return -1; - } - - /* Link the new entry in our table at the end of the list */ - table = &ht->table[id]; - entry->table = table; - - /* Add the new entry */ - mk_list_add(&entry->_head, &table->chains); - mk_list_add(&entry->_head_parent, &ht->entries); - - /* Update counters */ - table->count++; - ht->total_count++; - - return id; -} - -int flb_hash_table_get(struct flb_hash_table *ht, - const char *key, int key_len, - void **out_buf, size_t *out_size) -{ - int id; - struct flb_hash_table_entry *entry; - time_t expiration; - - entry = hash_get_entry(ht, key, key_len, &id); - if (!entry) { - return -1; - } - - if (ht->cache_ttl > 0) { - expiration = entry->created + ht->cache_ttl; - if (time(NULL) > expiration) { - flb_hash_table_entry_free(ht, entry); - return -1; - } - } - - entry->hits++; - *out_buf = entry->val; - *out_size = entry->val_size; - - return id; -} - -/* check if a hash exists */ -int flb_hash_table_exists(struct flb_hash_table *ht, uint64_t hash) -{ - int id; - struct mk_list *head; - struct flb_hash_table_chain *table; - struct flb_hash_table_entry *entry; - - id = (hash % ht->size); - table = &ht->table[id]; - - /* Iterate entries */ - mk_list_foreach(head, &table->chains) { - entry = mk_list_entry(head, struct flb_hash_table_entry, _head); - if (entry->hash == hash) { - return FLB_TRUE; - } - } - - return FLB_FALSE; -} - -/* - * Get an entry based in the table id. Note that a table id might have multiple - * entries so the 'key' parameter is required to get an exact match. - */ -int flb_hash_table_get_by_id(struct flb_hash_table *ht, int id, - const char *key, - const char **out_buf, size_t * out_size) -{ - struct mk_list *head; - struct flb_hash_table_entry *entry = NULL; - struct flb_hash_table_chain *table; - - if (ht->size <= id) { - return -1; - } - - table = &ht->table[id]; - if (table->count == 0) { - return -1; - } - - if (table->count == 1) { - entry = mk_list_entry_first(&table->chains, - struct flb_hash_table_entry, _head); - } - else { - mk_list_foreach(head, &table->chains) { - entry = mk_list_entry(head, struct flb_hash_table_entry, _head); - if (strcmp(entry->key, key) == 0) { - break; - } - entry = NULL; - } - } - - if (!entry) { - return -1; - } - - *out_buf = entry->val; - *out_size = entry->val_size; - - return 0; -} - -void *flb_hash_table_get_ptr(struct flb_hash_table *ht, const char *key, int key_len) -{ - int id; - struct flb_hash_table_entry *entry; - - entry = hash_get_entry(ht, key, key_len, &id); - if (!entry) { - return NULL; - } - - entry->hits++; - return entry->val; -} - -int flb_hash_table_del(struct flb_hash_table *ht, const char *key) -{ - int id; - int len; - uint64_t hash; - struct mk_list *head; - struct flb_hash_table_entry *entry = NULL; - struct flb_hash_table_chain *table; - - if (!key) { - return -1; - } - - len = strlen(key); - if (len == 0) { - return -1; - } - - hash = cfl_hash_64bits(key, len); - id = (hash % ht->size); - - table = &ht->table[id]; - if (table->count == 1) { - entry = mk_list_entry_first(&table->chains, - struct flb_hash_table_entry, - _head); - if (strcmp(entry->key, key) != 0) { - entry = NULL; - } - } - else { - mk_list_foreach(head, &table->chains) { - entry = mk_list_entry(head, struct flb_hash_table_entry, _head); - if (strcmp(entry->key, key) == 0) { - break; - } - entry = NULL; - } - } - - if (!entry) { - return -1; - } - - flb_hash_table_entry_free(ht, entry); - - return 0; -} diff --git a/fluent-bit/src/flb_help.c b/fluent-bit/src/flb_help.c deleted file mode 100644 index d93f9680b..000000000 --- a/fluent-bit/src/flb_help.c +++ /dev/null @@ -1,655 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <fluent-bit/flb_info.h> -#include <fluent-bit/flb_version.h> -#include <fluent-bit/flb_utils.h> -#include <fluent-bit/flb_help.h> -#include <fluent-bit/flb_pack.h> -#include <fluent-bit/flb_mp.h> -#include <fluent-bit/flb_custom.h> -#include <fluent-bit/flb_input.h> -#include <fluent-bit/flb_filter.h> -#include <fluent-bit/flb_output.h> - - - -static inline void pack_str_s(msgpack_packer *mp_pck, char *str, int size) -{ - int len; - - len = strlen(str); - msgpack_pack_str(mp_pck, len); - - if (len > 0) { - msgpack_pack_str_body(mp_pck, str, len); - } -} - -static inline void pack_str(msgpack_packer *mp_pck, char *str) -{ - int size = strlen(str); - pack_str_s(mp_pck, str, size); -} - -int pack_config_map_entry(msgpack_packer *mp_pck, struct flb_config_map *m) -{ - int len; - struct flb_mp_map_header mh; - - flb_mp_map_header_init(&mh, mp_pck); - - /* name */ - flb_mp_map_header_append(&mh); - pack_str(mp_pck, "name"); - pack_str(mp_pck, m->name); - - /* description */ - flb_mp_map_header_append(&mh); - pack_str(mp_pck, "description"); - if (m->desc) { - pack_str(mp_pck, m->desc); - } - else { - pack_str(mp_pck, ""); - } - - /* default value */ - flb_mp_map_header_append(&mh); - pack_str(mp_pck, "default"); - if (m->def_value) { - pack_str(mp_pck, m->def_value); - } - else { - msgpack_pack_nil(mp_pck); - } - - /* type */ - flb_mp_map_header_append(&mh); - pack_str(mp_pck, "type"); - - if (m->type == FLB_CONFIG_MAP_STR) { - pack_str(mp_pck, "string"); - } - else if (m->type == FLB_CONFIG_MAP_DEPRECATED) { - pack_str(mp_pck, "deprecated"); - } - else if (m->type == FLB_CONFIG_MAP_INT) { - pack_str(mp_pck, "integer"); - } - else if (m->type == FLB_CONFIG_MAP_BOOL) { - pack_str(mp_pck, "boolean"); - } - else if(m->type == FLB_CONFIG_MAP_DOUBLE) { - pack_str(mp_pck, "double"); - } - else if (m->type == FLB_CONFIG_MAP_SIZE) { - pack_str(mp_pck, "size"); - } - else if (m->type == FLB_CONFIG_MAP_TIME) { - pack_str(mp_pck, "time"); - } - else if (flb_config_map_mult_type(m->type) == FLB_CONFIG_MAP_CLIST) { - len = flb_config_map_expected_values(m->type); - if (len == -1) { - pack_str(mp_pck, "multiple comma delimited strings"); - } - else { - char tmp[64]; - snprintf(tmp, sizeof(tmp) - 1, - "comma delimited strings (minimum %i)", len); - pack_str(mp_pck, tmp); - } - } - else if (flb_config_map_mult_type(m->type) == FLB_CONFIG_MAP_SLIST) { - len = flb_config_map_expected_values(m->type); - if (len == -1) { - pack_str(mp_pck, "multiple space delimited strings"); - } - else { - char tmp[64]; - snprintf(tmp, sizeof(tmp) - 1, - "space delimited strings (minimum %i)", len); - pack_str(mp_pck, tmp); - } - } - else if (m->type == FLB_CONFIG_MAP_STR_PREFIX) { - pack_str(mp_pck, "prefixed string"); - } - - flb_mp_map_header_end(&mh); - return 0; -} - -int flb_help_custom(struct flb_custom_instance *ins, void **out_buf, size_t *out_size) -{ - struct mk_list *head; - struct mk_list *config_map; - struct flb_mp_map_header mh; - struct flb_config_map *m; - msgpack_sbuffer mp_sbuf; - msgpack_packer mp_pck; - - msgpack_sbuffer_init(&mp_sbuf); - msgpack_packer_init(&mp_pck, &mp_sbuf, msgpack_sbuffer_write); - - msgpack_pack_map(&mp_pck, 4); - - /* plugin type */ - pack_str(&mp_pck, "type"); - pack_str(&mp_pck, "custom"); - - /* plugin name */ - pack_str(&mp_pck, "name"); - pack_str(&mp_pck, ins->p->name); - - /* description */ - pack_str(&mp_pck, "description"); - pack_str(&mp_pck, ins->p->description); - - /* list of properties */ - pack_str(&mp_pck, "properties"); - flb_mp_map_header_init(&mh, &mp_pck); - - /* properties['options']: options exposed by the plugin */ - if (ins->p->config_map) { - flb_mp_map_header_append(&mh); - pack_str(&mp_pck, "options"); - - config_map = flb_config_map_create(ins->config, ins->p->config_map); - msgpack_pack_array(&mp_pck, mk_list_size(config_map)); - mk_list_foreach(head, config_map) { - m = mk_list_entry(head, struct flb_config_map, _head); - pack_config_map_entry(&mp_pck, m); - } - flb_config_map_destroy(config_map); - } - - flb_mp_map_header_end(&mh); - - *out_buf = mp_sbuf.data; - *out_size = mp_sbuf.size; - - return 0; -} - -int flb_help_input(struct flb_input_instance *ins, void **out_buf, size_t *out_size) -{ - struct mk_list *head; - struct mk_list *config_map; - struct flb_mp_map_header mh; - struct flb_config_map *m; - msgpack_sbuffer mp_sbuf; - msgpack_packer mp_pck; - int options_size = 0; - struct mk_list *tls_config; - struct flb_config_map m_input_net_listen = { - .type = FLB_CONFIG_MAP_STR, - .name = "host", - .def_value = "0.0.0.0", - .desc = "Listen Address", - }; - struct flb_config_map m_input_net_port = { - .type = FLB_CONFIG_MAP_INT, - .name = "port", - .def_value = "0", - .desc = "Listen Port", - }; - - - msgpack_sbuffer_init(&mp_sbuf); - msgpack_packer_init(&mp_pck, &mp_sbuf, msgpack_sbuffer_write); - - msgpack_pack_map(&mp_pck, 4); - - /* plugin type */ - pack_str(&mp_pck, "type"); - pack_str(&mp_pck, "input"); - - /* plugin name */ - pack_str(&mp_pck, "name"); - pack_str(&mp_pck, ins->p->name); - - /* description */ - pack_str(&mp_pck, "description"); - pack_str(&mp_pck, ins->p->description); - - /* list of properties */ - pack_str(&mp_pck, "properties"); - flb_mp_map_header_init(&mh, &mp_pck); - - /* properties['options']: options exposed by the plugin */ - if (ins->p->config_map) { - flb_mp_map_header_append(&mh); - pack_str(&mp_pck, "options"); - - config_map = flb_config_map_create(ins->config, ins->p->config_map); - options_size = mk_list_size(config_map); - - if ((ins->flags & (FLB_INPUT_NET | FLB_INPUT_NET_SERVER)) != 0) { - options_size += 2; - } - if (ins->flags & FLB_IO_OPT_TLS) { - tls_config = flb_tls_get_config_map(ins->config); - options_size += mk_list_size(tls_config); - } - - msgpack_pack_array(&mp_pck, options_size); - - if ((ins->flags & (FLB_INPUT_NET | FLB_INPUT_NET_SERVER)) != 0) { - pack_config_map_entry(&mp_pck, &m_input_net_listen); - pack_config_map_entry(&mp_pck, &m_input_net_port); - } - if (ins->flags & FLB_IO_OPT_TLS) { - mk_list_foreach(head, tls_config) { - m = mk_list_entry(head, struct flb_config_map, _head); - pack_config_map_entry(&mp_pck, m); - } - flb_config_map_destroy(tls_config); - } - - mk_list_foreach(head, config_map) { - m = mk_list_entry(head, struct flb_config_map, _head); - pack_config_map_entry(&mp_pck, m); - } - flb_config_map_destroy(config_map); - } - - flb_mp_map_header_end(&mh); - - *out_buf = mp_sbuf.data; - *out_size = mp_sbuf.size; - - return 0; -} - -int flb_help_filter(struct flb_filter_instance *ins, void **out_buf, size_t *out_size) -{ - struct mk_list *head; - struct mk_list *config_map; - struct flb_mp_map_header mh; - struct flb_config_map *m; - msgpack_sbuffer mp_sbuf; - msgpack_packer mp_pck; - - msgpack_sbuffer_init(&mp_sbuf); - msgpack_packer_init(&mp_pck, &mp_sbuf, msgpack_sbuffer_write); - - msgpack_pack_map(&mp_pck, 4); - - /* plugin type */ - pack_str(&mp_pck, "type"); - pack_str(&mp_pck, "filter"); - - /* plugin name */ - pack_str(&mp_pck, "name"); - pack_str(&mp_pck, ins->p->name); - - /* description */ - pack_str(&mp_pck, "description"); - pack_str(&mp_pck, ins->p->description); - - /* list of properties */ - pack_str(&mp_pck, "properties"); - flb_mp_map_header_init(&mh, &mp_pck); - - /* properties['options']: options exposed by the plugin */ - if (ins->p->config_map) { - flb_mp_map_header_append(&mh); - pack_str(&mp_pck, "options"); - - config_map = flb_config_map_create(ins->config, ins->p->config_map); - msgpack_pack_array(&mp_pck, mk_list_size(config_map)); - mk_list_foreach(head, config_map) { - m = mk_list_entry(head, struct flb_config_map, _head); - pack_config_map_entry(&mp_pck, m); - } - flb_config_map_destroy(config_map); - } - - flb_mp_map_header_end(&mh); - - *out_buf = mp_sbuf.data; - *out_size = mp_sbuf.size; - - return 0; -} - -int flb_help_output(struct flb_output_instance *ins, void **out_buf, size_t *out_size) -{ - struct mk_list *head; - struct mk_list *config_map; - struct flb_mp_map_header mh; - struct flb_config_map *m; - msgpack_sbuffer mp_sbuf; - msgpack_packer mp_pck; - int options_size = 0; - struct mk_list *tls_config; - struct flb_config_map m_output_net_host = { - .type = FLB_CONFIG_MAP_STR, - .name = "host", - .def_value = "", - .flags = 0, - .desc = "Host Address", - }; - struct flb_config_map m_output_net_port = { - .type = FLB_CONFIG_MAP_INT, - .name = "port", - .def_value = "0", - .flags = 0, - .desc = "host Port", - }; - - msgpack_sbuffer_init(&mp_sbuf); - msgpack_packer_init(&mp_pck, &mp_sbuf, msgpack_sbuffer_write); - - msgpack_pack_map(&mp_pck, 4); - - /* plugin type */ - pack_str(&mp_pck, "type"); - pack_str(&mp_pck, "output"); - - /* plugin name */ - pack_str(&mp_pck, "name"); - pack_str(&mp_pck, ins->p->name); - - /* description */ - pack_str(&mp_pck, "description"); - pack_str(&mp_pck, ins->p->description); - - /* list of properties */ - pack_str(&mp_pck, "properties"); - flb_mp_map_header_init(&mh, &mp_pck); - - /* properties['options']: options exposed by the plugin */ - if (ins->p->config_map) { - flb_mp_map_header_append(&mh); - pack_str(&mp_pck, "options"); - - config_map = flb_config_map_create(ins->config, ins->p->config_map); - options_size = mk_list_size(config_map); - - options_size = mk_list_size(config_map); - if (ins->flags & FLB_OUTPUT_NET) { - options_size += 2; - } - if (ins->flags & FLB_IO_OPT_TLS) { - tls_config = flb_tls_get_config_map(ins->config); - options_size += mk_list_size(tls_config); - } - - msgpack_pack_array(&mp_pck, options_size); - - if (ins->flags & FLB_OUTPUT_NET) { - pack_config_map_entry(&mp_pck, &m_output_net_host); - pack_config_map_entry(&mp_pck, &m_output_net_port); - } - if (ins->flags & FLB_IO_OPT_TLS) { - mk_list_foreach(head, tls_config) { - m = mk_list_entry(head, struct flb_config_map, _head); - pack_config_map_entry(&mp_pck, m); - } - flb_config_map_destroy(tls_config); - } - - mk_list_foreach(head, config_map) { - m = mk_list_entry(head, struct flb_config_map, _head); - pack_config_map_entry(&mp_pck, m); - } - flb_config_map_destroy(config_map); - } - - if (ins->p->flags & FLB_OUTPUT_NET) { - flb_mp_map_header_append(&mh); - pack_str(&mp_pck, "networking"); - - config_map = flb_upstream_get_config_map(ins->config); - msgpack_pack_array(&mp_pck, mk_list_size(config_map)); - mk_list_foreach(head, config_map) { - m = mk_list_entry(head, struct flb_config_map, _head); - pack_config_map_entry(&mp_pck, m); - } - flb_config_map_destroy(config_map); - } - - if (ins->p->flags & (FLB_IO_TLS | FLB_IO_OPT_TLS)) { - flb_mp_map_header_append(&mh); - pack_str(&mp_pck, "network_tls"); - - config_map = flb_tls_get_config_map(ins->config); - msgpack_pack_array(&mp_pck, mk_list_size(config_map)); - - /* Adjust 'tls' default value based on plugin type" */ - m = mk_list_entry_first(config_map, struct flb_config_map, _head); - if (ins->p->flags & FLB_IO_TLS) { - m->value.val.boolean = FLB_TRUE; - } - else if (ins->p->flags & FLB_IO_OPT_TLS) { - m->value.val.boolean = FLB_FALSE; - } - mk_list_foreach(head, config_map) { - m = mk_list_entry(head, struct flb_config_map, _head); - pack_config_map_entry(&mp_pck, m); - } - flb_config_map_destroy(config_map); - } - flb_mp_map_header_end(&mh); - - *out_buf = mp_sbuf.data; - *out_size = mp_sbuf.size; - - return 0; -} - -static int build_plugin_help(struct flb_config *config, int type, char *name, - char **out_buf, size_t *out_size) -{ - void *help_buf = NULL; - size_t help_size = 0; - struct flb_custom_instance *c = NULL; - struct flb_input_instance *i = NULL; - struct flb_filter_instance *f = NULL; - struct flb_output_instance *o = NULL; - - if (type == FLB_HELP_PLUGIN_CUSTOM) { - c = flb_custom_new(config, name, NULL); - if (!c) { - fprintf(stderr, "invalid custom plugin '%s'", name); - return -1; - } - flb_help_custom(c, &help_buf, &help_size); - flb_custom_instance_destroy(c); - } - else if (type == FLB_HELP_PLUGIN_INPUT) { - i = flb_input_new(config, name, 0, FLB_TRUE); - if (!i) { - fprintf(stderr, "invalid input plugin '%s'", name); - return -1; - } - flb_help_input(i, &help_buf, &help_size); - flb_input_instance_destroy(i); - } - else if (type == FLB_HELP_PLUGIN_FILTER) { - f = flb_filter_new(config, name, 0); - if (!f) { - fprintf(stderr, "invalid filter plugin '%s'", name); - return -1; - } - flb_help_filter(f, &help_buf, &help_size); - flb_filter_instance_destroy(f); - } - else if (type == FLB_HELP_PLUGIN_OUTPUT) { - o = flb_output_new(config, name, 0, FLB_TRUE); - if (!o) { - fprintf(stderr, "invalid output plugin '%s'", name); - return -1; - } - flb_help_output(o, &help_buf, &help_size); - flb_output_instance_destroy(o); - } - - *out_buf = help_buf; - *out_size = help_size; - - return 0; -} - -static void pack_map_kv(msgpack_packer *mp_pck, char *key, char *val) -{ - int k_len; - int v_len; - - k_len = strlen(key); - v_len = strlen(val); - - msgpack_pack_str(mp_pck, k_len); - msgpack_pack_str_body(mp_pck, key, k_len); - - msgpack_pack_str(mp_pck, v_len); - msgpack_pack_str_body(mp_pck, val, v_len); - -} - -flb_sds_t flb_help_build_json_schema(struct flb_config *config) -{ - int ret; - char *out_buf; - flb_sds_t json; - size_t out_size; - struct mk_list *head; - struct flb_custom_plugin *c; - struct flb_input_plugin *i; - struct flb_filter_plugin *f; - struct flb_output_plugin *o; - msgpack_sbuffer mp_sbuf; - msgpack_packer mp_pck; - struct flb_mp_map_header mh; - - /* initialize buffer */ - msgpack_sbuffer_init(&mp_sbuf); - msgpack_packer_init(&mp_pck, &mp_sbuf, msgpack_sbuffer_write); - - /* - * Root map for entries: - * - * - fluent-bit - * - customs - * - inputs - * - filters - * - outputs - */ - msgpack_pack_map(&mp_pck, 5); - - /* Fluent Bit */ - msgpack_pack_str(&mp_pck, 10); - msgpack_pack_str_body(&mp_pck, "fluent-bit", 10); - - /* fluent-bit['version'], fluent-bit['help_version'] and fluent-bit['os'] */ - msgpack_pack_map(&mp_pck, 3); - - pack_map_kv(&mp_pck, "version", FLB_VERSION_STR); - pack_map_kv(&mp_pck, "schema_version", FLB_HELP_SCHEMA_VERSION); - pack_map_kv(&mp_pck, "os", (char *) flb_utils_get_os_name()); - - /* customs */ - msgpack_pack_str(&mp_pck, 7); - msgpack_pack_str_body(&mp_pck, "customs", 7); - - flb_mp_array_header_init(&mh, &mp_pck); - mk_list_foreach(head, &config->custom_plugins) { - c = mk_list_entry(head, struct flb_custom_plugin, _head); - ret = build_plugin_help(config, FLB_HELP_PLUGIN_CUSTOM, c->name, - &out_buf, &out_size); - if (ret == -1) { - continue; - } - - flb_mp_array_header_append(&mh); - msgpack_sbuffer_write(&mp_sbuf, out_buf, out_size); - flb_free(out_buf); - } - flb_mp_array_header_end(&mh); - - - /* inputs */ - msgpack_pack_str(&mp_pck, 6); - msgpack_pack_str_body(&mp_pck, "inputs", 6); - - flb_mp_array_header_init(&mh, &mp_pck); - mk_list_foreach(head, &config->in_plugins) { - i = mk_list_entry(head, struct flb_input_plugin, _head); - if (i->flags & FLB_INPUT_PRIVATE){ - continue; - } - ret = build_plugin_help(config, FLB_HELP_PLUGIN_INPUT, i->name, - &out_buf, &out_size); - if (ret == -1) { - continue; - } - flb_mp_array_header_append(&mh); - msgpack_sbuffer_write(&mp_sbuf, out_buf, out_size); - flb_free(out_buf); - } - flb_mp_array_header_end(&mh); - - /* filters */ - msgpack_pack_str(&mp_pck, 7); - msgpack_pack_str_body(&mp_pck, "filters", 7); - - flb_mp_array_header_init(&mh, &mp_pck); - mk_list_foreach(head, &config->filter_plugins) { - f = mk_list_entry(head, struct flb_filter_plugin, _head); - ret = build_plugin_help(config, FLB_HELP_PLUGIN_FILTER, f->name, - &out_buf, &out_size); - if (ret == -1) { - continue; - } - - flb_mp_array_header_append(&mh); - msgpack_sbuffer_write(&mp_sbuf, out_buf, out_size); - flb_free(out_buf); - } - flb_mp_array_header_end(&mh); - - /* outputs */ - msgpack_pack_str(&mp_pck, 7); - msgpack_pack_str_body(&mp_pck, "outputs", 7); - - flb_mp_array_header_init(&mh, &mp_pck); - mk_list_foreach(head, &config->out_plugins) { - o = mk_list_entry(head, struct flb_output_plugin, _head); - if (o->flags & FLB_OUTPUT_PRIVATE){ - continue; - } - ret = build_plugin_help(config, FLB_HELP_PLUGIN_OUTPUT, o->name, - &out_buf, &out_size); - if (ret == -1) { - continue; - } - flb_mp_array_header_append(&mh); - msgpack_sbuffer_write(&mp_sbuf, out_buf, out_size); - flb_free(out_buf); - } - flb_mp_array_header_end(&mh); - - json = flb_msgpack_raw_to_json_sds(mp_sbuf.data, mp_sbuf.size); - msgpack_sbuffer_destroy(&mp_sbuf); - - return json; -} diff --git a/fluent-bit/src/flb_hmac.c b/fluent-bit/src/flb_hmac.c deleted file mode 100644 index 9c159c635..000000000 --- a/fluent-bit/src/flb_hmac.c +++ /dev/null @@ -1,382 +0,0 @@ -/* Fluent Bit - * ========== - * Copyright (C) 2019-2020 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <fluent-bit/flb_hmac.h> -#include <fluent-bit/flb_mem.h> - -#if FLB_CRYPTO_OPENSSL_COMPAT_MODE >= 3 -#include <openssl/params.h> -#endif - -#include <openssl/evp.h> -#include <openssl/bio.h> -#include <string.h> - -#if FLB_CRYPTO_OPENSSL_COMPAT_MODE >= 3 -static const char *flb_crypto_get_algorithm_name_by_id(int algorithm_id) -{ - const char *algorithm_name; - - if (algorithm_id == FLB_HASH_SHA256) { - algorithm_name = "SHA-256"; - } - else if (algorithm_id == FLB_HASH_SHA512) { - algorithm_name = "SHA-512"; - } - else if (algorithm_id == FLB_HASH_MD5) { - algorithm_name = "MD5"; - } - else { - algorithm_name = NULL; - } - - return algorithm_name; -} - -int flb_hmac_init(struct flb_hmac *context, - int algorithm_id, - unsigned char *key, - size_t key_length) -{ - const char *digest_algorithm_name; - OSSL_PARAM hmac_parameters[2]; - int result; - - - if (context == NULL) { - return FLB_CRYPTO_INVALID_ARGUMENT; - } - - if (key == NULL) { - return FLB_CRYPTO_INVALID_ARGUMENT; - } - - if (key_length == 0) { - return FLB_CRYPTO_INVALID_ARGUMENT; - } - - memset(context, 0, sizeof(struct flb_hmac)); - - digest_algorithm_name = flb_crypto_get_algorithm_name_by_id(algorithm_id); - - if (digest_algorithm_name == NULL) { - return FLB_CRYPTO_INVALID_ARGUMENT; - } - - context->mac_algorithm = EVP_MAC_fetch(NULL, "HMAC", NULL); - - if (context->mac_algorithm == NULL) { - context->last_error = ERR_get_error(); - - flb_hmac_cleanup(context); - - return FLB_CRYPTO_INVALID_ARGUMENT; - } - - context->backend_context = EVP_MAC_CTX_new(context->mac_algorithm); - - if (context->backend_context == NULL) { - context->last_error = ERR_get_error(); - - flb_hmac_cleanup(context); - - return FLB_CRYPTO_BACKEND_ERROR; - } - - hmac_parameters[0] = OSSL_PARAM_construct_utf8_string("digest", - (char *) digest_algorithm_name, - 0); - hmac_parameters[1] = OSSL_PARAM_construct_end(); - - - result = EVP_MAC_init(context->backend_context, - key, key_length, - hmac_parameters); - - if (result == 0) { - context->last_error = ERR_get_error(); - - return FLB_CRYPTO_BACKEND_ERROR; - } - - context->digest_size = EVP_MAC_CTX_get_mac_size(context->backend_context); - - return FLB_CRYPTO_SUCCESS; -} - -#else - -static const EVP_MD *flb_crypto_get_digest_algorithm_instance_by_id(int algorithm_id) -{ - const EVP_MD *algorithm; - - if (algorithm_id == FLB_HASH_SHA256) { - algorithm = EVP_sha256(); - } - else if (algorithm_id == FLB_HASH_SHA512) { - algorithm = EVP_sha512(); - } - else if (algorithm_id == FLB_HASH_MD5) { - algorithm = EVP_md5(); - } - else { - algorithm = NULL; - } - - return algorithm; -} - -int flb_hmac_init(struct flb_hmac *context, - int algorithm_id, - unsigned char *key, - size_t key_length) -{ - const EVP_MD *digest_algorithm_instance; - int result; - - - if (context == NULL) { - return FLB_CRYPTO_INVALID_ARGUMENT; - } - - if (key == NULL) { - return FLB_CRYPTO_INVALID_ARGUMENT; - } - - if (key_length == 0) { - return FLB_CRYPTO_INVALID_ARGUMENT; - } - - memset(context, 0, sizeof(struct flb_hmac)); - - digest_algorithm_instance = flb_crypto_get_digest_algorithm_instance_by_id(algorithm_id); - - if (digest_algorithm_instance == NULL) { - return FLB_CRYPTO_INVALID_ARGUMENT; - } - -#if FLB_CRYPTO_OPENSSL_COMPAT_MODE == 0 - context->backend_context = flb_calloc(1, sizeof(HMAC_CTX)); - - if (context->backend_context == NULL) { - return FLB_CRYPTO_ALLOCATION_ERROR; - } - - HMAC_CTX_init(context->backend_context); -#else - context->backend_context = HMAC_CTX_new(); - - if (context->backend_context == NULL) { - context->last_error = ERR_get_error(); - - return FLB_CRYPTO_BACKEND_ERROR; - } -#endif - - result = HMAC_Init_ex(context->backend_context, - key, key_length, - digest_algorithm_instance, - NULL); - - if (result != 1) { - context->last_error = ERR_get_error(); - - return FLB_CRYPTO_BACKEND_ERROR; - } - - context->digest_size = EVP_MD_size(digest_algorithm_instance); - - return FLB_CRYPTO_SUCCESS; -} -#endif - -int flb_hmac_finalize(struct flb_hmac *context, - unsigned char *signature_buffer, - size_t signature_buffer_size) -{ - size_t signature_length; - int error_detected; - int result; - - if (context->backend_context == NULL) { - return FLB_CRYPTO_INVALID_ARGUMENT; - } - - if (signature_buffer == NULL) { - return FLB_CRYPTO_INVALID_ARGUMENT; - } - - if (signature_buffer_size < context->digest_size) { - return FLB_CRYPTO_INVALID_ARGUMENT; - } - -#if FLB_CRYPTO_OPENSSL_COMPAT_MODE >= 3 - result = EVP_MAC_final(context->backend_context, - signature_buffer, - &signature_length, - signature_buffer_size); - - error_detected = (result == 0); -#else - signature_length = 0; - - result = HMAC_Final(context->backend_context, - signature_buffer, - (unsigned int *) &signature_length); - - error_detected = (result != 1); -#endif - - if (error_detected) { - context->last_error = ERR_get_error(); - - return FLB_CRYPTO_BACKEND_ERROR; - } - - (void) signature_length; - - return FLB_CRYPTO_SUCCESS; -} - -int flb_hmac_update(struct flb_hmac *context, - unsigned char *data, - size_t data_length) -{ - int error_detected; - int result; - - if (context->backend_context == NULL) { - return FLB_CRYPTO_INVALID_ARGUMENT; - } - - if (data == NULL) { - return FLB_CRYPTO_INVALID_ARGUMENT; - } - -#if FLB_CRYPTO_OPENSSL_COMPAT_MODE >= 3 - result = EVP_MAC_update(context->backend_context, - data, - data_length); - - error_detected = (result == 0); -#else - result = HMAC_Update(context->backend_context, - data, - data_length); - - error_detected = (result != 1); -#endif - - if (error_detected) { - context->last_error = ERR_get_error(); - - return FLB_CRYPTO_BACKEND_ERROR; - } - - return FLB_CRYPTO_SUCCESS; -} - -int flb_hmac_cleanup(struct flb_hmac *context) -{ -#if FLB_CRYPTO_OPENSSL_COMPAT_MODE >= 3 - if (context->backend_context != NULL) { - EVP_MAC_CTX_free(context->backend_context); - - context->backend_context = NULL; - } - - if (context->mac_algorithm != NULL) { - EVP_MAC_free(context->mac_algorithm); - - context->mac_algorithm = NULL; - } -#else - if (context->backend_context != NULL) { -#if FLB_CRYPTO_OPENSSL_COMPAT_MODE == 0 - HMAC_CTX_cleanup(context->backend_context); - - flb_free(context->backend_context); -#else - HMAC_CTX_reset(context->backend_context); - - HMAC_CTX_free(context->backend_context); -#endif - - context->backend_context = NULL; - } -#endif - - return FLB_CRYPTO_SUCCESS; -} - -int flb_hmac_simple_batch(int hash_type, - unsigned char *key, size_t key_length, - size_t entry_count, - unsigned char **data_entries, - size_t *length_entries, - unsigned char *signature_buffer, - size_t signature_buffer_size) -{ - struct flb_hmac digest_context; - size_t entry_index; - int result; - - result = flb_hmac_init(&digest_context, - hash_type, - key, key_length); - - if (result == FLB_CRYPTO_SUCCESS) { - for (entry_index = 0 ; - entry_index < entry_count && result == FLB_CRYPTO_SUCCESS; - entry_index++) { - result = flb_hmac_update(&digest_context, - data_entries[entry_index], - length_entries[entry_index]); - } - - if (result == FLB_CRYPTO_SUCCESS) { - result = flb_hmac_finalize(&digest_context, - signature_buffer, - signature_buffer_size); - } - - flb_hmac_cleanup(&digest_context); - } - - return result; -} - -int flb_hmac_simple(int hash_type, - unsigned char *key, size_t key_length, - unsigned char *data, size_t data_length, - unsigned char *signature_buffer, - size_t signature_buffer_size) -{ - size_t length_entries[1]; - unsigned char *data_entries[1]; - - length_entries[0] = data_length; - data_entries[0] = data; - - return flb_hmac_simple_batch(hash_type, - key, key_length, - 1, - data_entries, - length_entries, - signature_buffer, - signature_buffer_size); -} diff --git a/fluent-bit/src/flb_http_client.c b/fluent-bit/src/flb_http_client.c deleted file mode 100644 index 2d2803293..000000000 --- a/fluent-bit/src/flb_http_client.c +++ /dev/null @@ -1,1399 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * This is a very simple HTTP Client interface which aims to provide an - * easy way to issue HTTP requests and handle reponses from the input/output - * plugins. - * - * It scope is: - * - * - Use upstream connections. - * - Support 'retry' in case the HTTP server timeouts a connection. - * - Get return Status, Headers and Body content if found. - * - If Upstream supports keepalive, adjust headers - */ - -#define _GNU_SOURCE -#include <string.h> - -#include <fluent-bit/flb_info.h> -#include <fluent-bit/flb_kv.h> -#include <fluent-bit/flb_log.h> -#include <fluent-bit/flb_mem.h> -#include <fluent-bit/flb_http_client.h> -#include <fluent-bit/flb_http_client_debug.h> -#include <fluent-bit/flb_utils.h> -#include <fluent-bit/flb_base64.h> - - - -void flb_http_client_debug(struct flb_http_client *c, - struct flb_callback *cb_ctx) -{ -#ifdef FLB_HAVE_HTTP_CLIENT_DEBUG - if (cb_ctx) { - flb_http_client_debug_enable(c, cb_ctx); - } -#endif -} - -/* - * Removes the port from the host header - */ -int flb_http_strip_port_from_host(struct flb_http_client *c) -{ - struct mk_list *head; - struct flb_kv *kv; - char *out_host; - struct flb_upstream *u; - - u = c->u_conn->upstream; - - if (!c->host) { - if (!u->proxied_host) { - out_host = u->tcp_host; - } else { - out_host = u->proxied_host; - } - } else { - out_host = (char *) c->host; - } - - mk_list_foreach(head, &c->headers) { - kv = mk_list_entry(head, struct flb_kv, _head); - if (strcasecmp("Host", kv->key) == 0) { - flb_sds_destroy(kv->val); - kv->val = NULL; - kv->val = flb_sds_create(out_host); - if (!kv->val) { - flb_errno(); - return -1; - } - return 0; - } - } - - return -1; -} - -int flb_http_allow_duplicated_headers(struct flb_http_client *c, int allow) -{ - if (allow != FLB_TRUE && allow != FLB_FALSE) { - return -1; - } - - c->allow_dup_headers = allow; - return 0; -} - -/* check if there is enough space in the client header buffer */ -static int header_available(struct flb_http_client *c, int bytes) -{ - int available; - - available = c->header_size - c->header_len; - if (available < bytes) { - return -1; - } - - return 0; -} - -/* Try to find a header value in the buffer */ -static int header_lookup(struct flb_http_client *c, - const char *header, int header_len, - const char **out_val, int *out_len) -{ - char *p; - char *crlf; - char *end; - - if (!c->resp.data) { - return FLB_HTTP_MORE; - } - - /* Lookup the beginning of the header */ - p = strcasestr(c->resp.data, header); - end = strstr(c->resp.data, "\r\n\r\n"); - if (!p) { - if (end) { - /* The headers are complete but the header is not there */ - return FLB_HTTP_NOT_FOUND; - } - - /* We need more data */ - return FLB_HTTP_MORE; - } - - /* Exclude matches in the body */ - if (end && p > end) { - return FLB_HTTP_NOT_FOUND; - } - - /* Lookup CRLF (end of line \r\n) */ - crlf = strstr(p, "\r\n"); - if (!crlf) { - return FLB_HTTP_MORE; - } - - p += header_len; - - *out_val = p; - *out_len = (crlf - p); - - return FLB_HTTP_OK; -} - -/* HTTP/1.1: Check if we have a Chunked Transfer Encoding */ -static int check_chunked_encoding(struct flb_http_client *c) -{ - int ret; - int len; - const char *header = NULL; - - ret = header_lookup(c, "Transfer-Encoding: ", 19, - &header, &len); - if (ret == FLB_HTTP_NOT_FOUND) { - /* If the header is missing, this is fine */ - c->resp.chunked_encoding = FLB_FALSE; - return FLB_HTTP_OK; - } - else if (ret == FLB_HTTP_MORE) { - return FLB_HTTP_MORE; - } - - if (strncasecmp(header, "chunked", len) == 0) { - c->resp.chunked_encoding = FLB_TRUE; - } - - return FLB_HTTP_OK; -} - -/* Check response for a 'Content-Length' header */ -static int check_content_length(struct flb_http_client *c) -{ - int ret; - int len; - const char *header; - char tmp[256]; - - if (c->resp.status == 204) { - c->resp.content_length = -1; - return FLB_HTTP_OK; - } - - ret = header_lookup(c, "Content-Length: ", 16, - &header, &len); - if (ret == FLB_HTTP_MORE) { - return FLB_HTTP_MORE; - } - else if (ret == FLB_HTTP_NOT_FOUND) { - return FLB_HTTP_NOT_FOUND; - } - - if (len > sizeof(tmp) - 1) { - /* Value too long */ - return FLB_HTTP_ERROR; - } - - /* Copy to temporary buffer */ - memcpy(tmp, header, len); - tmp[len] = '\0'; - - c->resp.content_length = atoi(tmp); - return FLB_HTTP_OK; -} - -/* Check response for a 'Connection' header */ -static int check_connection(struct flb_http_client *c) -{ - int ret; - int len; - const char *header; - char *buf; - - ret = header_lookup(c, "Connection: ", 12, - &header, &len); - if (ret == FLB_HTTP_NOT_FOUND) { - return FLB_HTTP_NOT_FOUND; - } - else if (ret == FLB_HTTP_MORE) { - return FLB_HTTP_MORE; - } - - buf = flb_malloc(len + 1); - if (!buf) { - flb_errno(); - return -1; - } - - memcpy(buf, header, len); - buf[len] = '\0'; - - if (strncasecmp(buf, "close", 5) == 0) { - c->resp.connection_close = FLB_TRUE; - } - else if (strcasestr(buf, "keep-alive")) { - c->resp.connection_close = FLB_FALSE; - } - flb_free(buf); - return FLB_HTTP_OK; - -} - -static inline void consume_bytes(char *buf, int bytes, int length) -{ - memmove(buf, buf + bytes, length - bytes); -} - -static int process_chunked_data(struct flb_http_client *c) -{ - long len; - long drop; - long val; - char *p; - char tmp[32]; - struct flb_http_response *r = &c->resp; - - chunk_start: - p = strstr(r->chunk_processed_end, "\r\n"); - if (!p) { - return FLB_HTTP_MORE; - } - - /* Hexa string length */ - len = (p - r->chunk_processed_end); - if ((len > sizeof(tmp) - 1) || len == 0) { - return FLB_HTTP_ERROR; - } - p += 2; - - /* Copy hexa string to temporary buffer */ - memcpy(tmp, r->chunk_processed_end, len); - tmp[len] = '\0'; - - /* Convert hexa string to decimal */ - errno = 0; - val = strtol(tmp, NULL, 16); - if ((errno == ERANGE && (val == LONG_MAX || val == LONG_MIN)) - || (errno != 0 && val == 0)) { - flb_errno(); - return FLB_HTTP_ERROR; - } - if (val < 0) { - return FLB_HTTP_ERROR; - } - /* - * 'val' contains the expected number of bytes, check current lengths - * and do buffer adjustments. - * - * we do val + 2 because the chunk always ends with \r\n - */ - val += 2; - - /* Number of bytes after the Chunk header */ - len = r->data_len - (p - r->data); - if (len < val) { - return FLB_HTTP_MORE; - } - - /* From the current chunk we expect it ends with \r\n */ - if (p[val -2] != '\r' || p[val - 1] != '\n') { - return FLB_HTTP_ERROR; - } - - /* - * At this point we are just fine, the chunk is valid, next steps: - * - * 1. check possible last chunk - * 2. drop chunk header from the buffer - * 3. remove chunk ending \r\n - */ - - /* 1. Validate ending chunk */ - if (val - 2 == 0) { - /* - * For an ending chunk we expect: - * - * 0\r\n - * \r\n - * - * so at least we need 5 bytes in the buffer - */ - len = r->data_len - (r->chunk_processed_end - r->data); - if (len < 5) { - return FLB_HTTP_MORE; - } - - if (r->chunk_processed_end[3] != '\r' || - r->chunk_processed_end[4] != '\n') { - return FLB_HTTP_ERROR; - } - } - - /* 2. Drop chunk header */ - drop = (p - r->chunk_processed_end); - len = r->data_len - (r->chunk_processed_end - r->data); - consume_bytes(r->chunk_processed_end, drop, len); - r->data_len -= drop; - r->data[r->data_len] = '\0'; - - /* 3. Remove chunk ending \r\n */ - drop = 2; - r->chunk_processed_end += labs(val - 2); - len = r->data_len - (r->chunk_processed_end - r->data); - consume_bytes(r->chunk_processed_end, drop, len); - r->data_len -= drop; - - /* Always append a NULL byte */ - r->data[r->data_len] = '\0'; - - /* Is this the last chunk ? */ - if ((val - 2 == 0)) { - /* Update payload size */ - r->payload_size = r->data_len - (r->headers_end - r->data); - return FLB_HTTP_OK; - } - - /* If we have some remaining bytes, start over */ - len = r->data_len - (r->chunk_processed_end - r->data); - if (len > 0) { - goto chunk_start; - } - - return FLB_HTTP_MORE; -} - -static int process_data(struct flb_http_client *c) -{ - int ret; - char code[4]; - char *tmp; - - if (c->resp.data_len < 15) { - /* we need more data */ - return FLB_HTTP_MORE; - } - - /* HTTP response status */ - if (c->resp.status <= 0) { - memcpy(code, c->resp.data + 9, 3); - code[3] = '\0'; - c->resp.status = atoi(code); - if (c->resp.status < 100 || c->resp.status > 599) { - return FLB_HTTP_ERROR; - } - } - - /* Try to lookup content length */ - if (c->resp.content_length == -1 && c->resp.chunked_encoding == FLB_FALSE) { - ret = check_content_length(c); - if (ret == FLB_HTTP_ERROR) { - return FLB_HTTP_ERROR; - } - } - - /* Chunked encoding for HTTP/1.1 (no content length of course) */ - if ((c->flags & FLB_HTTP_11) && c->resp.content_length == -1) { - if (c->resp.chunked_encoding == FLB_FALSE) { - ret = check_chunked_encoding(c); - if (ret == FLB_HTTP_ERROR) { - return FLB_HTTP_ERROR; - } - } - } - - if (!c->resp.headers_end) { - tmp = strstr(c->resp.data, "\r\n\r\n"); - if (tmp) { - c->resp.headers_end = tmp + 4; - if (c->resp.chunked_encoding == FLB_TRUE) { - c->resp.chunk_processed_end = c->resp.headers_end; - } - - /* Mark the payload */ - if ((tmp - c->resp.data + 4) < c->resp.data_len) { - c->resp.payload = tmp += 4; - c->resp.payload_size = (c->resp.data_len - (tmp - c->resp.data)); - } - } - else { - return FLB_HTTP_MORE; - } - } - - /* Re-check if an ending exists, if so process payload if required */ - if (c->resp.headers_end) { - /* Mark the payload */ - if (!c->resp.payload && - c->resp.headers_end - c->resp.data < c->resp.data_len) { - c->resp.payload = c->resp.headers_end; - c->resp.payload_size = (c->resp.data_len - (c->resp.headers_end - c->resp.data)); - } - - if (c->resp.content_length >= 0) { - c->resp.payload_size = c->resp.data_len; - c->resp.payload_size -= (c->resp.headers_end - c->resp.data); - if (c->resp.payload_size >= c->resp.content_length) { - return FLB_HTTP_OK; - } - } - else if (c->resp.chunked_encoding == FLB_TRUE) { - ret = process_chunked_data(c); - if (ret == FLB_HTTP_ERROR) { - return FLB_HTTP_ERROR; - } - else if (ret == FLB_HTTP_OK) { - return FLB_HTTP_OK; - } - } - else { - return FLB_HTTP_OK; - } - } - else if (c->resp.headers_end && c->resp.content_length <= 0) { - return FLB_HTTP_OK; - } - - return FLB_HTTP_MORE; -} - -#if defined FLB_HAVE_TESTS_OSSFUZZ -int fuzz_process_data(struct flb_http_client *c); -int fuzz_process_data(struct flb_http_client *c) { - return process_data(c); -} - -int fuzz_check_connection(struct flb_http_client *c); -int fuzz_check_connection(struct flb_http_client *c) { - return check_connection(c); -} - -#endif - -static int proxy_parse(const char *proxy, struct flb_http_client *c) -{ - int len; - int port; - int off = 0; - const char *s; - const char *e; - const char *host; - - len = strlen(proxy); - if (len < 7) { - return -1; - } - - /* Protocol lookup */ - if (strncmp(proxy, "http://", 7) == 0) { - port = 80; - off = 7; - c->proxy.type = FLB_HTTP_PROXY_HTTP; - } - else if (strncmp(proxy, "https://", 8) == 0) { - port = 443; - off = 8; - c->proxy.type = FLB_HTTP_PROXY_HTTPS; - } - else { - return -1; - } - - /* Separate host/ip from port if any */ - s = proxy + off; - if (*s == '[') { - /* IPv6 address (RFC 3986) */ - e = strchr(++s, ']'); - if (!e) { - return -1; - } - host = strndup(s, e - s); - s = e + 1; - } else { - e = s; - while (!(*e == '\0' || *e == ':' || *e == '/')) { - ++e; - } - if (e == s) { - return -1; - } - host = strndup(s, e - s); - s = e; - } - if (*s == ':') { - port = atoi(++s); - } - - flb_trace("[http_client] proxy type=%i host=%s port=%i", - c->proxy.type, host, port); - - c->proxy.host = host; - c->proxy.port = port; - - return 0; -} - -static int add_host_and_content_length(struct flb_http_client *c) -{ - int len; - flb_sds_t tmp; - flb_sds_t host; - char *out_host; - int out_port; - size_t size; - struct flb_upstream *u = c->u_conn->upstream; - - if (!c->host) { - if (u->proxied_host) { - out_host = u->proxied_host; - } - else { - out_host = u->tcp_host; - } - } - else { - out_host = (char *) c->host; - } - - len = strlen(out_host); - host = flb_sds_create_size(len + 32); - if (!host) { - flb_error("[http_client] cannot create temporal buffer"); - return -1; - } - - if (c->port == 0) { - if (u->proxied_port != 0 ) { - out_port = u->proxied_port; - } - else { - out_port = u->tcp_port; - } - } - else { - out_port = c->port; - } - - if (c->flags & FLB_IO_TLS && out_port == 443) { - tmp = flb_sds_copy(host, out_host, strlen(out_host)); - } - else { - tmp = flb_sds_printf(&host, "%s:%i", out_host, out_port); - } - - if (!tmp) { - flb_sds_destroy(host); - flb_error("[http_client] cannot compose temporary host header"); - return -1; - } - host = tmp; - tmp = NULL; - - flb_http_add_header(c, "Host", 4, host, flb_sds_len(host)); - flb_sds_destroy(host); - - /* Content-Length */ - if (c->body_len >= 0) { - size = 32; - tmp = flb_malloc(size); - if (!tmp) { - flb_errno(); - return -1; - } - len = snprintf(tmp, size - 1, "%i", c->body_len); - flb_http_add_header(c, "Content-Length", 14, tmp, len); - flb_free(tmp); - } - - return 0; -} - -struct flb_http_client *flb_http_client(struct flb_connection *u_conn, - int method, const char *uri, - const char *body, size_t body_len, - const char *host, int port, - const char *proxy, int flags) -{ - int ret; - char *p; - char *buf = NULL; - char *str_method = NULL; - char *fmt_plain = \ - "%s %s HTTP/1.%i\r\n"; - char *fmt_proxy = \ - "%s http://%s:%i%s HTTP/1.%i\r\n" - "Proxy-Connection: KeepAlive\r\n"; - // TODO: IPv6 should have the format of [ip]:port - char *fmt_connect = \ - "%s %s:%i HTTP/1.%i\r\n" - "Proxy-Connection: KeepAlive\r\n"; - - struct flb_http_client *c; - - switch (method) { - case FLB_HTTP_GET: - str_method = "GET"; - break; - case FLB_HTTP_POST: - str_method = "POST"; - break; - case FLB_HTTP_PUT: - str_method = "PUT"; - break; - case FLB_HTTP_HEAD: - str_method = "HEAD"; - break; - case FLB_HTTP_CONNECT: - str_method = "CONNECT"; - break; - case FLB_HTTP_PATCH: - str_method = "PATCH"; - break; - }; - - buf = flb_calloc(1, FLB_HTTP_BUF_SIZE); - if (!buf) { - flb_errno(); - return NULL; - } - - /* FIXME: handler for HTTPS proxy */ - if (proxy) { - flb_debug("[http_client] using http_proxy %s for header", proxy); - ret = snprintf(buf, FLB_HTTP_BUF_SIZE, - fmt_proxy, - str_method, - host, - port, - uri, - flags & FLB_HTTP_10 ? 0 : 1); - } - else if (method == FLB_HTTP_CONNECT) { - flb_debug("[http_client] using HTTP CONNECT for proxy: proxy host %s, proxy port %i", host, port); - ret = snprintf(buf, FLB_HTTP_BUF_SIZE, - fmt_connect, - str_method, - host, - port, - flags & FLB_HTTP_10 ? 0 : 1); - } - else { - flb_debug("[http_client] not using http_proxy for header"); - ret = snprintf(buf, FLB_HTTP_BUF_SIZE, - fmt_plain, - str_method, - uri, - flags & FLB_HTTP_10 ? 0 : 1); - } - - if (ret == -1) { - flb_errno(); - flb_free(buf); - return NULL; - } - - c = flb_calloc(1, sizeof(struct flb_http_client)); - if (!c) { - flb_free(buf); - return NULL; - } - - c->u_conn = u_conn; - c->method = method; - c->uri = uri; - c->host = host; - c->port = port; - c->header_buf = buf; - c->header_size = FLB_HTTP_BUF_SIZE; - c->header_len = ret; - c->flags = flags; - c->allow_dup_headers = FLB_TRUE; - mk_list_init(&c->headers); - - /* Check if we have a query string */ - p = strchr(uri, '?'); - if (p) { - p++; - c->query_string = p; - } - - /* Is Upstream connection using keepalive mode ? */ - if (flb_stream_get_flag_status(&u_conn->upstream->base, FLB_IO_TCP_KA)) { - c->flags |= FLB_HTTP_KA; - } - - /* Response */ - c->resp.content_length = -1; - c->resp.connection_close = -1; - - if ((flags & FLB_HTTP_10) == 0) { - c->flags |= FLB_HTTP_11; - } - - if (body && body_len > 0) { - c->body_buf = body; - c->body_len = body_len; - } - - add_host_and_content_length(c); - - /* Check proxy data */ - if (proxy) { - flb_debug("[http_client] Using http_proxy: %s", proxy); - ret = proxy_parse(proxy, c); - if (ret != 0) { - flb_debug("[http_client] Something wrong with the http_proxy parsing"); - flb_http_client_destroy(c); - return NULL; - } - } - - /* 'Read' buffer size */ - c->resp.data = flb_malloc(FLB_HTTP_DATA_SIZE_MAX); - if (!c->resp.data) { - flb_errno(); - flb_http_client_destroy(c); - return NULL; - } - c->resp.data[0] = '\0'; - c->resp.data_len = 0; - c->resp.data_size = FLB_HTTP_DATA_SIZE_MAX; - c->resp.data_size_max = FLB_HTTP_DATA_SIZE_MAX; - - return c; -} - -/* - * By default the HTTP client have a fixed buffer to read a response for a - * simple request. But in certain situations the caller might expect a - * larger response that exceed the buffer limit. - * - * This function allows to set a maximum buffer size for the client - * response where: - * - * 1. size = 0 no limit, read as much as possible. - * 2. size = N: specific limit, upon reach limit discard data (default: 4KB) - */ -int flb_http_buffer_size(struct flb_http_client *c, size_t size) -{ - if (size < c->resp.data_size_max && size != 0) { - flb_error("[http] requested buffer size %lu (bytes) needs to be greater than " - "minimum size allowed %lu (bytes)", - size, c->resp.data_size_max); - return -1; - } - - c->resp.data_size_max = size; - return 0; -} - -size_t flb_http_buffer_available(struct flb_http_client *c) -{ - return (c->resp.data_size - c->resp.data_len); -} - -/* - * Increase the read buffer size based on the limits set by default or manually - * through the flb_http_buffer_size() function. - * - * The parameter 'size' is the amount of extra memory requested. - */ -int flb_http_buffer_increase(struct flb_http_client *c, size_t size, - size_t *out_size) -{ - int off_payload = 0; - int off_headers_end = 0; - int off_chunk_processed_end = 0; - char *tmp; - size_t new_size; - size_t allocated; - - *out_size = 0; - new_size = c->resp.data_size + size; - - /* Limit exceeded, adjust */ - if (c->resp.data_size_max != 0) { - if (new_size > c->resp.data_size_max) { - new_size = c->resp.data_size_max; - if (new_size <= c->resp.data_size) { - /* Can't expand the buffer any further. */ - return -1; - } - } - } - - - if (c->resp.headers_end) { - off_headers_end = c->resp.headers_end - c->resp.data; - } - if (c->resp.chunk_processed_end) { - off_chunk_processed_end = c->resp.chunk_processed_end - c->resp.data; - } - - /* - * The payload is a reference to a position of 'data' buffer, - * we need to adjust the pointer after a memory buffer size change. - */ - if (c->resp.payload_size > 0) { - off_payload = c->resp.payload - c->resp.data; - } - - tmp = flb_realloc(c->resp.data, new_size); - if (!tmp) { - flb_errno(); - return -1; - } - else { - allocated = new_size - c->resp.data_size; - c->resp.data = tmp; - c->resp.data_size = new_size; - - if (off_headers_end > 0) { - c->resp.headers_end = c->resp.data + off_headers_end; - } - if (off_chunk_processed_end > 0) { - c->resp.chunk_processed_end = c->resp.data + off_chunk_processed_end; - } - if (off_payload > 0) { - c->resp.payload = c->resp.data + off_payload; - } - } - - *out_size = allocated; - return 0; -} - - -/* Append a custom HTTP header to the request */ -int flb_http_add_header(struct flb_http_client *c, - const char *key, size_t key_len, - const char *val, size_t val_len) -{ - struct flb_kv *kv; - struct mk_list *tmp; - struct mk_list *head; - - if (key_len < 1 || val_len < 1) { - return -1; - } - - /* Check any previous header to avoid duplicates */ - if (c->allow_dup_headers == FLB_FALSE) { - mk_list_foreach_safe(head, tmp, &c->headers) { - kv = mk_list_entry(head, struct flb_kv, _head); - if (flb_sds_casecmp(kv->key, key, key_len) == 0) { - /* the header already exists, remove it */ - flb_kv_item_destroy(kv); - break; - } - } - } - - /* register new header in the temporal kv list */ - kv = flb_kv_item_create_len(&c->headers, - (char *) key, key_len, (char *) val, val_len); - if (!kv) { - return -1; - } - - return 0; -} - -/* - * flb_http_get_header looks up a first value of request header. - * The return value should be destroyed after using. - * The return value is NULL, if the value is not found. - */ -flb_sds_t flb_http_get_header(struct flb_http_client *c, - const char *key, size_t key_len) -{ - flb_sds_t ret_str; - struct flb_kv *kv; - struct mk_list *head = NULL; - struct mk_list *tmp = NULL; - - mk_list_foreach_safe(head, tmp, &c->headers) { - kv = mk_list_entry(head, struct flb_kv, _head); - if (flb_sds_casecmp(kv->key, key, key_len) == 0) { - ret_str = flb_sds_create(kv->val); - return ret_str; - } - } - - return NULL; -} - -static int http_header_push(struct flb_http_client *c, struct flb_kv *header) -{ - char *tmp; - const char *key; - const char *val; - size_t key_len; - size_t val_len; - size_t required; - size_t new_size; - - key = header->key; - key_len = flb_sds_len(header->key); - val = header->val; - val_len = flb_sds_len(header->val); - - /* - * The new header will need enough space in the buffer: - * - * key : length of the key - * separator: ': ' (2 bytes) - * val : length of the key value - * CRLF : '\r\n' (2 bytes) - */ - required = key_len + 2 + val_len + 2; - - if (header_available(c, required) != 0) { - if (required < 512) { - new_size = c->header_size + 512; - } - else { - new_size = c->header_size + required; - } - tmp = flb_realloc(c->header_buf, new_size); - if (!tmp) { - flb_errno(); - return -1; - } - c->header_buf = tmp; - c->header_size = new_size; - } - - /* append the header key */ - memcpy(c->header_buf + c->header_len, - key, key_len); - c->header_len += key_len; - - /* append the separator */ - c->header_buf[c->header_len++] = ':'; - c->header_buf[c->header_len++] = ' '; - - /* append the header value */ - memcpy(c->header_buf + c->header_len, - val, val_len); - c->header_len += val_len; - - /* Append the ending header CRLF */ - c->header_buf[c->header_len++] = '\r'; - c->header_buf[c->header_len++] = '\n'; - - return 0; -} - -static int http_headers_compose(struct flb_http_client *c) -{ - int ret; - struct mk_list *head; - struct flb_kv *header; - - /* Push header list to one buffer */ - mk_list_foreach(head, &c->headers) { - header = mk_list_entry(head, struct flb_kv, _head); - ret = http_header_push(c, header); - if (ret != 0) { - flb_error("[http_client] cannot compose request headers"); - return -1; - } - } - - return 0; -} - -static void http_headers_destroy(struct flb_http_client *c) -{ - flb_kv_release(&c->headers); -} - -int flb_http_set_keepalive(struct flb_http_client *c) -{ - /* check if 'keepalive' mode is enabled in the Upstream connection */ - if (flb_stream_is_keepalive(c->u_conn->stream)) { - return -1; - } - - /* append header */ - return flb_http_add_header(c, - FLB_HTTP_HEADER_CONNECTION, - sizeof(FLB_HTTP_HEADER_CONNECTION) - 1, - FLB_HTTP_HEADER_KA, - sizeof(FLB_HTTP_HEADER_KA) - 1); -} - -/* Adds a header specifying that the payload is compressed with gzip */ -int flb_http_set_content_encoding_gzip(struct flb_http_client *c) -{ - int ret; - - ret = flb_http_add_header(c, - FLB_HTTP_HEADER_CONTENT_ENCODING, - sizeof(FLB_HTTP_HEADER_CONTENT_ENCODING) - 1, - "gzip", 4); - return ret; -} - -int flb_http_set_callback_context(struct flb_http_client *c, - struct flb_callback *cb_ctx) -{ - c->cb_ctx = cb_ctx; - return 0; -} - -int flb_http_add_auth_header(struct flb_http_client *c, - const char *user, const char *passwd, const char *header) { - int ret; - int len_u; - int len_p; - int len_h; - int len_out; - char tmp[1024]; - char *p; - size_t b64_len; - - /* - * We allow a max of 255 bytes for user and password (255 each), meaning - * we need at least: - * - * 'Basic base64(user : passwd)' => ~688 bytes - * - */ - - len_u = strlen(user); - - if (passwd) { - len_p = strlen(passwd); - } - else { - len_p = 0; - } - - p = flb_malloc(len_u + len_p + 2); - if (!p) { - flb_errno(); - return -1; - } - - memcpy(p, user, len_u); - p[len_u] = ':'; - len_out = len_u + 1; - - if (passwd) { - memcpy(p + len_out, passwd, len_p); - len_out += len_p; - } - p[len_out] = '\0'; - - memcpy(tmp, "Basic ", 6); - ret = flb_base64_encode((unsigned char *) tmp + 6, sizeof(tmp) - 7, &b64_len, - (unsigned char *) p, len_out); - if (ret != 0) { - flb_free(p); - return -1; - } - - flb_free(p); - b64_len += 6; - - len_h = strlen(header); - ret = flb_http_add_header(c, - header, - len_h, - tmp, b64_len); - return ret; -} - -int flb_http_basic_auth(struct flb_http_client *c, - const char *user, const char *passwd) -{ - return flb_http_add_auth_header(c, user, passwd, FLB_HTTP_HEADER_AUTH); -} - -int flb_http_proxy_auth(struct flb_http_client *c, - const char *user, const char *passwd) -{ - return flb_http_add_auth_header(c, user, passwd, FLB_HTTP_HEADER_PROXY_AUTH); -} - -int flb_http_bearer_auth(struct flb_http_client *c, const char *token) -{ - flb_sds_t header_buffer; - flb_sds_t header_line; - int result; - - result = -1; - - if (token == NULL) { - token = ""; - - /* Shouldn't we log this and return instead of sending - * a malformed value? - */ - } - - header_buffer = flb_sds_create_size(strlen(token) + 64); - - if (header_buffer == NULL) { - return -1; - } - - header_line = flb_sds_printf(&header_buffer, "Bearer %s", token); - - if (header_line != NULL) { - result = flb_http_add_header(c, - FLB_HTTP_HEADER_AUTH, - strlen(FLB_HTTP_HEADER_AUTH), - header_line, - flb_sds_len(header_line)); - } - - flb_sds_destroy(header_buffer); - - return result; -} - - -int flb_http_do(struct flb_http_client *c, size_t *bytes) -{ - int ret; - int r_bytes; - int crlf = 2; - int new_size; - ssize_t available; - size_t out_size; - size_t bytes_header = 0; - size_t bytes_body = 0; - char *tmp; - - /* Append pending headers */ - ret = http_headers_compose(c); - if (ret == -1) { - return -1; - } - - /* check enough space for the ending CRLF */ - if (header_available(c, crlf) != 0) { - new_size = c->header_size + 2; - tmp = flb_realloc(c->header_buf, new_size); - if (!tmp) { - flb_errno(); - return -1; - } - c->header_buf = tmp; - c->header_size = new_size; - } - - /* Append the ending header CRLF */ - c->header_buf[c->header_len++] = '\r'; - c->header_buf[c->header_len++] = '\n'; - -#ifdef FLB_HAVE_HTTP_CLIENT_DEBUG - /* debug: request_headers callback */ - flb_http_client_debug_cb(c, "_debug.http.request_headers"); - - /* debug: request_payload callback */ - if (c->body_len > 0) { - flb_http_client_debug_cb(c, "_debug.http.request_payload"); - } -#endif - - /* Write the header */ - ret = flb_io_net_write(c->u_conn, - c->header_buf, c->header_len, - &bytes_header); - if (ret == -1) { - /* errno might be changed from the original call */ - if (errno != 0) { - flb_errno(); - } - return -1; - } - - if (c->body_len > 0) { - ret = flb_io_net_write(c->u_conn, - c->body_buf, c->body_len, - &bytes_body); - if (ret == -1) { - flb_errno(); - return -1; - } - } - - /* number of sent bytes */ - *bytes = (bytes_header + bytes_body); - - /* Read the server response, we need at least 19 bytes */ - c->resp.data_len = 0; - while (1) { - available = flb_http_buffer_available(c) - 1; - if (available <= 1) { - /* - * If there is no more space available on our buffer, try to - * increase it. - */ - ret = flb_http_buffer_increase(c, FLB_HTTP_DATA_CHUNK, - &out_size); - if (ret == -1) { - /* - * We could not allocate more space, let the caller handle - * this. - */ - flb_warn("[http_client] cannot increase buffer: current=%zu " - "requested=%zu max=%zu", c->resp.data_size, - c->resp.data_size + FLB_HTTP_DATA_CHUNK, - c->resp.data_size_max); - flb_upstream_conn_recycle(c->u_conn, FLB_FALSE); - return 0; - } - available = flb_http_buffer_available(c) - 1; - } - - r_bytes = flb_io_net_read(c->u_conn, - c->resp.data + c->resp.data_len, - available); - if (r_bytes <= 0) { - if (c->flags & FLB_HTTP_10) { - break; - } - } - - /* Always append a NULL byte */ - if (r_bytes >= 0) { - c->resp.data_len += r_bytes; - c->resp.data[c->resp.data_len] = '\0'; - - ret = process_data(c); - if (ret == FLB_HTTP_ERROR) { - flb_warn("[http_client] malformed HTTP response from %s:%i on " - "connection #%i", - c->u_conn->upstream->tcp_host, - c->u_conn->upstream->tcp_port, - c->u_conn->fd); - return -1; - } - else if (ret == FLB_HTTP_OK) { - break; - } - else if (ret == FLB_HTTP_MORE) { - continue; - } - } - else { - flb_error("[http_client] broken connection to %s:%i ?", - c->u_conn->upstream->tcp_host, - c->u_conn->upstream->tcp_port); - return -1; - } - } - - /* Check 'Connection' response header */ - ret = check_connection(c); - if (ret == FLB_HTTP_OK) { - /* - * If the server replied that the connection will be closed - * and our Upstream connection is in keepalive mode, we must - * inactivate the connection. - */ - if (c->resp.connection_close == FLB_TRUE) { - /* Do not recycle the connection (no more keepalive) */ - flb_upstream_conn_recycle(c->u_conn, FLB_FALSE); - flb_debug("[http_client] server %s:%i will close connection #%i", - c->u_conn->upstream->tcp_host, - c->u_conn->upstream->tcp_port, - c->u_conn->fd); - } - } - -#ifdef FLB_HAVE_HTTP_CLIENT_DEBUG - flb_http_client_debug_cb(c, "_debug.http.response_headers"); - if (c->resp.payload_size > 0) { - flb_http_client_debug_cb(c, "_debug.http.response_payload"); - } -#endif - - return 0; -} - -/* - * flb_http_client_proxy_connect opens a tunnel to a proxy server via - * http `CONNECT` method. This is needed for https traffic through a - * http proxy. - * More: https://developer.mozilla.org/en-US/docs/Web/HTTP/Methods/CONNECT - */ -int flb_http_client_proxy_connect(struct flb_connection *u_conn) -{ - struct flb_upstream *u = u_conn->upstream; - struct flb_http_client *c; - size_t b_sent; - int ret = -1; - - /* Don't pass proxy when using FLB_HTTP_CONNECT */ - flb_debug("[upstream] establishing http tunneling to proxy: host %s port %d", u->tcp_host, u->tcp_port); - c = flb_http_client(u_conn, FLB_HTTP_CONNECT, "", NULL, - 0, u->proxied_host, u->proxied_port, NULL, 0); - - /* Setup proxy's username and password */ - if (u->proxy_username && u->proxy_password) { - flb_debug("[upstream] proxy uses username %s password %s", u->proxy_username, u->proxy_password); - flb_http_proxy_auth(c, u->proxy_username, u->proxy_password); - } - - flb_http_buffer_size(c, 4192); - - flb_http_add_header(c, "User-Agent", 10, "Fluent-Bit", 10); - - /* Send HTTP request */ - ret = flb_http_do(c, &b_sent); - - /* Validate HTTP response */ - if (ret != 0) { - flb_error("[upstream] error in flb_establish_proxy: %d", ret); - ret = -1; - } - else { - /* The request was issued successfully, validate the 'error' field */ - flb_debug("[upstream] proxy returned %d", c->resp.status); - if (c->resp.status == 200) { - ret = 0; - } - else { - flb_error("flb_establish_proxy error: %s", c->resp.payload); - ret = -1; - } - } - - /* Cleanup */ - flb_http_client_destroy(c); - - return ret; -} - -void flb_http_client_destroy(struct flb_http_client *c) -{ - http_headers_destroy(c); - flb_free(c->resp.data); - flb_free(c->header_buf); - flb_free((void *)c->proxy.host); - flb_free(c); -} diff --git a/fluent-bit/src/flb_http_client_debug.c b/fluent-bit/src/flb_http_client_debug.c deleted file mode 100644 index 3f69547af..000000000 --- a/fluent-bit/src/flb_http_client_debug.c +++ /dev/null @@ -1,196 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <fluent-bit/flb_info.h> -#include <fluent-bit/flb_log.h> -#include <fluent-bit/flb_mem.h> -#include <fluent-bit/flb_utils.h> -#include <fluent-bit/flb_callback.h> -#include <fluent-bit/flb_http_client.h> - -/* - * Callbacks - * ========= - */ -static void debug_cb_request_headers(char *name, void *p1, void *p2) -{ - struct flb_http_client *c = p1; - - flb_idebug("[http] request headers\n%s", c->header_buf); -} - -static void debug_cb_request_payload(char *name, void *p1, void *p2) -{ - unsigned char *ptr; - struct flb_http_client *c = p1; - - if (c->body_len > 3) { - ptr = (unsigned char *) c->body_buf; - if (ptr[0] == 0x1F && ptr[1] == 0x8B && ptr[2] == 0x08) { - flb_idebug("[http] request payload (%d bytes)\n[GZIP binary content...]", - c->body_len); - } - else { - flb_idebug("[http] request payload (%d bytes)\n%s", - c->body_len, c->body_buf); - } - } - else { - flb_idebug("[http] request payload (%d bytes)\n%s", - c->body_len, c->body_buf); - } -} - -static void debug_cb_response_headers(char *name, void *p1, void *p2) -{ - char tmp; - struct flb_http_client *c = p1; - - /* - * Just to make easier debugging, we are going to put a NULL byte after - * the header break (\r\n\r\n) and then restore it. - */ - tmp = *c->resp.headers_end; - *c->resp.headers_end = '\0'; - - flb_idebug("[http] response headers\n%s", c->resp.data); - *c->resp.headers_end = tmp; -} - -static void debug_cb_response_payload(char *name, void *p1, void *p2) -{ - struct flb_http_client *c = p1; - - flb_idebug("[http] response payload (%lu bytes)\n%s", - c->resp.payload_size, c->resp.payload); -} - -struct flb_http_callback { - char *name; - void (*func)(char *, void *, void *); -}; - -/* - * Callbacks Table - */ -struct flb_http_callback http_callbacks[] = { - /* request */ - { "_debug.http.request_headers", debug_cb_request_headers }, - { "_debug.http.request_payload", debug_cb_request_payload }, - - /* response */ - { "_debug.http.response_headers", debug_cb_response_headers }, - { "_debug.http.response_payload", debug_cb_response_payload }, - { 0 } -}; - -/* - * Exported Functions - * ================== - */ -/* Determinate if a http.debug property is valid or not */ -int flb_http_client_debug_property_is_valid(char *key, char *val) -{ - int i; - int ret; - int len; - struct flb_http_callback *cb; - - if (!key) { - flb_error("[http_client] given property is invalid"); - return -1; - } - - if (!val) { - flb_error("[http_client] property '%s' don't have a valid value", - key); - return -1; - } - - len = (sizeof(http_callbacks) / sizeof(struct flb_http_callback)) - 1; - for (i = 0; i < len ; i++) { - cb = &http_callbacks[i]; - if (strcasecmp(key, cb->name) == 0) { - ret = flb_utils_bool(val); - if (ret == -1) { - return FLB_FALSE; - } - return FLB_TRUE; - } - } - - return FLB_FALSE; -} - - -int flb_http_client_debug_cb(struct flb_http_client *c, char *name) -{ - int ret; - - ret = flb_callback_do(c->cb_ctx, name, c, NULL); - return ret; -} - -/* - * This function helps to setup 'HTTP' debugging mode on a HTTP client context - * using the list of configuration properties set by the instance. On this - * specific case we don't pass the 'plugin instance' reference since it can be - * an input, filter or output, we try to make this agnostic. - */ -int flb_http_client_debug_setup(struct flb_callback *cb_ctx, - struct mk_list *props) -{ - int i; - int len; - int ret; - const char *tmp; - struct flb_http_callback *cb; - - /* - * Iterate table of callbacks, if the callbacks are not pre-defined in the - * context (this might happen when callbacks are overrided by a library - * caller), set the default ones. - */ - len = (sizeof(http_callbacks) / sizeof(struct flb_http_callback)) - 1; - for (i = 0; i < len ; i++) { - cb = &http_callbacks[i]; - - /* Check if the debug property has been enabled */ - tmp = flb_config_prop_get(cb->name, props); - if (!tmp) { - continue; - } - - ret = flb_utils_bool(tmp); - if (ret == FLB_FALSE) { - continue; - } - - ret = flb_callback_exists(cb_ctx, cb->name); - if (ret == FLB_FALSE) { - /* Set default callback */ - ret = flb_callback_set(cb_ctx, cb->name, cb->func); - if (ret == -1) { - flb_error("[http_client] error setting up default " - "callback '%s'", cb->name); - } - } - } - return 0; -} diff --git a/fluent-bit/src/flb_input.c b/fluent-bit/src/flb_input.c deleted file mode 100644 index 1c4faad4d..000000000 --- a/fluent-bit/src/flb_input.c +++ /dev/null @@ -1,1965 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <stdlib.h> - -#include <monkey/mk_core.h> -#include <fluent-bit/flb_info.h> -#include <fluent-bit/flb_mem.h> -#include <fluent-bit/flb_str.h> -#include <fluent-bit/flb_env.h> -#include <fluent-bit/flb_pipe.h> -#include <fluent-bit/flb_macros.h> -#include <fluent-bit/flb_input.h> -#include <fluent-bit/flb_input_thread.h> -#include <fluent-bit/flb_error.h> -#include <fluent-bit/flb_utils.h> -#include <fluent-bit/flb_plugin_proxy.h> -#include <fluent-bit/flb_engine.h> -#include <fluent-bit/flb_metrics.h> -#include <fluent-bit/flb_storage.h> -#include <fluent-bit/flb_downstream.h> -#include <fluent-bit/flb_plugin.h> -#include <fluent-bit/flb_kv.h> -#include <fluent-bit/flb_hash_table.h> -#include <fluent-bit/flb_scheduler.h> -#include <fluent-bit/flb_ring_buffer.h> -#include <fluent-bit/flb_processor.h> - -/* input plugin macro helpers */ -#include <fluent-bit/flb_input_plugin.h> - -#ifdef FLB_HAVE_CHUNK_TRACE -#include <fluent-bit/flb_chunk_trace.h> -#endif /* FLB_HAVE_CHUNK_TRACE */ - -struct flb_libco_in_params libco_in_param; -pthread_key_t libco_in_param_key; - -#define protcmp(a, b) strncasecmp(a, b, strlen(a)) - -/* - * Ring buffer size: we make space for 512 entries that each input instance can - * use to enqueue data. Note that this value is fixed and only affect input plugins - * which runs in threaded mode (separate thread) - * - * Ring buffer window: the current window size is set to 5% which means that the - * ring buffer will emit a flush request whenever there are 51 records or more - * awaiting to be consumed. - */ - -#define FLB_INPUT_RING_BUFFER_SIZE (sizeof(void *) * 1024) -#define FLB_INPUT_RING_BUFFER_WINDOW (5) - - -static int check_protocol(const char *prot, const char *output) -{ - int len; - - len = strlen(prot); - if (len != strlen(output)) { - return 0; - } - - if (protcmp(prot, output) != 0) { - return 0; - } - - return 1; -} - -static inline int instance_id(struct flb_input_plugin *p, - struct flb_config *config) \ -{ - int c = 0; - struct mk_list *head; - struct flb_input_instance *entry; - - mk_list_foreach(head, &config->inputs) { - entry = mk_list_entry(head, struct flb_input_instance, _head); - if (entry->id == c) { - c++; - } - } - - return c; -} - -/* Generate a new collector ID for the instance in question */ -static int collector_id(struct flb_input_instance *ins) -{ - int id = 0; - struct flb_input_collector *collector; - - if (mk_list_is_empty(&ins->collectors) == 0) { - return id; - } - - collector = mk_list_entry_last(&ins->collectors, - struct flb_input_collector, - _head); - return (collector->id + 1); -} - -void flb_input_net_default_listener(const char *listen, int port, - struct flb_input_instance *ins) -{ - /* Set default network configuration */ - if (!ins->host.listen) { - ins->host.listen = flb_sds_create(listen); - } - if (ins->host.port == 0) { - ins->host.port = port; - } -} - -/* Check input plugin's log level. - * Not for core plugins but for Golang plugins. - * Golang plugins do not have thread-local flb_worker_ctx information. */ -int flb_input_log_check(struct flb_input_instance *ins, int l) -{ - if (ins->log_level < l) { - return FLB_FALSE; - } - - return FLB_TRUE; -} - -/* Create an input plugin instance */ -struct flb_input_instance *flb_input_new(struct flb_config *config, - const char *input, void *data, - int public_only) -{ - int id; - int ret; - int flags = 0; - struct mk_list *head; - struct flb_input_plugin *plugin; - struct flb_input_instance *instance = NULL; - -/* use for locking the use of the chunk trace context. */ -#ifdef FLB_HAVE_CHUNK_TRACE - pthread_mutexattr_t attr = {0}; - pthread_mutexattr_init(&attr); -#endif - - if (!input) { - return NULL; - } - - mk_list_foreach(head, &config->in_plugins) { - plugin = mk_list_entry(head, struct flb_input_plugin, _head); - if (!check_protocol(plugin->name, input)) { - plugin = NULL; - continue; - } - - /* - * Check if the plugin is private and validate the 'public_only' - * requirement. - */ - if (public_only == FLB_TRUE && plugin->flags & FLB_INPUT_PRIVATE) { - return NULL; - } - - /* Create plugin instance */ - instance = flb_calloc(1, sizeof(struct flb_input_instance)); - if (!instance) { - flb_errno(); - return NULL; - } - instance->config = config; - - /* Get an ID */ - id = instance_id(plugin, config); - - /* Index for log Chunks (hash table) */ - instance->ht_log_chunks = flb_hash_table_create(FLB_HASH_TABLE_EVICT_NONE, - 512, 0); - if (!instance->ht_log_chunks) { - flb_free(instance); - return NULL; - } - - /* Index for metric Chunks (hash table) */ - instance->ht_metric_chunks = flb_hash_table_create(FLB_HASH_TABLE_EVICT_NONE, - 512, 0); - if (!instance->ht_metric_chunks) { - flb_hash_table_destroy(instance->ht_log_chunks); - flb_free(instance); - return NULL; - } - - /* Index for trace Chunks (hash table) */ - instance->ht_trace_chunks = flb_hash_table_create(FLB_HASH_TABLE_EVICT_NONE, - 512, 0); - if (!instance->ht_trace_chunks) { - flb_hash_table_destroy(instance->ht_log_chunks); - flb_hash_table_destroy(instance->ht_metric_chunks); - flb_free(instance); - return NULL; - } - - /* format name (with instance id) */ - snprintf(instance->name, sizeof(instance->name) - 1, - "%s.%i", plugin->name, id); - - if (plugin->type == FLB_INPUT_PLUGIN_CORE) { - instance->context = NULL; - } - else { - struct flb_plugin_proxy_context *ctx; - - ctx = flb_calloc(1, sizeof(struct flb_plugin_proxy_context)); - if (!ctx) { - flb_errno(); - flb_free(instance); - return NULL; - } - - ctx->proxy = plugin->proxy; - - instance->context = ctx; - } - - /* initialize remaining vars */ - instance->alias = NULL; - instance->id = id; - instance->flags = plugin->flags; - instance->p = plugin; - instance->tag = NULL; - instance->tag_len = 0; - instance->tag_default = FLB_FALSE; - instance->routable = FLB_TRUE; - instance->data = data; - instance->storage = NULL; - instance->storage_type = -1; - instance->log_level = -1; - instance->log_suppress_interval = -1; - instance->runs_in_coroutine = FLB_FALSE; - - /* net */ - instance->host.name = NULL; - instance->host.address = NULL; - instance->host.uri = NULL; - instance->host.listen = NULL; - instance->host.ipv6 = FLB_FALSE; - - /* Initialize list heads */ - mk_list_init(&instance->routes_direct); - mk_list_init(&instance->routes); - mk_list_init(&instance->tasks); - mk_list_init(&instance->chunks); - mk_list_init(&instance->collectors); - mk_list_init(&instance->input_coro_list); - mk_list_init(&instance->input_coro_list_destroy); - mk_list_init(&instance->downstreams); - mk_list_init(&instance->upstreams); - - /* Initialize properties list */ - flb_kv_init(&instance->properties); - flb_kv_init(&instance->net_properties); - - /* Plugin use networking */ - if (plugin->flags & (FLB_INPUT_NET | FLB_INPUT_NET_SERVER)) { - ret = flb_net_host_set(plugin->name, &instance->host, input); - if (ret != 0) { - flb_free(instance); - return NULL; - } - } - -/* initialize lock for access to chunk trace context. */ -#ifdef FLB_HAVE_CHUNK_TRACE - pthread_mutex_init(&instance->chunk_trace_lock, &attr); -#endif - - /* Parent plugin flags */ - flags = instance->flags; - if (flags & FLB_IO_TCP) { - instance->use_tls = FLB_FALSE; - } - else if (flags & FLB_IO_TLS) { - instance->use_tls = FLB_TRUE; - } - else if (flags & FLB_IO_OPT_TLS) { - /* TLS must be enabled manually in the config */ - instance->use_tls = FLB_FALSE; - instance->flags |= FLB_IO_TLS; - } - -#ifdef FLB_HAVE_TLS - instance->tls = NULL; - instance->tls_debug = -1; - instance->tls_verify = FLB_TRUE; - instance->tls_vhost = NULL; - instance->tls_ca_path = NULL; - instance->tls_ca_file = NULL; - instance->tls_crt_file = NULL; - instance->tls_key_file = NULL; - instance->tls_key_passwd = NULL; -#endif - - /* Plugin requires a co-routine context ? */ - if (plugin->flags & FLB_INPUT_CORO) { - instance->runs_in_coroutine = FLB_TRUE; - } - - /* Plugin will run in a separate thread ? */ - if (plugin->flags & FLB_INPUT_THREADED) { - instance->is_threaded = FLB_TRUE; - - } - - /* allocate a ring buffer */ - instance->rb = flb_ring_buffer_create(FLB_INPUT_RING_BUFFER_SIZE); - if (!instance->rb) { - flb_error("instance %s could not initialize ring buffer", - flb_input_name(instance)); - flb_free(instance); - return NULL; - } - - instance->mem_buf_status = FLB_INPUT_RUNNING; - instance->mem_buf_limit = 0; - instance->mem_chunks_size = 0; - instance->storage_buf_status = FLB_INPUT_RUNNING; - mk_list_add(&instance->_head, &config->inputs); - - /* processor instance */ - instance->processor = flb_processor_create(config, instance->name, instance, FLB_PLUGIN_INPUT); - } - - return instance; -} - -static inline int prop_key_check(const char *key, const char *kv, int k_len) -{ - int len; - - len = strlen(key); - - if (strncasecmp(key, kv, k_len) == 0 && len == k_len) { - return 0; - } - - return -1; -} - -struct flb_input_instance *flb_input_get_instance(struct flb_config *config, - int ins_id) -{ - struct mk_list *head; - struct flb_input_instance *ins; - - mk_list_foreach(head, &config->inputs) { - ins = mk_list_entry(head, struct flb_input_instance, _head); - if (ins->id == ins_id) { - break; - } - ins = NULL; - } - - if (!ins) { - return NULL; - } - - return ins; -} - -static void flb_input_coro_destroy(struct flb_input_coro *input_coro) -{ - flb_debug("[input coro] destroy coro_id=%i", input_coro->id); - - mk_list_del(&input_coro->_head); - flb_coro_destroy(input_coro->coro); - flb_free(input_coro); -} - -int flb_input_coro_finished(struct flb_config *config, int ins_id) -{ - struct mk_list *tmp; - struct mk_list *head; - struct flb_input_instance *ins; - struct flb_input_coro *input_coro; - - ins = flb_input_get_instance(config, ins_id); - if (!ins) { - return -1; - } - - /* Look for input coroutines that needs to be destroyed */ - mk_list_foreach_safe(head, tmp, &ins->input_coro_list_destroy) { - input_coro = mk_list_entry(head, struct flb_input_coro, _head); - flb_input_coro_destroy(input_coro); - } - - return 0; -} - -void flb_input_coro_prepare_destroy(struct flb_input_coro *input_coro) -{ - struct flb_input_instance *ins = input_coro->ins; - - /* move flb_input_coro from 'input_coro_list' to 'input_coro_list_destroy' */ - mk_list_del(&input_coro->_head); - mk_list_add(&input_coro->_head, &ins->input_coro_list_destroy); -} - -int flb_input_name_exists(const char *name, struct flb_config *config) -{ - struct mk_list *head; - struct flb_input_instance *ins; - - mk_list_foreach(head, &config->inputs) { - ins = mk_list_entry(head, struct flb_input_instance, _head); - if (strcmp(ins->name, name) == 0) { - return FLB_TRUE; - } - - if (ins->alias) { - if (strcmp(ins->alias, name) == 0) { - return FLB_TRUE; - } - } - } - - return FLB_FALSE; -} - -struct mk_event_loop *flb_input_event_loop_get(struct flb_input_instance *ins) -{ - struct flb_input_thread_instance *thi; - - if (flb_input_is_threaded(ins)) { - thi = ins->thi; - return thi->evl; - } - - return ins->config->evl; -} - -/* Override a configuration property for the given input_instance plugin */ -int flb_input_set_property(struct flb_input_instance *ins, - const char *k, const char *v) -{ - int len; - int ret; - int enabled; - ssize_t limit; - flb_sds_t tmp = NULL; - struct flb_kv *kv; - - len = strlen(k); - tmp = flb_env_var_translate(ins->config->env, v); - if (tmp) { - if (flb_sds_len(tmp) == 0) { - flb_sds_destroy(tmp); - tmp = NULL; - } - } - - /* Check if the key is a known/shared property */ - if (prop_key_check("tag", k, len) == 0 && tmp) { - flb_utils_set_plugin_string_property("tag", &ins->tag, tmp); - ins->tag_len = flb_sds_len(tmp); - ins->tag_default = FLB_FALSE; - } - else if (prop_key_check("log_level", k, len) == 0 && tmp) { - ret = flb_log_get_level_str(tmp); - flb_sds_destroy(tmp); - if (ret == -1) { - return -1; - } - ins->log_level = ret; - } - else if (prop_key_check("log_suppress_interval", k, len) == 0 && tmp) { - ret = flb_utils_time_to_seconds(tmp); - flb_sds_destroy(tmp); - if (ret == -1) { - return -1; - } - ins->log_suppress_interval = ret; - } - else if (prop_key_check("routable", k, len) == 0 && tmp) { - ins->routable = flb_utils_bool(tmp); - flb_sds_destroy(tmp); - } - else if (prop_key_check("alias", k, len) == 0 && tmp) { - flb_utils_set_plugin_string_property("alias", &ins->alias, tmp); - } - else if (prop_key_check("mem_buf_limit", k, len) == 0 && tmp) { - limit = flb_utils_size_to_bytes(tmp); - flb_sds_destroy(tmp); - if (limit == -1) { - return -1; - } - ins->mem_buf_limit = (size_t) limit; - } - else if (prop_key_check("listen", k, len) == 0) { - flb_utils_set_plugin_string_property("listen", &ins->host.listen, tmp); - } - else if (prop_key_check("host", k, len) == 0) { - flb_utils_set_plugin_string_property("host", &ins->host.name, tmp); - } - else if (prop_key_check("port", k, len) == 0) { - if (tmp) { - ins->host.port = atoi(tmp); - flb_sds_destroy(tmp); - } - } - else if (prop_key_check("ipv6", k, len) == 0 && tmp) { - ins->host.ipv6 = flb_utils_bool(tmp); - flb_sds_destroy(tmp); - } - else if (strncasecmp("net.", k, 4) == 0 && tmp) { - kv = flb_kv_item_create(&ins->net_properties, (char *) k, NULL); - if (!kv) { - if (tmp) { - flb_sds_destroy(tmp); - } - return -1; - } - kv->val = tmp; - } - -#ifdef FLB_HAVE_TLS - else if (prop_key_check("tls", k, len) == 0 && tmp) { - ins->use_tls = flb_utils_bool(tmp); - if (ins->use_tls == FLB_TRUE && ((ins->flags & FLB_IO_TLS) == 0)) { - flb_error("[config] %s does not support TLS", ins->name); - flb_sds_destroy(tmp); - return -1; - } - flb_sds_destroy(tmp); - } - else if (prop_key_check("tls.verify", k, len) == 0 && tmp) { - ins->tls_verify = flb_utils_bool(tmp); - flb_sds_destroy(tmp); - } - else if (prop_key_check("tls.debug", k, len) == 0 && tmp) { - ins->tls_debug = atoi(tmp); - flb_sds_destroy(tmp); - } - else if (prop_key_check("tls.vhost", k, len) == 0) { - flb_utils_set_plugin_string_property("tls.vhost", &ins->tls_vhost, tmp); - } - else if (prop_key_check("tls.ca_path", k, len) == 0) { - flb_utils_set_plugin_string_property("tls.ca_path", &ins->tls_ca_path, tmp); - } - else if (prop_key_check("tls.ca_file", k, len) == 0) { - flb_utils_set_plugin_string_property("tls.ca_file", &ins->tls_ca_file, tmp); - } - else if (prop_key_check("tls.crt_file", k, len) == 0) { - flb_utils_set_plugin_string_property("tls.crt_file", &ins->tls_crt_file, tmp); - } - else if (prop_key_check("tls.key_file", k, len) == 0) { - flb_utils_set_plugin_string_property("tls.key_file", &ins->tls_key_file, tmp); - } - else if (prop_key_check("tls.key_passwd", k, len) == 0) { - flb_utils_set_plugin_string_property("tls.key_passwd", &ins->tls_key_passwd, tmp); - } -#endif - else if (prop_key_check("storage.type", k, len) == 0 && tmp) { - /* Set the storage type */ - if (strcasecmp(tmp, "filesystem") == 0) { - ins->storage_type = FLB_STORAGE_FS; - } - else if (strcasecmp(tmp, "memory") == 0) { - ins->storage_type = FLB_STORAGE_MEM; - } - else if (strcasecmp(tmp, "memrb") == 0) { - ins->storage_type = FLB_STORAGE_MEMRB; - } - else { - flb_sds_destroy(tmp); - return -1; - } - flb_sds_destroy(tmp); - } - else if (prop_key_check("threaded", k, len) == 0 && tmp) { - enabled = flb_utils_bool(tmp); - flb_sds_destroy(tmp); - - if (enabled == -1) { - return -1; - } - - ins->is_threaded = enabled; - } - else if (prop_key_check("storage.pause_on_chunks_overlimit", k, len) == 0 && tmp) { - if (ins->storage_type == FLB_STORAGE_FS) { - ret = flb_utils_bool(tmp); - flb_sds_destroy(tmp); - if (ret == -1) { - return -1; - } - ins->storage_pause_on_chunks_overlimit = ret; - } - } - else { - /* - * Create the property, we don't pass the value since we will - * map it directly to avoid an extra memory allocation. - */ - kv = flb_kv_item_create(&ins->properties, (char *) k, NULL); - if (!kv) { - if (tmp) { - flb_sds_destroy(tmp); - } - return -1; - } - kv->val = tmp; - } - - return 0; -} - -const char *flb_input_get_property(const char *key, - struct flb_input_instance *ins) -{ - return flb_config_prop_get(key, &ins->properties); -} - -#ifdef FLB_HAVE_METRICS -void *flb_input_get_cmt_instance(struct flb_input_instance *ins) -{ - return (void *)ins->cmt; -} -#endif - -/* Return an instance name or alias */ -const char *flb_input_name(struct flb_input_instance *ins) -{ - if (ins->alias) { - return ins->alias; - } - - return ins->name; -} - -void flb_input_instance_destroy(struct flb_input_instance *ins) -{ - struct mk_list *tmp; - struct mk_list *head; - struct flb_input_collector *collector; - - if (ins->alias) { - flb_sds_destroy(ins->alias); - } - - /* Remove URI context */ - if (ins->host.uri) { - flb_uri_destroy(ins->host.uri); - } - - if (ins->host.name) { - flb_sds_destroy(ins->host.name); - } - if (ins->host.address) { - flb_sds_destroy(ins->host.address); - } - if (ins->host.listen) { - flb_sds_destroy(ins->host.listen); - } - -#ifdef FLB_HAVE_TLS - if (ins->use_tls) { - if (ins->tls != NULL) { - flb_tls_destroy(ins->tls); - } - } - - if (ins->tls_config_map) { - flb_config_map_destroy(ins->tls_config_map); - } -#endif - - if (ins->tls_vhost) { - flb_sds_destroy(ins->tls_vhost); - } - - if (ins->tls_ca_path) { - flb_sds_destroy(ins->tls_ca_path); - } - - if (ins->tls_ca_file) { - flb_sds_destroy(ins->tls_ca_file); - } - - if (ins->tls_crt_file) { - flb_sds_destroy(ins->tls_crt_file); - } - - if (ins->tls_key_file) { - flb_sds_destroy(ins->tls_key_file); - } - - if (ins->tls_key_passwd) { - flb_sds_destroy(ins->tls_key_passwd); - } - - /* release the tag if any */ - flb_sds_destroy(ins->tag); - - /* Let the engine remove any pending task */ - flb_engine_destroy_tasks(&ins->tasks); - - /* release properties */ - flb_kv_release(&ins->properties); - flb_kv_release(&ins->net_properties); - - -#ifdef FLB_HAVE_CHUNK_TRACE - flb_chunk_trace_context_destroy(ins); -#endif /* FLB_HAVE_CHUNK_TRACE */ - - /* Remove metrics */ -#ifdef FLB_HAVE_METRICS - if (ins->cmt) { - cmt_destroy(ins->cmt); - } - - if (ins->metrics) { - flb_metrics_destroy(ins->metrics); - } -#endif - - if (ins->storage) { - flb_storage_input_destroy(ins); - } - - /* destroy config map */ - if (ins->config_map) { - flb_config_map_destroy(ins->config_map); - } - - if (ins->net_config_map) { - flb_config_map_destroy(ins->net_config_map); - } - - /* hash table for chunks */ - if (ins->ht_log_chunks) { - flb_hash_table_destroy(ins->ht_log_chunks); - } - - if (ins->ht_metric_chunks) { - flb_hash_table_destroy(ins->ht_metric_chunks); - } - - if (ins->ht_trace_chunks) { - flb_hash_table_destroy(ins->ht_trace_chunks); - } - - if (ins->ch_events[0] > 0) { - mk_event_closesocket(ins->ch_events[0]); - } - - if (ins->ch_events[1] > 0) { - mk_event_closesocket(ins->ch_events[1]); - } - - /* Collectors */ - mk_list_foreach_safe(head, tmp, &ins->collectors) { - collector = mk_list_entry(head, struct flb_input_collector, _head); - mk_list_del(&collector->_head); - flb_input_collector_destroy(collector); - } - - /* delete storage context */ - flb_storage_input_destroy(ins); - - mk_list_del(&ins->_head); - - /* ring buffer */ - if (ins->rb) { - flb_input_chunk_ring_buffer_cleanup(ins); - flb_ring_buffer_destroy(ins->rb); - } - - /* processor */ - if (ins->processor) { - flb_processor_destroy(ins->processor); - } - flb_free(ins); -} - -int flb_input_coro_id_get(struct flb_input_instance *ins) -{ - int id; - int max = (2 << 13) - 1; /* max for 14 bits */ - - id = ins->input_coro_id; - ins->input_coro_id++; - - /* reset once it reach the maximum allowed */ - if (ins->input_coro_id > max) { - ins->input_coro_id = 0; - } - - return id; -} - -static int input_instance_channel_events_init(struct flb_input_instance *ins) -{ - int ret; - struct mk_event_loop *evl; - - evl = flb_input_event_loop_get(ins); - - /* Input event channel: used for co-routines to report return status */ - ret = mk_event_channel_create(evl, - &ins->ch_events[0], - &ins->ch_events[1], - ins); - if (ret != 0) { - flb_error("could not create events channels for '%s'", - flb_input_name(ins)); - return -1; - } - - flb_debug("[%s:%s] created event channels: read=%i write=%i", - ins->p->name, flb_input_name(ins), - ins->ch_events[0], ins->ch_events[1]); - - /* - * Note: mk_event_channel_create() sets a type = MK_EVENT_NOTIFICATION by - * default, we need to overwrite this value so we can do a clean check - * into the Engine when the event is triggered. - */ - ins->event.type = FLB_ENGINE_EV_INPUT; - - return 0; -} - -int flb_input_net_property_check(struct flb_input_instance *ins, - struct flb_config *config) -{ - int ret = 0; - - /* Get Downstream net_setup configmap */ - ins->net_config_map = flb_downstream_get_config_map(config); - if (!ins->net_config_map) { - flb_input_instance_destroy(ins); - return -1; - } - - /* - * Validate 'net.*' properties: if the plugin use the Downstream interface, - * it might receive some networking settings. - */ - if (mk_list_size(&ins->net_properties) > 0) { - ret = flb_config_map_properties_check(ins->p->name, - &ins->net_properties, - ins->net_config_map); - if (ret == -1) { - if (config->program_name) { - flb_helper("try the command: %s -i %s -h\n", - config->program_name, ins->p->name); - } - return -1; - } - } - - return 0; -} - -int flb_input_plugin_property_check(struct flb_input_instance *ins, - struct flb_config *config) -{ - int ret = 0; - struct mk_list *config_map; - struct flb_input_plugin *p = ins->p; - - if (p->config_map) { - /* - * Create a dynamic version of the configmap that will be used by the specific - * instance in question. - */ - config_map = flb_config_map_create(config, p->config_map); - if (!config_map) { - flb_error("[input] error loading config map for '%s' plugin", - p->name); - flb_input_instance_destroy(ins); - return -1; - } - ins->config_map = config_map; - - /* Validate incoming properties against config map */ - ret = flb_config_map_properties_check(ins->p->name, - &ins->properties, ins->config_map); - if (ret == -1) { - if (config->program_name) { - flb_helper("try the command: %s -i %s -h\n", - config->program_name, ins->p->name); - } - return -1; - } - } - - return 0; -} - -int flb_input_instance_init(struct flb_input_instance *ins, - struct flb_config *config) -{ - int ret; - struct flb_config *ctx = ins->config; - struct flb_input_plugin *p = ins->p; - int tls_session_mode; - - if (ins->log_level == -1 && config->log != NULL) { - ins->log_level = config->log->level; - } - - /* Skip pseudo input plugins */ - if (!p) { - return 0; - } - - -#ifdef FLB_HAVE_METRICS - uint64_t ts; - char *name; - - name = (char *) flb_input_name(ins); - ts = cfl_time_now(); - - /* CMetrics */ - ins->cmt = cmt_create(); - if (!ins->cmt) { - flb_error("[input] could not create cmetrics context: %s", - flb_input_name(ins)); - return -1; - } - - /* - * Register generic input plugin metrics - * ------------------------------------- - */ - - /* fluentbit_input_bytes_total */ - ins->cmt_bytes = \ - cmt_counter_create(ins->cmt, - "fluentbit", "input", "bytes_total", - "Number of input bytes.", - 1, (char *[]) {"name"}); - cmt_counter_set(ins->cmt_bytes, ts, 0, 1, (char *[]) {name}); - - /* fluentbit_input_records_total */ - ins->cmt_records = \ - cmt_counter_create(ins->cmt, - "fluentbit", "input", "records_total", - "Number of input records.", - 1, (char *[]) {"name"}); - cmt_counter_set(ins->cmt_records, ts, 0, 1, (char *[]) {name}); - - /* Storage Metrics */ - if (ctx->storage_metrics == FLB_TRUE) { - /* fluentbit_input_storage_overlimit */ - ins->cmt_storage_overlimit = \ - cmt_gauge_create(ins->cmt, - "fluentbit", "input", - "storage_overlimit", - "Is the input memory usage overlimit ?.", - 1, (char *[]) {"name"}); - cmt_gauge_set(ins->cmt_storage_overlimit, ts, 0, 1, (char *[]) {name}); - - /* fluentbit_input_storage_memory_bytes */ - ins->cmt_storage_memory_bytes = \ - cmt_gauge_create(ins->cmt, - "fluentbit", "input", - "storage_memory_bytes", - "Memory bytes used by the chunks.", - 1, (char *[]) {"name"}); - cmt_gauge_set(ins->cmt_storage_memory_bytes, ts, 0, 1, (char *[]) {name}); - - /* fluentbit_input_storage_chunks */ - ins->cmt_storage_chunks = \ - cmt_gauge_create(ins->cmt, - "fluentbit", "input", - "storage_chunks", - "Total number of chunks.", - 1, (char *[]) {"name"}); - cmt_gauge_set(ins->cmt_storage_chunks, ts, 0, 1, (char *[]) {name}); - - /* fluentbit_input_storage_chunks_up */ - ins->cmt_storage_chunks_up = \ - cmt_gauge_create(ins->cmt, - "fluentbit", "input", - "storage_chunks_up", - "Total number of chunks up in memory.", - 1, (char *[]) {"name"}); - cmt_gauge_set(ins->cmt_storage_chunks_up, ts, 0, 1, (char *[]) {name}); - - /* fluentbit_input_storage_chunks_down */ - ins->cmt_storage_chunks_down = \ - cmt_gauge_create(ins->cmt, - "fluentbit", "input", - "storage_chunks_down", - "Total number of chunks down.", - 1, (char *[]) {"name"}); - cmt_gauge_set(ins->cmt_storage_chunks_down, ts, 0, 1, (char *[]) {name}); - - /* fluentbit_input_storage_chunks_busy */ - ins->cmt_storage_chunks_busy = \ - cmt_gauge_create(ins->cmt, - "fluentbit", "input", - "storage_chunks_busy", - "Total number of chunks in a busy state.", - 1, (char *[]) {"name"}); - cmt_gauge_set(ins->cmt_storage_chunks_busy, ts, 0, 1, (char *[]) {name}); - - /* fluentbit_input_storage_chunks_busy_bytes */ - ins->cmt_storage_chunks_busy_bytes = \ - cmt_gauge_create(ins->cmt, - "fluentbit", "input", - "storage_chunks_busy_bytes", - "Total number of bytes used by chunks in a busy state.", - 1, (char *[]) {"name"}); - cmt_gauge_set(ins->cmt_storage_chunks_busy_bytes, ts, 0, 1, (char *[]) {name}); - } - - if (ins->storage_type == FLB_STORAGE_MEMRB) { - /* fluentbit_input_memrb_dropped_chunks */ - ins->cmt_memrb_dropped_chunks = cmt_counter_create(ins->cmt, - "fluentbit", "input", - "memrb_dropped_chunks", - "Number of memrb dropped chunks.", - 1, (char *[]) {"name"}); - cmt_counter_set(ins->cmt_memrb_dropped_chunks, ts, 0, 1, (char *[]) {name}); - - - /* fluentbit_input_memrb_dropped_bytes */ - ins->cmt_memrb_dropped_bytes = cmt_counter_create(ins->cmt, - "fluentbit", "input", - "memrb_dropped_bytes", - "Number of memrb dropped bytes.", - 1, (char *[]) {"name"}); - - cmt_counter_set(ins->cmt_memrb_dropped_bytes, ts, 0, 1, (char *[]) {name}); - } - - /* OLD Metrics */ - ins->metrics = flb_metrics_create(name); - if (ins->metrics) { - flb_metrics_add(FLB_METRIC_N_RECORDS, "records", ins->metrics); - flb_metrics_add(FLB_METRIC_N_BYTES, "bytes", ins->metrics); - } -#endif - - /* - * Before to call the initialization callback, make sure that the received - * configuration parameters are valid if the plugin is registering a config map. - */ - if (flb_input_plugin_property_check(ins, config) == -1) { - return -1; - } - -#ifdef FLB_HAVE_TLS - if (ins->use_tls == FLB_TRUE) { - if ((p->flags & FLB_INPUT_NET_SERVER) != 0) { - if (ins->tls_crt_file == NULL) { - flb_error("[input %s] error initializing TLS context " - "(certificate file missing)", - ins->name); - - return -1; - } - else if (ins->tls_key_file == NULL) { - flb_error("[input %s] error initializing TLS context " - "(private key file missing)", - ins->name); - - return -1; - } - - tls_session_mode = FLB_TLS_SERVER_MODE; - } - else { - tls_session_mode = FLB_TLS_CLIENT_MODE; - } - - ins->tls = flb_tls_create(tls_session_mode, - ins->tls_verify, - ins->tls_debug, - ins->tls_vhost, - ins->tls_ca_path, - ins->tls_ca_file, - ins->tls_crt_file, - ins->tls_key_file, - ins->tls_key_passwd); - - if (ins->tls == NULL) { - flb_error("[input %s] error initializing TLS context", - ins->name); - - return -1; - } - } - - struct flb_config_map *m; - - /* TLS config map (just for 'help' formatting purposes) */ - ins->tls_config_map = flb_tls_get_config_map(config); - - if (ins->tls_config_map == NULL) { - return -1; - } - - /* Override first configmap value based on it plugin flag */ - m = mk_list_entry_first(ins->tls_config_map, struct flb_config_map, _head); - if (p->flags & FLB_IO_TLS) { - m->value.val.boolean = FLB_TRUE; - } - else { - m->value.val.boolean = FLB_FALSE; - } -#endif - - /* Init network defaults */ - flb_net_setup_init(&ins->net_setup); - - if (flb_input_net_property_check(ins, config) == -1) { - return -1; - } - - /* Initialize the input */ - if (p->cb_init) { - flb_plg_info(ins, "initializing"); - flb_plg_info(ins, "storage_strategy=%s", flb_storage_get_type(ins->storage_type)); - - /* Sanity check: all non-dynamic tag input plugins must have a tag */ - if (!ins->tag) { - flb_input_set_property(ins, "tag", ins->name); - ins->tag_default = FLB_TRUE; - } - - if (flb_input_is_threaded(ins)) { - /* - * Create a thread for a new instance. Now the plugin initialization callback will be invoked and report an early failure - * or an 'ok' status, we will wait for that return value on flb_input_thread_instance_get_status() below. - */ - ret = flb_input_thread_instance_init(config, ins); - if (ret != 0) { - flb_error("failed initialize input %s", - ins->name); - return -1; - } - - /* initialize channel events */ - ret = input_instance_channel_events_init(ins); - if (ret != 0) { - flb_error("failed initialize channel events on input %s", - ins->name); - return -1; - } - - /* register the ring buffer */ - ret = flb_ring_buffer_add_event_loop(ins->rb, config->evl, FLB_INPUT_RING_BUFFER_WINDOW); - if (ret) { - flb_error("failed while registering ring buffer events on input %s", - ins->name); - return -1; - } - } - else { - /* initialize channel events */ - ret = input_instance_channel_events_init(ins); - if (ret != 0) { - flb_error("failed initialize channel events on input %s", - ins->name); - } - ret = p->cb_init(ins, config, ins->data); - if (ret != 0) { - flb_error("failed initialize input %s", - ins->name); - return -1; - } - } - } - - /* initialize processors */ - ret = flb_processor_init(ins->processor); - if (ret == -1) { - return -1; - } - - return 0; -} - -int flb_input_instance_pre_run(struct flb_input_instance *ins, struct flb_config *config) -{ - int ret; - - if (flb_input_is_threaded(ins)) { - return flb_input_thread_instance_pre_run(config, ins); - } - else if (ins->p->cb_pre_run) { - ret = ins->p->cb_pre_run(ins, config, ins->context); - if (ret == -1) { - return -1; - } - return 0; - } - - return 0; -} - -/* Initialize all inputs */ -int flb_input_init_all(struct flb_config *config) -{ - int ret; - struct mk_list *tmp; - struct mk_list *head; - struct flb_input_instance *ins; - struct flb_input_plugin *p; - - /* Initialize thread-id table */ - memset(&config->in_table_id, '\0', sizeof(config->in_table_id)); - - /* Iterate all active input instance plugins */ - mk_list_foreach_safe(head, tmp, &config->inputs) { - ins = mk_list_entry(head, struct flb_input_instance, _head); - p = ins->p; - - /* Skip pseudo input plugins */ - if (!p) { - continue; - } - - /* Initialize instance */ - ret = flb_input_instance_init(ins, config); - if (ret == -1) { - flb_input_instance_destroy(ins); - return -1; - } - } - - return 0; -} - -/* Invoke all pre-run input callbacks */ -void flb_input_pre_run_all(struct flb_config *config) -{ - struct mk_list *head; - struct flb_input_instance *ins; - struct flb_input_plugin *p; - - mk_list_foreach(head, &config->inputs) { - ins = mk_list_entry(head, struct flb_input_instance, _head); - p = ins->p; - if (!p) { - continue; - } - - flb_input_instance_pre_run(ins, config); - } -} - -void flb_input_instance_exit(struct flb_input_instance *ins, - struct flb_config *config) -{ - struct flb_input_plugin *p; - - /* if the instance runs in a separate thread, signal the thread */ - if (flb_input_is_threaded(ins)) { - flb_input_thread_instance_exit(ins); - return; - } - - p = ins->p; - if (p->cb_exit && ins->context) { - /* Multi-threaded input plugins use the same function signature for exit callbacks. */ - p->cb_exit(ins->context, config); - } -} - -/* Invoke all exit input callbacks */ -void flb_input_exit_all(struct flb_config *config) -{ - struct mk_list *tmp; - struct mk_list *head; - struct flb_input_instance *ins; - struct flb_input_plugin *p; - - /* Iterate instances */ - mk_list_foreach_safe_r(head, tmp, &config->inputs) { - ins = mk_list_entry(head, struct flb_input_instance, _head); - p = ins->p; - if (!p) { - continue; - } - - /* invoke plugin instance exit callback */ - flb_input_instance_exit(ins, config); - - /* destroy the instance */ - flb_input_instance_destroy(ins); - } -} - -/* Check that at least one Input is enabled */ -int flb_input_check(struct flb_config *config) -{ - if (mk_list_is_empty(&config->inputs) == 0) { - return -1; - } - - return 0; -} - -/* - * API for Input plugins - * ===================== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * The Input interface provides a certain number of functions that can be - * used by Input plugins to configure it own behavior and request specific - * - * 1. flb_input_set_context() - * - * let an Input plugin set a context data reference that can be used - * later when invoking other callbacks. - * - * 2. flb_input_set_collector_time() - * - * request the Engine to trigger a specific collector callback at a - * certain interval time. Note that this callback will run in the main - * thread so it computing time must be short, otherwise it will block - * the main loop. - * - * The collector can runs in timeouts of the order of seconds.nanoseconds - * - * note: 1 Second = 1000000000 Nanosecond - * - * 3. flb_input_set_collector_event() - * - * for a registered file descriptor, associate the READ events to a - * specified plugin. Every time there is some data to read, the collector - * callback will be triggered. Oriented to a file descriptor that already - * have information that may be read through iotctl(..FIONREAD..); - * - * 4. flb_input_set_collector_server() - * - * it register a collector based on TCP socket events. It register a socket - * who did bind() and listen() and for each event on the socket it triggers - * the registered callbacks. - */ - -/* Assign an Configuration context to an Input */ -void flb_input_set_context(struct flb_input_instance *in, void *context) -{ - in->context = context; -} - -int flb_input_channel_init(struct flb_input_instance *in) -{ - return flb_pipe_create(in->channel); -} - -static struct flb_input_collector *collector_create(int type, - struct flb_input_instance *ins, - int (*cb) ( - struct flb_input_instance *, - struct flb_config *, void *), - struct flb_config *config) -{ - struct flb_input_collector *coll; - struct flb_input_thread_instance *thi; - - coll = flb_calloc(1, sizeof(struct flb_input_collector)); - if (!coll) { - flb_errno(); - return NULL; - } - - coll->id = collector_id(ins); - coll->type = type; - coll->running = FLB_FALSE; - coll->fd_event = -1; - coll->fd_timer = -1; - coll->seconds = -1; - coll->nanoseconds = -1; - coll->cb_collect = cb; - coll->instance = ins; - MK_EVENT_ZERO(&coll->event); - - if (flb_input_is_threaded(ins)) { - thi = ins->thi; - coll->evl = thi->evl; - } - else { - coll->evl = config->evl; - } - - /* - * Collectors created from a threaded input instance are only added to the - * instance `collectors` list. For instances in non-threaded mode, they are - * added to both lists, the global config collectors list and the instance - * list. - */ - mk_list_add(&coll->_head, &ins->collectors); - - return coll; -} - - -int flb_input_set_collector_time(struct flb_input_instance *ins, - int (*cb_collect) (struct flb_input_instance *, - struct flb_config *, void *), - time_t seconds, - long nanoseconds, - struct flb_config *config) -{ - struct flb_input_collector *coll; - - coll = collector_create(FLB_COLLECT_TIME, ins, cb_collect, config); - if (!coll) { - return -1; - } - - /* specific collector initialization */ - coll->seconds = seconds; - coll->nanoseconds = nanoseconds; - - return coll->id; -} - -int flb_input_set_collector_event(struct flb_input_instance *ins, - int (*cb_collect) (struct flb_input_instance *, - struct flb_config *, void *), - flb_pipefd_t fd, - struct flb_config *config) -{ - struct flb_input_collector *coll; - - coll = collector_create(FLB_COLLECT_FD_EVENT, ins, cb_collect, config); - if (!coll) { - return -1; - } - - /* specific collector initialization */ - coll->fd_event = fd; - - return coll->id; -} - -int flb_input_set_collector_socket(struct flb_input_instance *ins, - int (*cb_new_connection) (struct flb_input_instance *, - struct flb_config *, - void *), - flb_pipefd_t fd, - struct flb_config *config) -{ - struct flb_input_collector *coll; - - - coll = collector_create(FLB_COLLECT_FD_SERVER, ins, cb_new_connection, config); - if (!coll) { - return -1; - } - - /* specific collector initialization */ - coll->fd_event = fd; - - return coll->id; -} - - -static int collector_start(struct flb_input_collector *coll, - struct flb_config *config) -{ - int fd; - int ret; - struct mk_event *event; - - if (coll->running == FLB_TRUE) { - return 0; - } - - event = &coll->event; - event->mask = MK_EVENT_EMPTY; - event->status = MK_EVENT_NONE; - - if (coll->type == FLB_COLLECT_TIME) { - fd = mk_event_timeout_create(coll->evl, coll->seconds, - coll->nanoseconds, event); - if (fd == -1) { - flb_error("[input collector] COLLECT_TIME registration failed"); - coll->running = FLB_FALSE; - return -1; - } - coll->fd_timer = fd; - } - else if (coll->type & (FLB_COLLECT_FD_EVENT | FLB_COLLECT_FD_SERVER)) { - event->fd = coll->fd_event; - ret = mk_event_add(coll->evl, - coll->fd_event, - FLB_ENGINE_EV_CORE, - MK_EVENT_READ, event); - if (ret == -1) { - flb_error("[input collector] COLLECT_EVENT registration failed"); - mk_event_closesocket(coll->fd_event); - coll->running = FLB_FALSE; - return -1; - } - } - - coll->running = FLB_TRUE; - return 0; -} - -int flb_input_collector_start(int coll_id, struct flb_input_instance *in) -{ - int ret; - struct mk_list *head; - struct flb_input_collector *coll; - - mk_list_foreach(head, &in->collectors) { - coll = mk_list_entry(head, struct flb_input_collector, _head); - if (coll->id == coll_id) { - ret = collector_start(coll, in->config); - if (ret == -1) { - flb_error("[input] error starting collector #%i: %s", - coll_id, in->name); - } - return ret; - } - } - - return -1; -} - -/* start collectors for main thread, no threaded plugins */ -int flb_input_collectors_signal_start(struct flb_input_instance *ins) -{ - int ret; - struct mk_list *head; - struct flb_input_collector *coll; - - if (flb_input_is_threaded(ins)) { - flb_error("input plugin '%s' is threaded", flb_input_name(ins)); - return -1; - } - - mk_list_foreach(head, &ins->collectors) { - coll = mk_list_entry(head, struct flb_input_collector, _head); - ret = flb_input_collector_start(coll->id, ins); - if (ret < 0) { - return -1; - } - } - - return 0; -} - -/* - * Start all collectors: this function is invoked from the engine interface and aim - * to start the local collectors and also signal the threaded input plugins to start - * their own collectors. - */ -int flb_input_collectors_start(struct flb_config *config) -{ - int ret; - struct mk_list *head; - struct flb_input_instance *ins; - - /* Signal threaded input plugins to start their collectors */ - mk_list_foreach(head, &config->inputs) { - ins = mk_list_entry(head, struct flb_input_instance, _head); - if (flb_input_is_threaded(ins)) { - ret = flb_input_thread_collectors_signal_start(ins); - if (ret != 0) { - flb_error("could not start collectors for threaded plugin '%s'", - flb_input_name(ins)); - } - } - else { - ret = flb_input_collectors_signal_start(ins); - if (ret != 0) { - flb_error("could not start collectors for plugin '%s'", - flb_input_name(ins)); - } - } - } - - return 0; -} - -static struct flb_input_collector *get_collector(int id, - struct flb_input_instance *in) -{ - struct mk_list *head; - struct flb_input_collector *coll; - - mk_list_foreach(head, &in->collectors) { - coll = mk_list_entry(head, struct flb_input_collector, _head); - if (coll->id == id) { - return coll; - } - } - - return NULL; -} - -int flb_input_collector_running(int coll_id, struct flb_input_instance *in) -{ - struct flb_input_collector *coll; - - coll = get_collector(coll_id, in); - if (!coll) { - return FLB_FALSE; - } - - return coll->running; -} - -struct mk_event *flb_input_collector_get_event(int coll_id, - struct flb_input_instance *ins) -{ - struct flb_input_collector *collector; - - collector = get_collector(coll_id, ins); - - if (collector == NULL) { - return NULL; - } - - return &collector->event; -} - -/* - * TEST: this is a test function that can be used by input plugins to check the - * 'pause' and 'resume' callback operations. - * - * After is invoked, it will schedule an internal event to wake up the instance - * after 'sleep_seconds'. - */ -int flb_input_test_pause_resume(struct flb_input_instance *ins, int sleep_seconds) -{ - /* - * This is a fake pause/resume implementation since it's only used to test the plugin - * callbacks for such purposes. - */ - - /* pause the instance */ - flb_input_pause(ins); - - /* wait */ - sleep(sleep_seconds); - - /* resume again */ - flb_input_resume(ins); - - return 0; -} - -int flb_input_pause(struct flb_input_instance *ins) -{ - /* if the instance is already paused, just return */ - if (flb_input_buf_paused(ins)) { - return -1; - } - - /* Pause only if a callback is set and a local context exists */ - if (ins->p->cb_pause && ins->context) { - if (flb_input_is_threaded(ins)) { - /* signal the thread event loop about the 'pause' operation */ - flb_input_thread_instance_pause(ins); - } - else { - flb_info("[input] pausing %s", flb_input_name(ins)); - ins->p->cb_pause(ins->context, ins->config); - } - } - - return 0; -} - -int flb_input_resume(struct flb_input_instance *ins) -{ - if (ins->p->cb_resume) { - if (flb_input_is_threaded(ins)) { - /* signal the thread event loop about the 'resume' operation */ - flb_input_thread_instance_resume(ins); - } - else { - ins->p->cb_resume(ins->context, ins->config); - } - } - - return 0; -} - -int flb_input_pause_all(struct flb_config *config) -{ - int ret; - int paused = 0; - struct mk_list *head; - struct flb_input_instance *ins; - - mk_list_foreach(head, &config->inputs) { - ins = mk_list_entry(head, struct flb_input_instance, _head); - /* - * Inform the plugin that is being paused, the source type is set to 'FLB_INPUT_PAUSE_MEM_BUF', no real reason, we - * just need to get it paused. - */ - ret = flb_input_pause(ins); - if (ret == 0) { - paused++; - } - } - - return paused; -} - -int flb_input_collector_destroy(struct flb_input_collector *coll) -{ - struct flb_config *config = coll->instance->config; - - if (coll->type == FLB_COLLECT_TIME) { - if (coll->fd_timer > 0) { - mk_event_timeout_destroy(config->evl, &coll->event); - mk_event_closesocket(coll->fd_timer); - } - } - else { - mk_event_del(config->evl, &coll->event); - } - - flb_free(coll); - - return 0; -} - -int flb_input_collector_pause(int coll_id, struct flb_input_instance *in) -{ - int ret; - flb_pipefd_t fd; - struct flb_input_collector *coll; - - coll = get_collector(coll_id, in); - if (!coll) { - return -1; - } - - if (coll->running == FLB_FALSE) { - return 0; - } - - if (coll->type == FLB_COLLECT_TIME) { - /* - * For a collector time, it's better to just remove the file - * descriptor associated to the time out, when resumed a new - * one can be created. - * - * Note: Invalidate fd_timer first in case closing a socket - * invokes another event. - */ - fd = coll->fd_timer; - coll->fd_timer = -1; - mk_event_timeout_destroy(coll->evl, &coll->event); - mk_event_closesocket(fd); - } - else if (coll->type & (FLB_COLLECT_FD_SERVER | FLB_COLLECT_FD_EVENT)) { - ret = mk_event_del(coll->evl, &coll->event); - if (ret != 0) { - flb_warn("[input] cannot disable event for %s", in->name); - return -1; - } - } - - coll->running = FLB_FALSE; - - return 0; -} - -int flb_input_collector_delete(int coll_id, struct flb_input_instance *in) -{ - struct flb_input_collector *coll; - - coll = get_collector(coll_id, in); - if (!coll) { - return -1; - } - if (flb_input_collector_pause(coll_id, in) < 0) { - return -1; - } - - - pthread_mutex_lock(&in->config->collectors_mutex); - mk_list_del(&coll->_head); - pthread_mutex_unlock(&in->config->collectors_mutex); - - flb_free(coll); - return 0; -} - -int flb_input_collector_resume(int coll_id, struct flb_input_instance *in) -{ - int fd; - int ret; - struct flb_input_collector *coll; - struct flb_config *config; - struct mk_event *event; - - coll = get_collector(coll_id, in); - if (!coll) { - return -1; - } - - if (coll->running == FLB_TRUE) { - flb_error("[input] cannot resume collector %s:%i, already running", - in->name, coll_id); - return -1; - } - - config = in->config; - event = &coll->event; - - /* If data ingestion has been paused, the collector cannot resume */ - if (config->is_ingestion_active == FLB_FALSE) { - return 0; - } - - if (coll->type == FLB_COLLECT_TIME) { - event->mask = MK_EVENT_EMPTY; - event->status = MK_EVENT_NONE; - fd = mk_event_timeout_create(coll->evl, coll->seconds, - coll->nanoseconds, event); - if (fd == -1) { - flb_error("[input collector] resume COLLECT_TIME failed"); - return -1; - } - coll->fd_timer = fd; - } - else if (coll->type & (FLB_COLLECT_FD_SERVER | FLB_COLLECT_FD_EVENT)) { - event->fd = coll->fd_event; - event->mask = MK_EVENT_EMPTY; - event->status = MK_EVENT_NONE; - - ret = mk_event_add(coll->evl, - coll->fd_event, - FLB_ENGINE_EV_CORE, - MK_EVENT_READ, event); - if (ret == -1) { - flb_error("[input] cannot disable/pause event for %s", in->name); - return -1; - } - } - - coll->running = FLB_TRUE; - - return 0; -} - -int flb_input_collector_fd(flb_pipefd_t fd, struct flb_config *config) -{ - struct mk_list *head; - struct mk_list *head_coll; - struct flb_input_instance *ins; - struct flb_input_collector *collector = NULL; - struct flb_input_coro *input_coro; - - mk_list_foreach(head, &config->inputs) { - ins = mk_list_entry(head, struct flb_input_instance, _head); - mk_list_foreach(head_coll, &ins->collectors) { - collector = mk_list_entry(head_coll, struct flb_input_collector, _head); - if (collector->fd_event == fd) { - break; - } - else if (collector->fd_timer == fd) { - flb_utils_timer_consume(fd); - break; - } - collector = NULL; - } - - if (collector) { - break; - } - } - - /* No matches */ - if (!collector) { - return -1; - } - - if (collector->running == FLB_FALSE) { - return -1; - } - - /* Trigger the collector callback */ - if (collector->instance->runs_in_coroutine) { - input_coro = flb_input_coro_collect(collector, config); - if (!input_coro) { - return -1; - } - flb_input_coro_resume(input_coro); - } - else { - if (collector->cb_collect(collector->instance, config, - collector->instance->context) == -1) { - return -1; - } - } - - return 0; -} - -int flb_input_upstream_set(struct flb_upstream *u, struct flb_input_instance *ins) -{ - if (!u) { - return -1; - } - - /* - * if the input instance runs in threaded mode, make sure to flag the - * upstream context so the lists operations are done in thread safe mode - */ - if (flb_input_is_threaded(ins)) { - flb_upstream_thread_safe(u); - mk_list_add(&u->base._head, &ins->upstreams); - } - - /* Set networking options 'net.*' received through instance properties */ - memcpy(&u->base.net, &ins->net_setup, sizeof(struct flb_net_setup)); - - return 0; -} - -int flb_input_downstream_set(struct flb_downstream *stream, - struct flb_input_instance *ins) -{ - if (stream == NULL) { - return -1; - } - - /* - * If the input plugin will run in multiple threads, enable - * the thread safe mode for the Downstream context. - */ - if (flb_input_is_threaded(ins)) { - flb_stream_enable_thread_safety(&stream->base); - - mk_list_add(&stream->base._head, &ins->downstreams); - } - - return 0; -} diff --git a/fluent-bit/src/flb_input_chunk.c b/fluent-bit/src/flb_input_chunk.c deleted file mode 100644 index c71ae3ef0..000000000 --- a/fluent-bit/src/flb_input_chunk.c +++ /dev/null @@ -1,2009 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#define FS_CHUNK_SIZE_DEBUG(op) {flb_trace("[%d] %s -> fs_chunks_size = %zu", \ - __LINE__, op->name, op->fs_chunks_size);} -#define FS_CHUNK_SIZE_DEBUG_MOD(op, chunk, mod) {flb_trace( \ - "[%d] %s -> fs_chunks_size = %zu mod=%zd chunk=%s", __LINE__, \ - op->name, op->fs_chunks_size, mod, flb_input_chunk_get_name(chunk));} - -#include <fluent-bit/flb_info.h> -#include <fluent-bit/flb_config.h> -#include <fluent-bit/flb_input.h> -#include <fluent-bit/flb_input_chunk.h> -#include <fluent-bit/flb_input_plugin.h> -#include <fluent-bit/flb_storage.h> -#include <fluent-bit/flb_time.h> -#include <fluent-bit/flb_router.h> -#include <fluent-bit/flb_task.h> -#include <fluent-bit/flb_routes_mask.h> -#include <fluent-bit/flb_metrics.h> -#include <fluent-bit/stream_processor/flb_sp.h> -#include <fluent-bit/flb_ring_buffer.h> -#include <chunkio/chunkio.h> -#include <monkey/mk_core.h> - - -#ifdef FLB_HAVE_CHUNK_TRACE -#include <fluent-bit/flb_chunk_trace.h> -#endif /* FLB_HAVE_CHUNK_TRACE */ - - -#define BLOCK_UNTIL_KEYPRESS() {char temp_keypress_buffer; read(0, &temp_keypress_buffer, 1);} - -#define FLB_INPUT_CHUNK_RELEASE_SCOPE_LOCAL 0 -#define FLB_INPUT_CHUNK_RELEASE_SCOPE_GLOBAL 1 - -struct input_chunk_raw { - struct flb_input_instance *ins; - int event_type; - size_t records; - flb_sds_t tag; - void *buf_data; - size_t buf_size; -}; - -#ifdef FLB_HAVE_IN_STORAGE_BACKLOG - -extern ssize_t sb_get_releasable_output_queue_space(struct flb_output_instance *output_plugin, - size_t required_space); - -extern int sb_release_output_queue_space(struct flb_output_instance *output_plugin, - ssize_t *required_space); - - -#else - -ssize_t sb_get_releasable_output_queue_space(struct flb_output_instance *output_plugin, - size_t required_space) -{ - return 0; -} - -int sb_release_output_queue_space(struct flb_output_instance *output_plugin, - ssize_t *required_space) -{ - return 0; -} - -#endif - -static int flb_input_chunk_safe_delete(struct flb_input_chunk *ic, - struct flb_input_chunk *old_ic, - uint64_t o_id); - -static int flb_input_chunk_is_task_safe_delete(struct flb_task *task); - -static int flb_input_chunk_drop_task_route( - struct flb_task *task, - struct flb_output_instance *o_ins); - -static ssize_t flb_input_chunk_get_real_size(struct flb_input_chunk *ic); - -static int flb_input_chunk_release_space( - struct flb_input_chunk *new_input_chunk, - struct flb_input_instance *input_plugin, - struct flb_output_instance *output_plugin, - ssize_t *required_space, - int release_scope) -{ - struct mk_list *input_chunk_iterator_tmp; - struct mk_list *input_chunk_iterator; - int chunk_destroy_flag; - struct flb_input_chunk *old_input_chunk; - ssize_t released_space; - int chunk_released; - ssize_t chunk_size; - - released_space = 0; - - mk_list_foreach_safe(input_chunk_iterator, input_chunk_iterator_tmp, - &input_plugin->chunks) { - old_input_chunk = mk_list_entry(input_chunk_iterator, - struct flb_input_chunk, _head); - - if (!flb_routes_mask_get_bit(old_input_chunk->routes_mask, - output_plugin->id)) { - continue; - } - - if (flb_input_chunk_safe_delete(new_input_chunk, - old_input_chunk, - output_plugin->id) == FLB_FALSE) { - continue; - } - - if (flb_input_chunk_drop_task_route(old_input_chunk->task, - output_plugin) == FLB_FALSE) { - continue; - } - - chunk_size = flb_input_chunk_get_real_size(old_input_chunk); - chunk_released = FLB_FALSE; - chunk_destroy_flag = FLB_FALSE; - - if (release_scope == FLB_INPUT_CHUNK_RELEASE_SCOPE_LOCAL) { - flb_routes_mask_clear_bit(old_input_chunk->routes_mask, - output_plugin->id); - - FS_CHUNK_SIZE_DEBUG_MOD(output_plugin, old_input_chunk, chunk_size); - output_plugin->fs_chunks_size -= chunk_size; - - chunk_destroy_flag = flb_routes_mask_is_empty( - old_input_chunk->routes_mask); - - chunk_released = FLB_TRUE; - } - else if (release_scope == FLB_INPUT_CHUNK_RELEASE_SCOPE_GLOBAL) { - chunk_destroy_flag = FLB_TRUE; - } - - if (chunk_destroy_flag) { - if (old_input_chunk->task != NULL) { - /* - * If the chunk is referenced by a task and task has no active route, - * we need to destroy the task as well. - */ - if (old_input_chunk->task->users == 0) { - flb_debug("[task] drop task_id %d with no active route from input plugin %s", - old_input_chunk->task->id, new_input_chunk->in->name); - flb_task_destroy(old_input_chunk->task, FLB_TRUE); - - chunk_released = FLB_TRUE; - } - } - else { - flb_debug("[input chunk] drop chunk %s with no output route from input plugin %s", - flb_input_chunk_get_name(old_input_chunk), new_input_chunk->in->name); - - flb_input_chunk_destroy(old_input_chunk, FLB_TRUE); - - chunk_released = FLB_TRUE; - } - } - - if (chunk_released) { - released_space += chunk_size; - } - - if (released_space >= *required_space) { - break; - } - } - - *required_space -= released_space; - - return 0; -} - -static void generate_chunk_name(struct flb_input_instance *in, - char *out_buf, int buf_size) -{ - struct flb_time tm; - (void) in; - - flb_time_get(&tm); - snprintf(out_buf, buf_size - 1, - "%i-%lu.%4lu.flb", - getpid(), - tm.tm.tv_sec, tm.tm.tv_nsec); -} - -ssize_t flb_input_chunk_get_size(struct flb_input_chunk *ic) -{ - return cio_chunk_get_content_size(ic->chunk); -} - -/* - * When chunk is set to DOWN from memory, data_size is set to 0 and - * cio_chunk_get_content_size(1) returns the data_size. fs_chunks_size - * is used to track the size of chunks in filesystem so we need to call - * cio_chunk_get_real_size to return the original size in the file system - */ -static ssize_t flb_input_chunk_get_real_size(struct flb_input_chunk *ic) -{ - ssize_t meta_size; - ssize_t size; - - size = cio_chunk_get_real_size(ic->chunk); - - if (size != 0) { - return size; - } - - // Real size is not synced to chunk yet - size = flb_input_chunk_get_size(ic); - if (size == 0) { - flb_debug("[input chunk] no data in the chunk %s", - flb_input_chunk_get_name(ic)); - return -1; - } - - meta_size = cio_meta_size(ic->chunk); - size += meta_size - /* See https://github.com/edsiper/chunkio#file-layout for more details */ - + 2 /* HEADER BYTES */ - + 4 /* CRC32 */ - + 16 /* PADDING */ - + 2; /* METADATA LENGTH BYTES */ - - return size; -} - -int flb_input_chunk_write(void *data, const char *buf, size_t len) -{ - int ret; - struct flb_input_chunk *ic; - - ic = (struct flb_input_chunk *) data; - - ret = cio_chunk_write(ic->chunk, buf, len); - return ret; -} - -int flb_input_chunk_write_at(void *data, off_t offset, - const char *buf, size_t len) -{ - int ret; - struct flb_input_chunk *ic; - - ic = (struct flb_input_chunk *) data; - - ret = cio_chunk_write_at(ic->chunk, offset, buf, len); - return ret; -} - -static int flb_input_chunk_drop_task_route( - struct flb_task *task, - struct flb_output_instance *output_plugin) -{ - int route_status; - int result; - - if (task == NULL) { - return FLB_TRUE; - } - - result = FLB_TRUE; - - if (task->users != 0) { - result = FLB_FALSE; - - if (output_plugin != NULL) { - flb_task_acquire_lock(task); - - route_status = flb_task_get_route_status(task, output_plugin); - - if (route_status == FLB_TASK_ROUTE_INACTIVE) { - flb_task_set_route_status(task, - output_plugin, - FLB_TASK_ROUTE_DROPPED); - - result = FLB_TRUE; - } - - flb_task_release_lock(task); - } - } - - return result; -} - - -/* - * For input_chunk referenced by an outgoing task, we need to check - * whether the chunk is in the middle of output flush callback - */ -static int flb_input_chunk_is_task_safe_delete(struct flb_task *task) -{ - if (!task) { - return FLB_TRUE; - } - - if (task->users != 0) { - return FLB_FALSE; - } - - return FLB_TRUE; -} - -static int flb_input_chunk_safe_delete(struct flb_input_chunk *ic, - struct flb_input_chunk *old_ic, - uint64_t o_id) -{ - /* The chunk we want to drop should not be the incoming chunk */ - if (ic == old_ic) { - return FLB_FALSE; - } - - /* - * Even if chunks from same input plugin have same routes_mask when created, - * the routes_mask could be modified when new chunks is ingested. Therefore, - * we still need to do the validation on the routes_mask with o_id. - */ - if (flb_routes_mask_get_bit(old_ic->routes_mask, o_id) == 0) { - return FLB_FALSE; - } - - return FLB_TRUE; -} - -int flb_input_chunk_release_space_compound( - struct flb_input_chunk *new_input_chunk, - struct flb_output_instance *output_plugin, - size_t *local_release_requirement, - int release_local_space) -{ - ssize_t required_space_remainder; - struct flb_input_instance *storage_backlog_instance; - struct flb_input_instance *input_plugin_instance; - struct mk_list *iterator; - int result; - - storage_backlog_instance = output_plugin->config->storage_input_plugin; - - *local_release_requirement = flb_input_chunk_get_real_size(new_input_chunk); - required_space_remainder = (ssize_t) *local_release_requirement; - - if (required_space_remainder > 0) { - result = flb_input_chunk_release_space(new_input_chunk, - storage_backlog_instance, - output_plugin, - &required_space_remainder, - FLB_INPUT_CHUNK_RELEASE_SCOPE_GLOBAL); - } - - if (required_space_remainder > 0) { - result = sb_release_output_queue_space(output_plugin, - &required_space_remainder); - } - - if (release_local_space) { - if (required_space_remainder > 0) { - result = flb_input_chunk_release_space(new_input_chunk, - new_input_chunk->in, - output_plugin, - &required_space_remainder, - FLB_INPUT_CHUNK_RELEASE_SCOPE_LOCAL); - } - } - - if (required_space_remainder > 0) { - mk_list_foreach(iterator, &output_plugin->config->inputs) { - input_plugin_instance = \ - mk_list_entry(iterator, struct flb_input_instance, _head); - - if (input_plugin_instance != new_input_chunk->in) { - result = flb_input_chunk_release_space( - new_input_chunk, - input_plugin_instance, - output_plugin, - &required_space_remainder, - FLB_INPUT_CHUNK_RELEASE_SCOPE_LOCAL); - } - - if (required_space_remainder <= 0) { - break; - } - } - } - - if (required_space_remainder < 0) { - required_space_remainder = 0; - } - - *local_release_requirement = (size_t) required_space_remainder; - - (void) result; - - return 0; -} - -/* - * Find a slot in the output instance to append the new data with size chunk_size, it - * will drop the the oldest chunks when the limitation on local disk is reached. - */ -int flb_input_chunk_find_space_new_data(struct flb_input_chunk *ic, - size_t chunk_size, int overlimit) -{ - int count; - int result; - struct mk_list *head; - struct flb_output_instance *o_ins; - size_t local_release_requirement; - - /* - * For each output instances that will be over the limit after adding the new chunk, - * we have to determine how many chunks needs to be removed. We will adjust the - * routes_mask to only route to the output plugin that have enough space after - * deleting some chunks fome the queue. - */ - count = 0; - - mk_list_foreach(head, &ic->in->config->outputs) { - o_ins = mk_list_entry(head, struct flb_output_instance, _head); - - if ((o_ins->total_limit_size == -1) || ((1 << o_ins->id) & overlimit) == 0 || - (flb_routes_mask_get_bit(ic->routes_mask, o_ins->id) == 0)) { - continue; - } - - local_release_requirement = 0; - - result = flb_input_chunk_release_space_compound( - ic, o_ins, - &local_release_requirement, - FLB_TRUE); - - if (result != 0 || - local_release_requirement != 0) { - count++; - } - } - - if (count != 0) { - flb_error("[input chunk] fail to drop enough chunks in order to place new data"); - exit(0); - } - - return 0; -} - -/* - * Returns a non-zero result if any output instances will reach the limit - * after buffering the new data - */ -int flb_input_chunk_has_overlimit_routes(struct flb_input_chunk *ic, - size_t chunk_size) -{ - int overlimit = 0; - struct mk_list *head; - struct flb_output_instance *o_ins; - - mk_list_foreach(head, &ic->in->config->outputs) { - o_ins = mk_list_entry(head, struct flb_output_instance, _head); - - if ((o_ins->total_limit_size == -1) || - (flb_routes_mask_get_bit(ic->routes_mask, o_ins->id) == 0)) { - continue; - } - - FS_CHUNK_SIZE_DEBUG(o_ins); - flb_debug("[input chunk] chunk %s required %ld bytes and %ld bytes left " - "in plugin %s", flb_input_chunk_get_name(ic), chunk_size, - o_ins->total_limit_size - - o_ins->fs_backlog_chunks_size - - o_ins->fs_chunks_size, - o_ins->name); - - if ((o_ins->fs_chunks_size + - o_ins->fs_backlog_chunks_size + - chunk_size) > o_ins->total_limit_size) { - overlimit |= (1 << o_ins->id); - } - } - - return overlimit; -} - -/* Find a slot for the incoming data to buffer it in local file system - * returns 0 if none of the routes can be written to - */ -int flb_input_chunk_place_new_chunk(struct flb_input_chunk *ic, size_t chunk_size) -{ - int overlimit; - overlimit = flb_input_chunk_has_overlimit_routes(ic, chunk_size); - if (overlimit != 0) { - flb_input_chunk_find_space_new_data(ic, chunk_size, overlimit); - } - - return !flb_routes_mask_is_empty(ic->routes_mask); -} - -/* Create an input chunk using a Chunk I/O */ -struct flb_input_chunk *flb_input_chunk_map(struct flb_input_instance *in, - int event_type, - void *chunk) -{ - int records = 0; - int tag_len; - int has_routes; - int ret; - uint64_t ts; - char *buf_data; - size_t buf_size; - size_t offset; - ssize_t bytes; - const char *tag_buf; - struct flb_input_chunk *ic; - - /* Create context for the input instance */ - ic = flb_calloc(1, sizeof(struct flb_input_chunk)); - if (!ic) { - flb_errno(); - return NULL; - } - ic->event_type = event_type; - ic->busy = FLB_FALSE; - ic->fs_counted = FLB_FALSE; - ic->fs_backlog = FLB_TRUE; - ic->chunk = chunk; - ic->in = in; - msgpack_packer_init(&ic->mp_pck, ic, flb_input_chunk_write); - - ret = cio_chunk_get_content(ic->chunk, &buf_data, &buf_size); - if (ret != CIO_OK) { - flb_error("[input chunk] error retrieving content for metrics"); - flb_free(ic); - return NULL; - } - - if (ic->event_type == FLB_INPUT_LOGS) { - /* Validate records in the chunk */ - ret = flb_mp_validate_log_chunk(buf_data, buf_size, &records, &offset); - if (ret == -1) { - /* If there are valid records, truncate the chunk size */ - if (records <= 0) { - flb_plg_error(in, - "chunk validation failed, data might be corrupted. " - "No valid records found, the chunk will be discarded."); - flb_free(ic); - return NULL; - } - if (records > 0 && offset > 32) { - flb_plg_warn(in, - "chunk validation failed, data might be corrupted. " - "Found %d valid records, failed content starts " - "right after byte %lu. Recovering valid records.", - records, offset); - - /* truncate the chunk to recover valid records */ - cio_chunk_write_at(chunk, offset, NULL, 0); - } - else { - flb_plg_error(in, - "chunk validation failed, data might be corrupted. " - "Found %d valid records, failed content starts " - "right after byte %lu. Cannot recover chunk,", - records, offset); - flb_free(ic); - return NULL; - } - } - } - else if (ic->event_type == FLB_INPUT_METRICS) { - ret = flb_mp_validate_metric_chunk(buf_data, buf_size, &records, &offset); - if (ret == -1) { - if (records <= 0) { - flb_plg_error(in, - "metrics chunk validation failed, data might be corrupted. " - "No valid records found, the chunk will be discarded."); - flb_free(ic); - return NULL; - } - if (records > 0 && offset > 32) { - flb_plg_warn(in, - "metrics chunk validation failed, data might be corrupted. " - "Found %d valid records, failed content starts " - "right after byte %lu. Recovering valid records.", - records, offset); - - /* truncate the chunk to recover valid records */ - cio_chunk_write_at(chunk, offset, NULL, 0); - } - else { - flb_plg_error(in, - "metrics chunk validation failed, data might be corrupted. " - "Found %d valid records, failed content starts " - "right after byte %lu. Cannot recover chunk,", - records, offset); - flb_free(ic); - return NULL; - } - - } - } - else if (ic->event_type == FLB_INPUT_TRACES) { - - } - - /* Skip chunks without content data */ - if (records == 0) { - flb_plg_error(in, - "chunk validation failed, data might be corrupted. " - "No valid records found, the chunk will be discarded."); - flb_free(ic); - return NULL; - } - - /* - * If the content is valid and the chunk has extra padding zeros, just - * perform an adjustment. - */ - bytes = cio_chunk_get_content_size(chunk); - if (bytes == -1) { - flb_free(ic); - return NULL; - } - if (offset < bytes) { - cio_chunk_write_at(chunk, offset, NULL, 0); - } - - /* Update metrics */ -#ifdef FLB_HAVE_METRICS - ic->total_records = records; - if (ic->total_records > 0) { - /* timestamp */ - ts = cfl_time_now(); - - /* fluentbit_input_records_total */ - cmt_counter_add(in->cmt_records, ts, ic->total_records, - 1, (char *[]) {(char *) flb_input_name(in)}); - - /* fluentbit_input_bytes_total */ - cmt_counter_add(in->cmt_bytes, ts, buf_size, - 1, (char *[]) {(char *) flb_input_name(in)}); - - /* OLD metrics */ - flb_metrics_sum(FLB_METRIC_N_RECORDS, ic->total_records, in->metrics); - flb_metrics_sum(FLB_METRIC_N_BYTES, buf_size, in->metrics); - } -#endif - - /* Get the the tag reference (chunk metadata) */ - ret = flb_input_chunk_get_tag(ic, &tag_buf, &tag_len); - if (ret == -1) { - flb_error("[input chunk] error retrieving tag of input chunk"); - flb_free(ic); - return NULL; - } - - bytes = flb_input_chunk_get_real_size(ic); - if (bytes < 0) { - flb_warn("[input chunk] could not retrieve chunk real size"); - flb_free(ic); - return NULL; - } - - has_routes = flb_routes_mask_set_by_tag(ic->routes_mask, tag_buf, tag_len, in); - if (has_routes == 0) { - flb_warn("[input chunk] no matching route for backoff log chunk %s", - flb_input_chunk_get_name(ic)); - } - - mk_list_add(&ic->_head, &in->chunks); - - flb_input_chunk_update_output_instances(ic, bytes); - - return ic; -} - -static int input_chunk_write_header(struct cio_chunk *chunk, int event_type, - char *tag, int tag_len) - -{ - int ret; - int meta_size; - char *meta; - - /* - * Prepare the Chunk metadata header - * ---------------------------------- - * m[0] = FLB_INPUT_CHUNK_MAGIC_BYTE_0 - * m[1] = FLB_INPUT_CHUNK_MAGIC_BYTE_1 - * m[2] = type (FLB_INPUT_CHUNK_TYPE_LOG or FLB_INPUT_CHUNK_TYPE_METRIC or FLB_INPUT_CHUNK_TYPE_TRACE - * m[3] = 0 (unused for now) - */ - - /* write metadata (tag) */ - if (tag_len > (65535 - FLB_INPUT_CHUNK_META_HEADER)) { - /* truncate length */ - tag_len = 65535 - FLB_INPUT_CHUNK_META_HEADER; - } - meta_size = FLB_INPUT_CHUNK_META_HEADER + tag_len; - - /* Allocate buffer for metadata header */ - meta = flb_calloc(1, meta_size); - if (!meta) { - flb_errno(); - return -1; - } - - /* - * Write chunk header in a temporary buffer - * ---------------------------------------- - */ - - /* magic bytes */ - meta[0] = FLB_INPUT_CHUNK_MAGIC_BYTE_0; - meta[1] = FLB_INPUT_CHUNK_MAGIC_BYTE_1; - - /* event type */ - if (event_type == FLB_INPUT_LOGS) { - meta[2] = FLB_INPUT_CHUNK_TYPE_LOGS; - } - else if (event_type == FLB_INPUT_METRICS) { - meta[2] = FLB_INPUT_CHUNK_TYPE_METRICS; - } - else if (event_type == FLB_INPUT_TRACES) { - meta[2] = FLB_INPUT_CHUNK_TYPE_TRACES; - } - - /* unused byte */ - meta[3] = 0; - - /* copy the tag after magic bytes */ - memcpy(meta + FLB_INPUT_CHUNK_META_HEADER, tag, tag_len); - - /* Write tag into metadata section */ - ret = cio_meta_write(chunk, (char *) meta, meta_size); - if (ret == -1) { - flb_error("[input chunk] could not write metadata"); - flb_free(meta); - return -1; - } - flb_free(meta); - - return 0; -} - -struct flb_input_chunk *flb_input_chunk_create(struct flb_input_instance *in, int event_type, - const char *tag, int tag_len) -{ - int ret; - int err; - int set_down = FLB_FALSE; - int has_routes; - char name[64]; - struct cio_chunk *chunk; - struct flb_storage_input *storage; - struct flb_input_chunk *ic; - - storage = in->storage; - - /* chunk name */ - generate_chunk_name(in, name, sizeof(name) - 1); - - /* open/create target chunk file */ - chunk = cio_chunk_open(storage->cio, storage->stream, name, - CIO_OPEN, FLB_INPUT_CHUNK_SIZE, &err); - if (!chunk) { - flb_error("[input chunk] could not create chunk file: %s:%s", - storage->stream->name, name); - return NULL; - } - /* - * If the returned chunk at open is 'down', just put it up, write the - * content and set it down again. - */ - ret = cio_chunk_is_up(chunk); - if (ret == CIO_FALSE) { - ret = cio_chunk_up_force(chunk); - if (ret == -1) { - cio_chunk_close(chunk, CIO_TRUE); - return NULL; - } - set_down = FLB_TRUE; - } - - /* Write chunk header */ - ret = input_chunk_write_header(chunk, event_type, (char *) tag, tag_len); - if (ret == -1) { - cio_chunk_close(chunk, CIO_TRUE); - return NULL; - } - - /* Create context for the input instance */ - ic = flb_calloc(1, sizeof(struct flb_input_chunk)); - if (!ic) { - flb_errno(); - cio_chunk_close(chunk, CIO_TRUE); - return NULL; - } - - /* - * Check chunk content type to be created: depending of the value set by - * the input plugin, this can be FLB_INPUT_LOGS, FLB_INPUT_METRICS or - * FLB_INPUT_TRACES. - */ - ic->event_type = event_type; - ic->busy = FLB_FALSE; - ic->fs_counted = FLB_FALSE; - ic->chunk = chunk; - ic->fs_backlog = FLB_FALSE; - ic->in = in; - ic->stream_off = 0; - ic->task = NULL; -#ifdef FLB_HAVE_METRICS - ic->total_records = 0; -#endif - - /* Calculate the routes_mask for the input chunk */ - has_routes = flb_routes_mask_set_by_tag(ic->routes_mask, tag, tag_len, in); - if (has_routes == 0) { - flb_trace("[input chunk] no matching route for input chunk '%s' with tag '%s'", - flb_input_chunk_get_name(ic), tag); - } - - msgpack_packer_init(&ic->mp_pck, ic, flb_input_chunk_write); - mk_list_add(&ic->_head, &in->chunks); - - if (set_down == FLB_TRUE) { - cio_chunk_down(chunk); - } - - if (event_type == FLB_INPUT_LOGS) { - flb_hash_table_add(in->ht_log_chunks, tag, tag_len, ic, 0); - } - else if (event_type == FLB_INPUT_METRICS) { - flb_hash_table_add(in->ht_metric_chunks, tag, tag_len, ic, 0); - } - else if (event_type == FLB_INPUT_TRACES) { - flb_hash_table_add(in->ht_trace_chunks, tag, tag_len, ic, 0); - } - - return ic; -} - -int flb_input_chunk_destroy_corrupted(struct flb_input_chunk *ic, - const char *tag_buf, int tag_len, - int del) -{ - ssize_t bytes; - struct mk_list *head; - struct flb_output_instance *o_ins; - - mk_list_foreach(head, &ic->in->config->outputs) { - o_ins = mk_list_entry(head, struct flb_output_instance, _head); - - if (o_ins->total_limit_size == -1) { - continue; - } - - bytes = flb_input_chunk_get_real_size(ic); - if (bytes == -1) { - // no data in the chunk - continue; - } - - if (flb_routes_mask_get_bit(ic->routes_mask, o_ins->id) != 0) { - if (ic->fs_counted == FLB_TRUE) { - FS_CHUNK_SIZE_DEBUG_MOD(o_ins, ic, -bytes); - o_ins->fs_chunks_size -= bytes; - flb_debug("[input chunk] remove chunk %s with %ld bytes from plugin %s, " - "the updated fs_chunks_size is %ld bytes", flb_input_chunk_get_name(ic), - bytes, o_ins->name, o_ins->fs_chunks_size); - } - } - } - - if (del == CIO_TRUE && tag_buf) { - /* - * "TRY" to delete any reference to this chunk ('ic') from the hash - * table. Note that maybe the value is not longer available in the - * entries if it was replaced: note that we always keep the last - * chunk for a specific Tag. - */ - if (ic->event_type == FLB_INPUT_LOGS) { - flb_hash_table_del_ptr(ic->in->ht_log_chunks, - tag_buf, tag_len, (void *) ic); - } - else if (ic->event_type == FLB_INPUT_METRICS) { - flb_hash_table_del_ptr(ic->in->ht_metric_chunks, - tag_buf, tag_len, (void *) ic); - } - else if (ic->event_type == FLB_INPUT_TRACES) { - flb_hash_table_del_ptr(ic->in->ht_trace_chunks, - tag_buf, tag_len, (void *) ic); - } - } - -#ifdef FLB_HAVE_CHUNK_TRACE - if (ic->trace != NULL) { - flb_chunk_trace_destroy(ic->trace); - } -#endif /* FLB_HAVE_CHUNK_TRACE */ - - cio_chunk_close(ic->chunk, del); - mk_list_del(&ic->_head); - flb_free(ic); - - return 0; -} - - -int flb_input_chunk_destroy(struct flb_input_chunk *ic, int del) -{ - int tag_len; - int ret; - ssize_t bytes; - const char *tag_buf = NULL; - struct mk_list *head; - struct flb_output_instance *o_ins; - - if (flb_input_chunk_is_up(ic) == FLB_FALSE) { - flb_input_chunk_set_up(ic); - } - - mk_list_foreach(head, &ic->in->config->outputs) { - o_ins = mk_list_entry(head, struct flb_output_instance, _head); - - if (o_ins->total_limit_size == -1) { - continue; - } - - bytes = flb_input_chunk_get_real_size(ic); - if (bytes == -1) { - // no data in the chunk - continue; - } - - if (flb_routes_mask_get_bit(ic->routes_mask, o_ins->id) != 0) { - if (ic->fs_counted == FLB_TRUE) { - FS_CHUNK_SIZE_DEBUG_MOD(o_ins, ic, -bytes); - o_ins->fs_chunks_size -= bytes; - flb_debug("[input chunk] remove chunk %s with %ld bytes from plugin %s, " - "the updated fs_chunks_size is %ld bytes", flb_input_chunk_get_name(ic), - bytes, o_ins->name, o_ins->fs_chunks_size); - } - } - } - - /* - * When a chunk is going to be destroyed, this can be in a down state, - * since the next step is to retrieve the Tag we need to have the - * content up. - */ - ret = flb_input_chunk_is_up(ic); - if (ret == FLB_FALSE) { - ret = cio_chunk_up_force(ic->chunk); - if (ret == -1) { - flb_error("[input chunk] cannot load chunk: %s", - flb_input_chunk_get_name(ic)); - } - } - - /* Retrieve Tag */ - ret = flb_input_chunk_get_tag(ic, &tag_buf, &tag_len); - if (ret == -1) { - flb_trace("[input chunk] could not retrieve chunk tag: %s", - flb_input_chunk_get_name(ic)); - } - - if (del == CIO_TRUE && tag_buf) { - /* - * "TRY" to delete any reference to this chunk ('ic') from the hash - * table. Note that maybe the value is not longer available in the - * entries if it was replaced: note that we always keep the last - * chunk for a specific Tag. - */ - if (ic->event_type == FLB_INPUT_LOGS) { - flb_hash_table_del_ptr(ic->in->ht_log_chunks, - tag_buf, tag_len, (void *) ic); - } - else if (ic->event_type == FLB_INPUT_METRICS) { - flb_hash_table_del_ptr(ic->in->ht_metric_chunks, - tag_buf, tag_len, (void *) ic); - } - else if (ic->event_type == FLB_INPUT_TRACES) { - flb_hash_table_del_ptr(ic->in->ht_trace_chunks, - tag_buf, tag_len, (void *) ic); - } - } - -#ifdef FLB_HAVE_CHUNK_TRACE - if (ic->trace != NULL) { - flb_chunk_trace_destroy(ic->trace); - } -#endif /* FLB_HAVE_CHUNK_TRACE */ - - cio_chunk_close(ic->chunk, del); - mk_list_del(&ic->_head); - flb_free(ic); - - return 0; -} - -/* Return or create an available chunk to write data */ -static struct flb_input_chunk *input_chunk_get(struct flb_input_instance *in, - int event_type, - const char *tag, int tag_len, - size_t chunk_size, int *set_down) -{ - int id = -1; - int ret; - int new_chunk = FLB_FALSE; - size_t out_size; - struct flb_input_chunk *ic = NULL; - - if (tag_len > FLB_INPUT_CHUNK_TAG_MAX) { - flb_plg_warn(in, - "Tag set exceeds limit, truncating from %i to %i bytes", - tag_len, FLB_INPUT_CHUNK_TAG_MAX); - tag_len = FLB_INPUT_CHUNK_TAG_MAX; - } - - if (event_type == FLB_INPUT_LOGS) { - id = flb_hash_table_get(in->ht_log_chunks, tag, tag_len, - (void *) &ic, &out_size); - } - else if (event_type == FLB_INPUT_METRICS) { - id = flb_hash_table_get(in->ht_metric_chunks, tag, tag_len, - (void *) &ic, &out_size); - } - else if (event_type == FLB_INPUT_TRACES) { - id = flb_hash_table_get(in->ht_trace_chunks, tag, tag_len, - (void *) &ic, &out_size); - } - - if (id >= 0) { - if (ic->busy == FLB_TRUE || cio_chunk_is_locked(ic->chunk)) { - ic = NULL; - } - else if (cio_chunk_is_up(ic->chunk) == CIO_FALSE) { - ret = cio_chunk_up_force(ic->chunk); - - if (ret == CIO_CORRUPTED) { - if (in->config->storage_del_bad_chunks) { - /* If the chunk is corrupted we need to discard it and - * set ic to NULL so the system tries to allocate a new - * chunk. - */ - - flb_error("[input chunk] discarding corrupted chunk"); - } - - flb_input_chunk_destroy_corrupted(ic, - tag, tag_len, - in->config->storage_del_bad_chunks); - - ic = NULL; - } - else if (ret != CIO_OK) { - ic = NULL; - } - - *set_down = FLB_TRUE; - } - } - - /* No chunk was found, we need to create a new one */ - if (!ic) { - ic = flb_input_chunk_create(in, event_type, (char *) tag, tag_len); - new_chunk = FLB_TRUE; - if (!ic) { - return NULL; - } - ic->event_type = event_type; - } - - /* - * If buffering this block of data will exceed one of the limit among all output instances - * that the chunk will flush to, we need to modify the routes_mask of the oldest chunks - * (based in creation time) to get enough space for the incoming chunk. - */ - if (!flb_routes_mask_is_empty(ic->routes_mask) - && flb_input_chunk_place_new_chunk(ic, chunk_size) == 0) { - /* - * If the chunk is not newly created, the chunk might already have logs inside. - * We cannot delete (reused) chunks here. - * If the routes_mask is cleared after trying to append new data, we destroy - * the chunk. - */ - if (new_chunk || flb_routes_mask_is_empty(ic->routes_mask) == FLB_TRUE) { - flb_input_chunk_destroy(ic, FLB_TRUE); - } - return NULL; - } - - return ic; -} - -static inline int flb_input_chunk_is_mem_overlimit(struct flb_input_instance *i) -{ - if (i->mem_buf_limit <= 0) { - return FLB_FALSE; - } - - if (i->mem_chunks_size >= i->mem_buf_limit) { - return FLB_TRUE; - } - - return FLB_FALSE; -} - -static inline int flb_input_chunk_is_storage_overlimit(struct flb_input_instance *i) -{ - struct flb_storage_input *storage = (struct flb_storage_input *)i->storage; - - if (storage->type == FLB_STORAGE_FS) { - if (i->storage_pause_on_chunks_overlimit == FLB_TRUE) { - if (storage->cio->total_chunks_up >= storage->cio->max_chunks_up) { - return FLB_TRUE; - } - } - } - - return FLB_FALSE; -} - -/* - * Check all chunks associated to the input instance and summarize - * the number of bytes in use. - */ -size_t flb_input_chunk_total_size(struct flb_input_instance *in) -{ - size_t total = 0; - struct flb_storage_input *storage; - - storage = (struct flb_storage_input *) in->storage; - total = cio_stream_size_chunks_up(storage->stream); - return total; -} - -/* - * Count and update the number of bytes being used by the instance. Also - * check if the instance is paused, if so, check if it can be resumed if - * is not longer over the limits. - * - * It always returns the number of bytes in use. - */ -size_t flb_input_chunk_set_limits(struct flb_input_instance *in) -{ - size_t total; - - /* Gather total number of enqueued bytes */ - total = flb_input_chunk_total_size(in); - - /* Register the total into the context variable */ - in->mem_chunks_size = total; - - /* - * After the adjustments, validate if the plugin is overlimit or paused - * and perform further adjustments. - */ - if (flb_input_chunk_is_mem_overlimit(in) == FLB_FALSE && - in->config->is_running == FLB_TRUE && - in->config->is_ingestion_active == FLB_TRUE && - in->mem_buf_status == FLB_INPUT_PAUSED) { - in->mem_buf_status = FLB_INPUT_RUNNING; - if (in->p->cb_resume) { - flb_input_resume(in); - flb_info("[input] %s resume (mem buf overlimit)", - in->name); - } - } - if (flb_input_chunk_is_storage_overlimit(in) == FLB_FALSE && - in->config->is_running == FLB_TRUE && - in->config->is_ingestion_active == FLB_TRUE && - in->storage_buf_status == FLB_INPUT_PAUSED) { - in->storage_buf_status = FLB_INPUT_RUNNING; - if (in->p->cb_resume) { - flb_input_resume(in); - flb_info("[input] %s resume (storage buf overlimit %zu/%zu)", - in->name, - ((struct flb_storage_input *)in->storage)->cio->total_chunks_up, - ((struct flb_storage_input *)in->storage)->cio->max_chunks_up); - } - } - - return total; -} - -/* - * If the number of bytes in use by the chunks are over the imposed limit - * by configuration, pause the instance. - */ -static inline int flb_input_chunk_protect(struct flb_input_instance *i) -{ - struct flb_storage_input *storage = i->storage; - - if (flb_input_chunk_is_storage_overlimit(i) == FLB_TRUE) { - flb_warn("[input] %s paused (storage buf overlimit %zu/%zu)", - i->name, - storage->cio->total_chunks_up, - storage->cio->max_chunks_up); - flb_input_pause(i); - i->storage_buf_status = FLB_INPUT_PAUSED; - return FLB_TRUE; - } - - if (storage->type == FLB_STORAGE_FS) { - return FLB_FALSE; - } - - if (flb_input_chunk_is_mem_overlimit(i) == FLB_TRUE) { - /* - * if the plugin is already overlimit and the strategy is based on - * a memory-ring-buffer logic, do not pause the plugin, upon next - * try of ingestion 'memrb' will make sure to release some bytes. - */ - if (i->storage_type == FLB_STORAGE_MEMRB) { - return FLB_FALSE; - } - - /* - * The plugin is using 'memory' buffering only and already reached - * it limit, just pause the ingestion. - */ - flb_warn("[input] %s paused (mem buf overlimit)", - i->name); - flb_input_pause(i); - i->mem_buf_status = FLB_INPUT_PAUSED; - return FLB_TRUE; - } - - return FLB_FALSE; -} - -/* - * Validate if the chunk coming from the input plugin based on config and - * resources usage must be 'up' or 'down' (applicable for filesystem storage - * type). - * - * FIXME: can we find a better name for this function ? - */ -int flb_input_chunk_set_up_down(struct flb_input_chunk *ic) -{ - size_t total; - struct flb_input_instance *in; - - in = ic->in; - - /* Gather total number of enqueued bytes */ - total = flb_input_chunk_total_size(in); - - /* Register the total into the context variable */ - in->mem_chunks_size = total; - - if (flb_input_chunk_is_mem_overlimit(in) == FLB_TRUE) { - if (cio_chunk_is_up(ic->chunk) == CIO_TRUE) { - cio_chunk_down(ic->chunk); - - /* Adjust new counters */ - total = flb_input_chunk_total_size(ic->in); - in->mem_chunks_size = total; - - return FLB_FALSE; - } - } - - return FLB_TRUE; -} - -int flb_input_chunk_is_up(struct flb_input_chunk *ic) -{ - return cio_chunk_is_up(ic->chunk); -} - -int flb_input_chunk_down(struct flb_input_chunk *ic) -{ - if (cio_chunk_is_up(ic->chunk) == CIO_TRUE) { - return cio_chunk_down(ic->chunk); - } - - return 0; -} - -int flb_input_chunk_set_up(struct flb_input_chunk *ic) -{ - if (cio_chunk_is_up(ic->chunk) == CIO_FALSE) { - return cio_chunk_up(ic->chunk); - } - - return 0; -} - -static int memrb_input_chunk_release_space(struct flb_input_instance *ins, - size_t required_space, - size_t *dropped_chunks, size_t *dropped_bytes) -{ - int ret; - int released; - size_t removed_chunks = 0; - ssize_t chunk_size; - ssize_t released_space = 0; - struct mk_list *tmp; - struct mk_list *head; - struct flb_input_chunk *ic; - - mk_list_foreach_safe(head, tmp, &ins->chunks) { - ic = mk_list_entry(head, struct flb_input_chunk, _head); - - /* check if is there any task or no users associated */ - ret = flb_input_chunk_is_task_safe_delete(ic->task); - if (ret == FLB_FALSE) { - continue; - } - - /* get chunk size */ - chunk_size = flb_input_chunk_get_real_size(ic); - - released = FLB_FALSE; - if (ic->task != NULL) { - if (ic->task->users == 0) { - flb_task_destroy(ic->task, FLB_TRUE); - released = FLB_TRUE; - } - } - else { - flb_input_chunk_destroy(ic, FLB_TRUE); - released = FLB_TRUE; - } - - if (released) { - released_space += chunk_size; - removed_chunks++; - } - - if (released_space >= required_space) { - break; - } - } - - /* no matter if we succeeded or not, set the counters */ - *dropped_bytes = released_space; - *dropped_chunks = removed_chunks; - - /* set the final status of the operation */ - if (released_space >= required_space) { - return 0; - } - - return -1; -} - -/* Append a RAW MessagPack buffer to the input instance */ -static int input_chunk_append_raw(struct flb_input_instance *in, - int event_type, - size_t n_records, - const char *tag, size_t tag_len, - const void *buf, size_t buf_size) -{ - int ret; - int set_down = FLB_FALSE; - int min; - int new_chunk = FLB_FALSE; - uint64_t ts; - char *name; - size_t dropped_chunks; - size_t dropped_bytes; - size_t content_size; - size_t real_diff; - size_t real_size; - size_t pre_real_size; - struct flb_input_chunk *ic; - struct flb_storage_input *si; - - /* memory ring-buffer checker */ - if (in->storage_type == FLB_STORAGE_MEMRB) { - /* check if we are overlimit */ - ret = flb_input_chunk_is_mem_overlimit(in); - if (ret) { - /* reset counters */ - dropped_chunks = 0; - dropped_bytes = 0; - - /* try to release 'buf_size' */ - ret = memrb_input_chunk_release_space(in, buf_size, - &dropped_chunks, &dropped_bytes); - - /* update metrics if required */ - if (dropped_chunks > 0 || dropped_bytes > 0) { - /* timestamp and input plugin name for label */ - ts = cfl_time_now(); - name = (char *) flb_input_name(in); - - /* update counters */ - cmt_counter_add(in->cmt_memrb_dropped_chunks, ts, - dropped_chunks, 1, (char *[]) {name}); - - cmt_counter_add(in->cmt_memrb_dropped_bytes, ts, - dropped_bytes, 1, (char *[]) {name}); - } - - if (ret != 0) { - /* we could not allocate the required space, just return */ - return -1; - } - } - } - - /* Check if the input plugin has been paused */ - if (flb_input_buf_paused(in) == FLB_TRUE) { - flb_debug("[input chunk] %s is paused, cannot append records", - in->name); - return -1; - } - - if (buf_size == 0) { - flb_debug("[input chunk] skip ingesting data with 0 bytes"); - return -1; - } - - /* - * Some callers might not set a custom tag, on that case just inherit - * the fixed instance tag or instance name. - */ - if (!tag) { - if (in->tag && in->tag_len > 0) { - tag = in->tag; - tag_len = in->tag_len; - } - else { - tag = in->name; - tag_len = strlen(in->name); - } - } - - /* - * Get a target input chunk, can be one with remaining space available - * or a new one. - */ - ic = input_chunk_get(in, event_type, tag, tag_len, buf_size, &set_down); - if (!ic) { - flb_error("[input chunk] no available chunk"); - return -1; - } - - /* newly created chunk */ - if (flb_input_chunk_get_size(ic) == 0) { - new_chunk = FLB_TRUE; - } - - /* We got the chunk, validate if is 'up' or 'down' */ - ret = flb_input_chunk_is_up(ic); - if (ret == FLB_FALSE) { - ret = cio_chunk_up_force(ic->chunk); - if (ret == -1) { - flb_error("[input chunk] cannot retrieve temporary chunk"); - return -1; - } - set_down = FLB_TRUE; - } - - /* - * Keep the previous real size to calculate the real size - * difference for flb_input_chunk_update_output_instances(), - * use 0 when the chunk is new since it's size will never - * have been calculated before. - */ - if (new_chunk == FLB_TRUE) { - pre_real_size = 0; - } - else { - pre_real_size = flb_input_chunk_get_real_size(ic); - } - - /* Write the new data */ - ret = flb_input_chunk_write(ic, buf, buf_size); - if (ret == -1) { - flb_error("[input chunk] error writing data from %s instance", - in->name); - cio_chunk_tx_rollback(ic->chunk); - return -1; - } - -#ifdef FLB_HAVE_CHUNK_TRACE - flb_chunk_trace_do_input(ic); -#endif /* FLB_HAVE_CHUNK_TRACE */ - - /* Update 'input' metrics */ -#ifdef FLB_HAVE_METRICS - if (ret == CIO_OK) { - ic->added_records = n_records; - ic->total_records += n_records; - } - - if (ic->total_records > 0) { - /* timestamp */ - ts = cfl_time_now(); - - /* fluentbit_input_records_total */ - cmt_counter_add(in->cmt_records, ts, ic->added_records, - 1, (char *[]) {(char *) flb_input_name(in)}); - - /* fluentbit_input_bytes_total */ - cmt_counter_add(in->cmt_bytes, ts, buf_size, - 1, (char *[]) {(char *) flb_input_name(in)}); - - /* OLD api */ - flb_metrics_sum(FLB_METRIC_N_RECORDS, ic->added_records, in->metrics); - flb_metrics_sum(FLB_METRIC_N_BYTES, buf_size, in->metrics); - } -#endif - - /* Apply filters */ - if (event_type == FLB_INPUT_LOGS) { - flb_filter_do(ic, - buf, buf_size, - tag, tag_len, in->config); - } - - /* get the chunks content size */ - content_size = cio_chunk_get_content_size(ic->chunk); - - /* - * There is a case that rewrite_tag will modify the tag and keep rule is set - * to drop the original record. The original record will still go through the - * flb_input_chunk_update_output_instances(2) to update the fs_chunks_size by - * metadata bytes (consisted by metadata bytes of the file chunk). This condition - * sets the diff to 0 in order to not update the fs_chunks_size. - */ - if (flb_input_chunk_get_size(ic) == 0) { - real_diff = 0; - } - - /* Lock buffers where size > 2MB */ - if (content_size > FLB_INPUT_CHUNK_FS_MAX_SIZE) { - cio_chunk_lock(ic->chunk); - } - - /* Make sure the data was not filtered out and the buffer size is zero */ - if (content_size == 0) { - flb_input_chunk_destroy(ic, FLB_TRUE); - flb_input_chunk_set_limits(in); - return 0; - } -#ifdef FLB_HAVE_STREAM_PROCESSOR - else if (in->config->stream_processor_ctx && - ic->event_type == FLB_INPUT_LOGS) { - char *c_data; - size_t c_size; - - /* Retrieve chunk (filtered) output content */ - cio_chunk_get_content(ic->chunk, &c_data, &c_size); - - /* Invoke stream processor */ - flb_sp_do(in->config->stream_processor_ctx, - in, - tag, tag_len, - c_data + ic->stream_off, c_size - ic->stream_off); - ic->stream_off += (c_size - ic->stream_off); - } -#endif - - if (set_down == FLB_TRUE) { - cio_chunk_down(ic->chunk); - } - - /* - * If the instance is not routable, there is no need to keep the - * content in the storage engine, just get rid of it. - */ - if (in->routable == FLB_FALSE) { - flb_input_chunk_destroy(ic, FLB_TRUE); - return 0; - } - - /* Update memory counters and adjust limits if any */ - flb_input_chunk_set_limits(in); - - /* - * Check if we are overlimit and validate if is there any filesystem - * storage type asociated to this input instance, if so, unload the - * chunk content from memory to respect imposed limits. - * - * Calling cio_chunk_down() the memory map associated and the file - * descriptor will be released. At any later time, it must be bring up - * for I/O operations. - */ - si = (struct flb_storage_input *) in->storage; - if (flb_input_chunk_is_mem_overlimit(in) == FLB_TRUE && - si->type == FLB_STORAGE_FS) { - if (cio_chunk_is_up(ic->chunk) == CIO_TRUE) { - /* - * If we are already over limit, a sub-sequent data ingestion - * might need a Chunk to write data in. As an optimization we - * will put this Chunk down ONLY IF it has less than 1% of - * it capacity as available space, otherwise keep it 'up' so - * it available space can be used. - */ - content_size = cio_chunk_get_content_size(ic->chunk); - - /* Do we have less than 1% available ? */ - min = (FLB_INPUT_CHUNK_FS_MAX_SIZE * 0.01); - if (FLB_INPUT_CHUNK_FS_MAX_SIZE - content_size < min) { - cio_chunk_down(ic->chunk); - } - } - } - - real_size = flb_input_chunk_get_real_size(ic); - real_diff = real_size - pre_real_size; - if (real_diff != 0) { - flb_debug("[input chunk] update output instances with new chunk size diff=%zd, records=%zu, input=%s", - real_diff, n_records, flb_input_name(in)); - flb_input_chunk_update_output_instances(ic, real_diff); - } - -#ifdef FLB_HAVE_CHUNK_TRACE - if (ic->trace) { - flb_chunk_trace_pre_output(ic->trace); - } -#endif /* FLB_HAVE_CHUNK_TRACE */ - - flb_input_chunk_protect(in); - return 0; -} - -static void destroy_chunk_raw(struct input_chunk_raw *cr) -{ - if (cr->buf_data) { - flb_free(cr->buf_data); - } - - if (cr->tag) { - flb_sds_destroy(cr->tag); - } - - flb_free(cr); -} - -static int append_to_ring_buffer(struct flb_input_instance *ins, - int event_type, - size_t records, - const char *tag, - size_t tag_len, - const void *buf, - size_t buf_size) - -{ - int ret; - int retries = 0; - int retry_limit = 10; - struct input_chunk_raw *cr; - - cr = flb_calloc(1, sizeof(struct input_chunk_raw)); - if (!cr) { - flb_errno(); - return -1; - } - cr->ins = ins; - cr->event_type = event_type; - - if (tag && tag_len > 0) { - cr->tag = flb_sds_create_len(tag, tag_len); - if (!cr->tag) { - flb_free(cr); - return -1; - } - } - else { - cr->tag = NULL; - } - - cr->records = records; - cr->buf_data = flb_malloc(buf_size); - if (!cr->buf_data) { - flb_errno(); - destroy_chunk_raw(cr); - return -1; - } - - /* - * this memory copy is just a simple overhead, the problem we have is that - * input instances always assume that they have to release their buffer since - * the append raw operation already did a copy. Not a big issue but maybe this - * is a tradeoff... - */ - memcpy(cr->buf_data, buf, buf_size); - cr->buf_size = buf_size; - - - -retry: - /* - * There is a little chance that the ring buffer is full or due to saturation - * from the main thread the data is not being consumed. On this scenario we - * retry up to 'retry_limit' times with a little wait time. - */ - if (retries >= retry_limit) { - flb_plg_error(ins, "could not enqueue records into the ring buffer"); - destroy_chunk_raw(cr); - return -1; - } - - /* append chunk raw context to the ring buffer */ - ret = flb_ring_buffer_write(ins->rb, (void *) &cr, sizeof(cr)); - if (ret == -1) { - flb_plg_debug(ins, "failed buffer write, retries=%i\n", - retries); - - /* sleep for 100000 microseconds (100 milliseconds) */ - usleep(100000); - retries++; - goto retry; - } - - return 0; -} - -/* iterate input instance ring buffer and remove any enqueued input_chunk_raw */ -void flb_input_chunk_ring_buffer_cleanup(struct flb_input_instance *ins) -{ - int ret; - struct input_chunk_raw *cr; - - if (!ins->rb) { - return; - } - - while ((ret = flb_ring_buffer_read(ins->rb, (void *) &cr, sizeof(cr))) == 0) { - if (cr) { - destroy_chunk_raw(cr); - cr = NULL; - } - } -} - -void flb_input_chunk_ring_buffer_collector(struct flb_config *ctx, void *data) -{ - int ret; - int tag_len = 0; - struct mk_list *head; - struct flb_input_instance *ins; - struct input_chunk_raw *cr; - - mk_list_foreach(head, &ctx->inputs) { - ins = mk_list_entry(head, struct flb_input_instance, _head); - cr = NULL; - - while (1) { - if (flb_input_buf_paused(ins) == FLB_TRUE) { - break; - } - - ret = flb_ring_buffer_read(ins->rb, - (void *) &cr, - sizeof(cr)); - if (ret != 0) { - break; - } - - if (cr) { - if (cr->tag) { - tag_len = flb_sds_len(cr->tag); - } - else { - tag_len = 0; - } - - input_chunk_append_raw(cr->ins, cr->event_type, cr->records, - cr->tag, tag_len, - cr->buf_data, cr->buf_size); - destroy_chunk_raw(cr); - } - cr = NULL; - } - - ins->rb->flush_pending = FLB_FALSE; - } -} - -int flb_input_chunk_append_raw(struct flb_input_instance *in, - int event_type, - size_t records, - const char *tag, size_t tag_len, - const void *buf, size_t buf_size) -{ - int ret; - - /* - * If the plugin instance registering the data runs in a separate thread, we must - * add the data reference to the ring buffer. - */ - if (flb_input_is_threaded(in)) { - ret = append_to_ring_buffer(in, event_type, records, - tag, tag_len, - buf, buf_size); - } - else { - ret = input_chunk_append_raw(in, event_type, records, - tag, tag_len, buf, buf_size); - } - - return ret; -} - -/* Retrieve a raw buffer from a dyntag node */ -const void *flb_input_chunk_flush(struct flb_input_chunk *ic, size_t *size) -{ - int ret; - size_t pre_size; - size_t post_size; - ssize_t diff_size; - char *buf = NULL; - - pre_size = flb_input_chunk_get_real_size(ic); - - if (cio_chunk_is_up(ic->chunk) == CIO_FALSE) { - ret = cio_chunk_up(ic->chunk); - if (ret == -1) { - return NULL; - } - } - - /* Lock the internal chunk - * - * This operation has to be performed before getting the chunk data - * pointer because in certain situations it could cause the chunk - * mapping to be relocated (ie. macos / windows on trim) - */ - cio_chunk_lock(ic->chunk); - - /* - * msgpack-c internal use a raw buffer for it operations, since we - * already appended data we just can take out the references to avoid - * a new memory allocation and skip a copy operation. - */ - ret = cio_chunk_get_content(ic->chunk, &buf, size); - - if (ret == -1) { - flb_error("[input chunk] error retrieving chunk content"); - return NULL; - } - - if (!buf) { - *size = 0; - return NULL; - } - - /* Set it busy as it likely it's a reference for an outgoing task */ - ic->busy = FLB_TRUE; - - post_size = flb_input_chunk_get_real_size(ic); - if (post_size != pre_size) { - diff_size = post_size - pre_size; - flb_input_chunk_update_output_instances(ic, diff_size); - } - return buf; -} - -int flb_input_chunk_release_lock(struct flb_input_chunk *ic) -{ - if (ic->busy == FLB_FALSE) { - return -1; - } - - ic->busy = FLB_FALSE; - return 0; -} - -flb_sds_t flb_input_chunk_get_name(struct flb_input_chunk *ic) -{ - struct cio_chunk *ch; - - ch = (struct cio_chunk *) ic->chunk; - return ch->name; -} - -static inline int input_chunk_has_magic_bytes(char *buf, int len) -{ - unsigned char *p; - - if (len < FLB_INPUT_CHUNK_META_HEADER) { - return FLB_FALSE; - } - - p = (unsigned char *) buf; - if (p[0] == FLB_INPUT_CHUNK_MAGIC_BYTE_0 && - p[1] == FLB_INPUT_CHUNK_MAGIC_BYTE_1 && p[3] == 0) { - return FLB_TRUE; - } - - return FLB_FALSE; -} - -/* - * Get the event type by retrieving metadata header. NOTE: this function only event type discovery by looking at the - * headers bytes of a chunk that exists on disk. - */ -int flb_input_chunk_get_event_type(struct flb_input_chunk *ic) -{ - int len; - int ret; - int type = -1; - char *buf = NULL; - - ret = cio_meta_read(ic->chunk, &buf, &len); - if (ret == -1) { - return -1; - } - - /* Check metadata header / magic bytes */ - if (input_chunk_has_magic_bytes(buf, len)) { - if (buf[2] == FLB_INPUT_CHUNK_TYPE_LOGS) { - type = FLB_INPUT_LOGS; - } - else if (buf[2] == FLB_INPUT_CHUNK_TYPE_METRICS) { - type = FLB_INPUT_METRICS; - } - else if (buf[2] == FLB_INPUT_CHUNK_TYPE_TRACES) { - type = FLB_INPUT_TRACES; - } - } - else { - type = FLB_INPUT_LOGS; - } - - - return type; -} - -int flb_input_chunk_get_tag(struct flb_input_chunk *ic, - const char **tag_buf, int *tag_len) -{ - int len; - int ret; - char *buf; - - ret = cio_meta_read(ic->chunk, &buf, &len); - if (ret == -1) { - *tag_len = -1; - *tag_buf = NULL; - return -1; - } - - /* If magic bytes exists, just set the offset */ - if (input_chunk_has_magic_bytes(buf, len)) { - *tag_len = len - FLB_INPUT_CHUNK_META_HEADER; - *tag_buf = buf + FLB_INPUT_CHUNK_META_HEADER; - } - else { - /* Old Chunk version without magic bytes */ - *tag_len = len; - *tag_buf = buf; - } - - return ret; -} - -/* - * Iterates all output instances that the chunk will be flushing to and summarize - * the total number of bytes in use after ingesting the new data. - */ -void flb_input_chunk_update_output_instances(struct flb_input_chunk *ic, - size_t chunk_size) -{ - struct mk_list *head; - struct flb_output_instance *o_ins; - - /* for each output plugin, we update the fs_chunks_size */ - mk_list_foreach(head, &ic->in->config->outputs) { - o_ins = mk_list_entry(head, struct flb_output_instance, _head); - if (o_ins->total_limit_size == -1) { - continue; - } - - if (flb_routes_mask_get_bit(ic->routes_mask, o_ins->id) != 0) { - /* - * if there is match on any index of 1's in the binary, it indicates - * that the input chunk will flush to this output instance - */ - FS_CHUNK_SIZE_DEBUG_MOD(o_ins, ic, chunk_size); - o_ins->fs_chunks_size += chunk_size; - ic->fs_counted = FLB_TRUE; - - flb_debug("[input chunk] chunk %s update plugin %s fs_chunks_size by %ld bytes, " - "the current fs_chunks_size is %ld bytes", flb_input_chunk_get_name(ic), - o_ins->name, chunk_size, o_ins->fs_chunks_size); - } - } -} diff --git a/fluent-bit/src/flb_input_log.c b/fluent-bit/src/flb_input_log.c deleted file mode 100644 index ed8fa8aa2..000000000 --- a/fluent-bit/src/flb_input_log.c +++ /dev/null @@ -1,123 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <fluent-bit/flb_info.h> -#include <fluent-bit/flb_input.h> -#include <fluent-bit/flb_input_chunk.h> -#include <fluent-bit/flb_input_log.h> -#include <fluent-bit/flb_input_plugin.h> -#include <fluent-bit/flb_processor.h> - -static int input_log_append(struct flb_input_instance *ins, - size_t processor_starting_stage, - size_t records, - const char *tag, size_t tag_len, - const void *buf, size_t buf_size) -{ - int ret; - int processor_is_active; - void *out_buf = (void *) buf; - size_t out_size = buf_size; - - processor_is_active = flb_processor_is_active(ins->processor); - if (processor_is_active) { - if (!tag) { - if (ins->tag && ins->tag_len > 0) { - tag = ins->tag; - tag_len = ins->tag_len; - } - else { - tag = ins->name; - tag_len = strlen(ins->name); - } - } - - ret = flb_processor_run(ins->processor, - processor_starting_stage, - FLB_PROCESSOR_LOGS, - tag, tag_len, - (char *) buf, buf_size, - &out_buf, &out_size); - if (ret == -1) { - return -1; - } - - if (out_size == 0) { - return 0; - } - - if (buf != out_buf) { - /* a new buffer was created, re-count the number of records */ - records = flb_mp_count(out_buf, out_size); - } - } - - ret = flb_input_chunk_append_raw(ins, FLB_INPUT_LOGS, records, - tag, tag_len, out_buf, out_size); - - - if (processor_is_active && buf != out_buf) { - flb_free(out_buf); - } - return ret; -} - -/* Take a msgpack serialized record and enqueue it as a chunk */ -int flb_input_log_append(struct flb_input_instance *ins, - const char *tag, size_t tag_len, - const void *buf, size_t buf_size) -{ - int ret; - size_t records; - - records = flb_mp_count(buf, buf_size); - ret = input_log_append(ins, 0, records, tag, tag_len, buf, buf_size); - return ret; -} - -/* Take a msgpack serialized record and enqueue it as a chunk */ -int flb_input_log_append_skip_processor_stages(struct flb_input_instance *ins, - size_t processor_starting_stage, - const char *tag, - size_t tag_len, - const void *buf, - size_t buf_size) -{ - return input_log_append(ins, - processor_starting_stage, - flb_mp_count(buf, buf_size), - tag, - tag_len, - buf, - buf_size); -} - -/* Take a msgpack serialized record and enqueue it as a chunk */ -int flb_input_log_append_records(struct flb_input_instance *ins, - size_t records, - const char *tag, size_t tag_len, - const void *buf, size_t buf_size) -{ - int ret; - - ret = input_log_append(ins, 0, records, tag, tag_len, buf, buf_size); - return ret; -} - - diff --git a/fluent-bit/src/flb_input_metric.c b/fluent-bit/src/flb_input_metric.c deleted file mode 100644 index f332d4203..000000000 --- a/fluent-bit/src/flb_input_metric.c +++ /dev/null @@ -1,101 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <fluent-bit/flb_info.h> -#include <fluent-bit/flb_input.h> -#include <fluent-bit/flb_input_chunk.h> -#include <fluent-bit/flb_input_metric.h> -#include <fluent-bit/flb_input_plugin.h> - -static int input_metrics_append(struct flb_input_instance *ins, - size_t processor_starting_stage, - const char *tag, size_t tag_len, - struct cmt *cmt) -{ - int ret; - char *mt_buf; - size_t mt_size; - int processor_is_active; - - processor_is_active = flb_processor_is_active(ins->processor); - if (processor_is_active) { - if (!tag) { - if (ins->tag && ins->tag_len > 0) { - tag = ins->tag; - tag_len = ins->tag_len; - } - else { - tag = ins->name; - tag_len = strlen(ins->name); - } - } - - ret = flb_processor_run(ins->processor, - processor_starting_stage, - FLB_PROCESSOR_METRICS, - tag, - tag_len, - (char *) cmt, - 0, NULL, NULL); - - if (ret == -1) { - return -1; - } - } - - /* Convert metrics to msgpack */ - ret = cmt_encode_msgpack_create(cmt, &mt_buf, &mt_size); - if (ret != 0) { - flb_plg_error(ins, "could not encode metrics"); - return -1; - - } - - /* Append packed metrics */ - ret = flb_input_chunk_append_raw(ins, FLB_INPUT_METRICS, 0, - tag, tag_len, mt_buf, mt_size); - - cmt_encode_msgpack_destroy(mt_buf); - - return ret; -} - -/* Take a metric context and enqueue it as a Metric's Chunk */ -int flb_input_metrics_append(struct flb_input_instance *ins, - const char *tag, size_t tag_len, - struct cmt *cmt) -{ - return input_metrics_append(ins, - 0, - tag, tag_len, - cmt); -} - -/* Take a metric context and enqueue it as a Metric's Chunk */ -int flb_input_metrics_append_skip_processor_stages( - struct flb_input_instance *ins, - size_t processor_starting_stage, - const char *tag, size_t tag_len, - struct cmt *cmt) -{ - return input_metrics_append(ins, - processor_starting_stage, - tag, tag_len, - cmt); -} diff --git a/fluent-bit/src/flb_input_thread.c b/fluent-bit/src/flb_input_thread.c deleted file mode 100644 index bf073296d..000000000 --- a/fluent-bit/src/flb_input_thread.c +++ /dev/null @@ -1,745 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2019-2021 The Fluent Bit Authors - * Copyright (C) 2015-2018 Treasure Data Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <fluent-bit/flb_info.h> -#include <fluent-bit/flb_pipe.h> -#include <fluent-bit/flb_engine.h> -#include <fluent-bit/flb_mem.h> -#include <fluent-bit/flb_mp.h> -#include <fluent-bit/flb_log.h> -#include <fluent-bit/flb_event_loop.h> -#include <fluent-bit/flb_utils.h> -#include <fluent-bit/flb_scheduler.h> -#include <fluent-bit/flb_downstream.h> -#include <fluent-bit/flb_input.h> -#include <fluent-bit/flb_input_plugin.h> -#include <fluent-bit/flb_input_thread.h> - -static int input_thread_instance_set_status(struct flb_input_instance *ins, uint32_t status); -static int input_thread_instance_get_status(struct flb_input_instance *ins); - -/* Cleanup function that runs every 1.5 second */ -static void cb_thread_sched_timer(struct flb_config *ctx, void *data) -{ - struct flb_input_instance *ins; - - (void) ctx; - - /* Downstream timeout handling */ - ins = (struct flb_input_instance *) data; - - flb_upstream_conn_timeouts(&ins->upstreams); - flb_downstream_conn_timeouts(&ins->downstreams); -} - -static inline int handle_input_event(flb_pipefd_t fd, struct flb_input_instance *ins) -{ - int bytes; - int ins_id; - uint32_t type; - uint32_t operation; - uint64_t val; - struct flb_config *config = ins->config; - - bytes = read(fd, &val, sizeof(val)); - if (bytes == -1) { - flb_errno(); - return -1; - } - - type = FLB_BITS_U64_HIGH(val); - operation = FLB_BITS_U64_LOW(val); - - /* At the moment we only support events coming from an input coroutine */ - if (type == FLB_ENGINE_IN_CORO) { - ins_id = ins->id; - flb_input_coro_finished(config, ins_id); - } - else if (type == FLB_INPUT_THREAD_TO_THREAD) { - if (operation == FLB_INPUT_THREAD_PAUSE) { - if (ins->p->cb_pause && ins->context) { - ins->p->cb_pause(ins->context, ins->config); - } - } - else if (operation == FLB_INPUT_THREAD_RESUME) { - if (ins->p->cb_resume) { - ins->p->cb_resume(ins->context, ins->config); - } - } - else if (operation == FLB_INPUT_THREAD_EXIT) { - return FLB_INPUT_THREAD_EXIT; - } - } - else { - flb_error("[thread event loop] it happends on fd=%i, invalid type=%i", fd, type); - return -1; - } - - return 0; -} - -static inline int handle_input_thread_event(flb_pipefd_t fd, struct flb_config *config) -{ - int bytes; - uint32_t type; - uint32_t ins_id; - uint64_t val; - - bytes = flb_pipe_r(fd, &val, sizeof(val)); - if (bytes == -1) { - flb_errno(); - return -1; - } - - /* Get type and key */ - type = FLB_BITS_U64_HIGH(val); - ins_id = FLB_BITS_U64_LOW(val); - - /* At the moment we only support events coming from an input coroutine */ - if (type == FLB_ENGINE_IN_CORO) { - flb_input_coro_finished(config, ins_id); - } - else { - flb_error("[thread event loop] invalid thread event type %i for input handler", - type); - return -1; - } - - return 0; -} - -static int input_collector_fd(flb_pipefd_t fd, struct flb_input_instance *ins) -{ - struct mk_list *head; - struct flb_input_collector *collector = NULL; - struct flb_input_coro *input_coro; - struct flb_config *config = ins->config; - - mk_list_foreach(head, &ins->collectors) { - collector = mk_list_entry(head, struct flb_input_collector, _head); - if (collector->fd_event == fd) { - break; - } - else if (collector->fd_timer == fd) { - flb_utils_timer_consume(fd); - break; - } - collector = NULL; - } - - /* No matches */ - if (!collector) { - return -1; - } - - if (collector->running == FLB_FALSE) { - return -1; - } - - /* Trigger the collector callback */ - if (collector->instance->runs_in_coroutine) { - input_coro = flb_input_coro_collect(collector, config); - if (!input_coro) { - return -1; - } - flb_input_coro_resume(input_coro); - } - else { - collector->cb_collect(collector->instance, config, - collector->instance->context); - } - - return 0; -} - -static FLB_INLINE int engine_handle_event(flb_pipefd_t fd, int mask, - struct flb_input_instance *ins, - struct flb_config *config) -{ - int ret; - - if (mask & MK_EVENT_READ) { - /* Try to match the file descriptor with a collector event */ - ret = input_collector_fd(fd, ins); - if (ret != -1) { - return ret; - } - } - - return 0; -} - -static void input_thread_instance_destroy(struct flb_input_thread_instance *thi) -{ - if (thi->evl) { - mk_event_loop_destroy(thi->evl); - } - - /* ch_parent_events */ - if (thi->ch_parent_events[0] > 0) { - mk_event_closesocket(thi->ch_parent_events[0]); - } - if (thi->ch_parent_events[1] > 0) { - mk_event_closesocket(thi->ch_parent_events[1]); - } - - /* ch_thread_events */ - if (thi->ch_thread_events[0] > 0) { - mk_event_closesocket(thi->ch_thread_events[0]); - } - if (thi->ch_thread_events[1] > 0) { - mk_event_closesocket(thi->ch_thread_events[1]); - } - - flb_tp_destroy(thi->tp); - flb_free(thi); -} - -static struct flb_input_thread_instance *input_thread_instance_create(struct flb_input_instance *ins) -{ - int ret; - struct flb_input_thread_instance *thi; - - /* context for thread */ - thi = flb_calloc(1, sizeof(struct flb_input_thread_instance)); - if (!thi) { - flb_errno(); - return NULL; - } - thi->ins = ins; - thi->config = ins->config; - - /* init status */ - thi->init_status = 0; - pthread_mutex_init(&thi->init_mutex, NULL); - - /* init condition */ - pthread_cond_init(&thi->init_condition, NULL); - - /* initialize lists */ - mk_list_init(&thi->input_coro_list); - mk_list_init(&thi->input_coro_list_destroy); - - /* event loop */ - thi->evl = mk_event_loop_create(256); - if (!thi->evl) { - input_thread_instance_destroy(thi); - return NULL; - } - - /* channel to receive parent (engine) notifications */ - ret = mk_event_channel_create(thi->evl, - &thi->ch_parent_events[0], - &thi->ch_parent_events[1], - &thi->event); - if (ret == -1) { - flb_error("could not initialize parent channels for %s", - flb_input_name(ins)); - input_thread_instance_destroy(thi); - return NULL; - } - thi->event.type = FLB_ENGINE_EV_INPUT; - - /* channel to send messages to local event loop */ - ret = mk_event_channel_create(thi->evl, - &thi->ch_thread_events[0], - &thi->ch_thread_events[1], - &thi->event_local); - if (ret == -1) { - flb_error("could not initialize parent channels for %s", - flb_input_name(ins)); - input_thread_instance_destroy(thi); - return NULL; - } - thi->event_local.type = FLB_ENGINE_EV_THREAD_INPUT; - - /* create thread pool, just one worker */ - thi->tp = flb_tp_create(ins->config); - if (!thi->tp) { - flb_error("could not create thread pool on input instance '%s'", - flb_input_name(ins)); - input_thread_instance_destroy(thi); - return NULL; - } - - return thi; -} - - -static void input_thread(void *data) -{ - int ret; - int thread_id; - char tmp[64]; - int instance_exit = FLB_FALSE; - struct mk_event *event; - struct flb_input_instance *ins; - struct flb_bucket_queue *evl_bktq = NULL; - struct flb_input_thread_instance *thi; - struct flb_input_plugin *p; - struct flb_sched *sched = NULL; - struct flb_net_dns dns_ctx = {0}; - - thi = (struct flb_input_thread_instance *) data; - ins = thi->ins; - p = ins->p; - - flb_engine_evl_set(thi->evl); - - /* Create a scheduler context */ - sched = flb_sched_create(ins->config, thi->evl); - if (!sched) { - flb_plg_error(ins, "could not create thread scheduler"); - return; - } - flb_sched_ctx_set(sched); - - /* - * Sched a permanent callback triggered every 1.5 second to let other - * components of this thread run tasks at that interval. - */ - ret = flb_sched_timer_cb_create(sched, - FLB_SCHED_TIMER_CB_PERM, - 1500, cb_thread_sched_timer, ins, NULL); - if (ret == -1) { - flb_error("could not schedule input thread permanent callback"); - return; - } - - flb_coro_thread_init(); - - flb_net_ctx_init(&dns_ctx); - flb_net_dns_ctx_set(&dns_ctx); - - thread_id = thi->th->id; - snprintf(tmp, sizeof(tmp) - 1, "flb-in-%s-w%i", ins->name, thread_id); - mk_utils_worker_rename(tmp); - - /* invoke plugin 'init' callback */ - ret = p->cb_init(ins, ins->config, ins->data); - if (ret == -1) { - flb_error("failed initialize input %s", flb_input_name(ins)); - /* message the parent thread that this thread could not be initialized */ - input_thread_instance_set_status(ins, FLB_INPUT_THREAD_ERROR); - return; - } - - flb_plg_debug(ins, "[thread init] initialization OK"); - input_thread_instance_set_status(ins, FLB_INPUT_THREAD_OK); - - /* - * Wait for parent thread to signal this thread so we can start collectors and - * get into the event loop - */ - ret = flb_input_thread_collectors_signal_wait(ins); - if (ret == -1) { - flb_error("could not retrieve collectors signal from parent thread on '%s'", - flb_input_name(ins)); - return; - } - - /* event loop queue */ - evl_bktq = flb_bucket_queue_create(FLB_ENGINE_PRIORITY_COUNT); - - /* Start collectors */ - flb_input_thread_collectors_start(ins); - - /* If the plugin contains a 'pre_run' callback, invoke it */ - if (p->cb_pre_run) { - ret = p->cb_pre_run(ins, ins->config, ins->context); - if (ret == -1) { - /* - * FIXME: how do we report a failed pre-run status to the parent thread ?, - * as of know it does not seems to be necessary since the only plugins - * using that callback are tail and systemd, but those are just writing a - * byte to a recently created pipe in the initialization. - */ - } - } - - while (1) { - mk_event_wait(thi->evl); - flb_event_priority_live_foreach(event, evl_bktq, thi->evl, FLB_ENGINE_LOOP_MAX_ITER) { - if (event->type == FLB_ENGINE_EV_CORE) { - ret = engine_handle_event(event->fd, event->mask, - ins, thi->config); - if (ret == FLB_ENGINE_STOP) { - continue; - } - else if (ret == FLB_ENGINE_SHUTDOWN) { - continue; - } - } - else if (event->type & FLB_ENGINE_EV_SCHED) { - /* Event type registered by the Scheduler */ - flb_sched_event_handler(ins->config, event); - } - else if (event->type == FLB_ENGINE_EV_THREAD_ENGINE) { - struct flb_output_flush *output_flush; - - /* Read the coroutine reference */ - ret = flb_pipe_r(event->fd, &output_flush, sizeof(struct flb_output_flush *)); - if (ret <= 0 || output_flush == 0) { - flb_errno(); - continue; - } - - /* Init coroutine */ - flb_coro_resume(output_flush->coro); - } - else if (event->type == FLB_ENGINE_EV_CUSTOM) { - event->handler(event); - } - else if (event->type == FLB_ENGINE_EV_THREAD) { - struct flb_connection *connection; - - /* - * Check if we have some co-routine associated to this event, - * if so, resume the co-routine - */ - connection = (struct flb_connection *) event; - - if (connection->coroutine != NULL) { - flb_trace("[engine] resuming coroutine=%p", - connection->coroutine); - - flb_coro_resume(connection->coroutine); - } - } - else if (event->type == FLB_ENGINE_EV_INPUT) { - ret = handle_input_event(event->fd, ins); - if (ret == FLB_INPUT_THREAD_EXIT) { - instance_exit = FLB_TRUE; - } - } - else if (event->type == FLB_ENGINE_EV_THREAD_INPUT) { - handle_input_thread_event(event->fd, ins->config); - } - } - - flb_net_dns_lookup_context_cleanup(&dns_ctx); - - /* Destroy upstream connections from the 'pending destroy list' */ - flb_upstream_conn_pending_destroy_list(&ins->upstreams); - - /* Destroy downstream connections from the 'pending destroy list' */ - flb_downstream_conn_pending_destroy_list(&ins->downstreams); - flb_sched_timer_cleanup(sched); - - /* Check if the instance must exit */ - if (instance_exit) { - /* Invoke exit callback */ - if (ins->p->cb_exit && ins->context) { - ins->p->cb_exit(ins->context, ins->config); - } - - /* break the loop */ - break; - } - } - - /* Create the bucket queue (FLB_ENGINE_PRIORITY_COUNT priorities) */ - flb_bucket_queue_destroy(evl_bktq); - flb_sched_destroy(sched); - input_thread_instance_destroy(thi); -} - - -/* - * Signal the thread event loop to pause the running plugin instance. This function - * must be called only from the main thread/pipeline. - */ -int flb_input_thread_instance_pause(struct flb_input_instance *ins) -{ - int ret; - uint64_t val; - struct flb_input_thread_instance *thi = ins->thi; - - flb_plg_debug(ins, "thread pause instance"); - - /* compose message to pause the thread */ - val = FLB_BITS_U64_SET(FLB_INPUT_THREAD_TO_THREAD, - FLB_INPUT_THREAD_PAUSE); - - ret = flb_pipe_w(thi->ch_parent_events[1], &val, sizeof(val)); - if (ret <= 0) { - flb_errno(); - return -1; - } - - return 0; -} - -/* - * Signal the thread event loop to resume the running plugin instance. This function - * must be called only from the main thread/pipeline. - */ -int flb_input_thread_instance_resume(struct flb_input_instance *ins) -{ - int ret; - uint64_t val; - struct flb_input_thread_instance *thi = ins->thi; - - flb_plg_debug(ins, "thread resume instance"); - - /* compose message to resume the thread */ - val = FLB_BITS_U64_SET(FLB_INPUT_THREAD_TO_THREAD, - FLB_INPUT_THREAD_RESUME); - - ret = flb_pipe_w(thi->ch_parent_events[1], &val, sizeof(val)); - if (ret <= 0) { - flb_errno(); - return -1; - } - - return 0; -} - -int flb_input_thread_instance_exit(struct flb_input_instance *ins) -{ - int ret; - uint64_t val; - struct flb_input_thread_instance *thi = ins->thi; - pthread_t tid; - - memcpy(&tid, &thi->th->tid, sizeof(pthread_t)); - - /* compose message to pause the thread */ - val = FLB_BITS_U64_SET(FLB_INPUT_THREAD_TO_THREAD, - FLB_INPUT_THREAD_EXIT); - - ret = flb_pipe_w(thi->ch_parent_events[1], &val, sizeof(val)); - if (ret <= 0) { - flb_errno(); - return -1; - } - - pthread_join(tid, NULL); - flb_plg_debug(ins, "thread exit instance"); - - return 0; -} - - -/* Initialize a plugin under a threaded context */ -int flb_input_thread_instance_init(struct flb_config *config, struct flb_input_instance *ins) -{ - int ret; - struct flb_tp_thread *th; - struct flb_input_thread_instance *thi; - - /* Create the threaded context for the instance in question */ - thi = input_thread_instance_create(ins); - if (!thi) { - return -1; - } - ins->thi = thi; - - /* Spawn the thread */ - th = flb_tp_thread_create(thi->tp, input_thread, thi, config); - if (!th) { - flb_plg_error(ins, "could not register worker thread"); - input_thread_instance_destroy(thi); - return -1; - } - thi->th = th; - - /* start the thread */ - ret = flb_tp_thread_start(thi->tp, thi->th); - if (ret != 0) { - return -1; - } - - ret = input_thread_instance_get_status(ins); - if (ret == -1) { - flb_plg_error(ins, "unexpected error loading plugin instance"); - } - else if (ret == FLB_FALSE) { - flb_plg_error(ins, "could not initialize threaded plugin instance"); - } - else if (ret == FLB_TRUE) { - flb_plg_info(ins, "thread instance initialized"); - } - - return 0; -} - -int flb_input_thread_instance_pre_run(struct flb_config *config, struct flb_input_instance *ins) -{ - int ret; - - if (ins->p->cb_pre_run) { - /* - * the pre_run callback is invoked automatically from the instance thread. we just need to check for the - * final status. - */ - ret = input_thread_instance_get_status(ins); - if (ret == -1) { - return -1; - } - else if (ret == FLB_FALSE) { - return -1; - } - else if (ret == FLB_TRUE) { - return 0; - } - } - - return 0; -} - -static int input_thread_instance_set_status(struct flb_input_instance *ins, uint32_t status) -{ - struct flb_input_thread_instance *thi; - - thi = ins->thi; - - pthread_mutex_lock(&thi->init_mutex); - - thi->init_status = status; - - pthread_cond_signal(&thi->init_condition); - pthread_mutex_unlock(&thi->init_mutex); - - return 0; -} - -static int input_thread_instance_get_status(struct flb_input_instance *ins) -{ - - uint32_t status; - struct flb_input_thread_instance *thi; - - thi = ins->thi; - - /* Wait for thread to report a status */ - pthread_mutex_lock(&thi->init_mutex); - while (thi->init_status == 0) { - pthread_cond_wait(&thi->init_condition, &thi->init_mutex); - } - pthread_mutex_unlock(&thi->init_mutex); - - /* re-initialize condition */ - pthread_cond_destroy(&thi->init_condition); - pthread_cond_init(&thi->init_condition, NULL); - - /* get the final status */ - status = thi->init_status; - if (status == FLB_INPUT_THREAD_OK) { - return FLB_TRUE; - } - else if (status == FLB_INPUT_THREAD_ERROR) { - return FLB_FALSE;; - } - - return -1; -} - -/* Wait for an input thread instance to become ready, or a failure status */ -int flb_input_thread_wait_until_is_ready(struct flb_input_instance *ins) -{ - uint64_t status = 0; - size_t bytes; - struct flb_input_thread_instance *thi; - - thi = ins->thi; - - bytes = read(thi->ch_parent_events[0], &status, sizeof(uint64_t)); - if (bytes <= 0) { - flb_errno(); - return -1; - } - - if (status == 0) { - return -1; - } - - return FLB_TRUE; -} - - -/* - * Invoked from the main 'input' interface to signal the threaded plugin instance so - * it can start the collectors. - */ -int flb_input_thread_collectors_signal_start(struct flb_input_instance *ins) -{ - int ret; - uint64_t val; - struct flb_input_thread_instance *thi; - - thi = ins->thi; - - /* compose message */ - val = FLB_BITS_U64_SET(FLB_INPUT_THREAD_TO_THREAD, - FLB_INPUT_THREAD_START_COLLECTORS); - - ret = flb_pipe_w(thi->ch_parent_events[1], &val, sizeof(uint64_t)); - if (ret <= 0) { - flb_errno(); - return -1; - } - - return 0; -} - -int flb_input_thread_collectors_signal_wait(struct flb_input_instance *ins) -{ - size_t bytes; - uint32_t type; - uint32_t op; - uint64_t val = 0; - struct flb_input_thread_instance *thi; - - thi = ins->thi; - bytes = flb_pipe_r(thi->ch_parent_events[0], &val, sizeof(uint64_t)); - if (bytes <= 0) { - flb_errno(); - return -1; - } - - /* Get type and status */ - type = FLB_BITS_U64_HIGH(val); - op = FLB_BITS_U64_LOW(val); - - if (type != FLB_INPUT_THREAD_TO_THREAD || op != FLB_INPUT_THREAD_START_COLLECTORS) { - flb_plg_error(ins, "wrong event, type=%i op=%i\n", type, op); fflush(stdout); - return -1; - } - - return 0; -} - -int flb_input_thread_collectors_start(struct flb_input_instance *ins) -{ - int ret; - struct mk_list *head; - struct flb_input_collector *coll; - - mk_list_foreach(head, &ins->collectors) { - coll = mk_list_entry(head, struct flb_input_collector, _head); - ret = flb_input_collector_start(coll->id, ins); - if (ret < 0) { - return -1; - } - } - - return 0; -} diff --git a/fluent-bit/src/flb_input_trace.c b/fluent-bit/src/flb_input_trace.c deleted file mode 100644 index ced8e9ee2..000000000 --- a/fluent-bit/src/flb_input_trace.c +++ /dev/null @@ -1,102 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <fluent-bit/flb_info.h> -#include <fluent-bit/flb_input.h> -#include <fluent-bit/flb_input_chunk.h> -#include <fluent-bit/flb_input_trace.h> -#include <fluent-bit/flb_input_plugin.h> - -#include <ctraces/ctraces.h> -#include <ctraces/ctr_decode_msgpack.h> - -static int input_trace_append(struct flb_input_instance *ins, - size_t processor_starting_stage, - const char *tag, size_t tag_len, - struct ctrace *ctr) -{ - int ret; - char *out_buf; - size_t out_size; - int processor_is_active; - - processor_is_active = flb_processor_is_active(ins->processor); - if (processor_is_active) { - if (!tag) { - if (ins->tag && ins->tag_len > 0) { - tag = ins->tag; - tag_len = ins->tag_len; - } - else { - tag = ins->name; - tag_len = strlen(ins->name); - } - } - - ret = flb_processor_run(ins->processor, - processor_starting_stage, - FLB_PROCESSOR_TRACES, - tag, tag_len, - (char *) ctr, - 0, NULL, NULL); - - if (ret == -1) { - return -1; - } - } - - /* Convert trace context to msgpack */ - ret = ctr_encode_msgpack_create(ctr, &out_buf, &out_size); - if (ret != 0) { - flb_plg_error(ins, "could not encode traces"); - return -1; - } - - /* Append packed metrics */ - ret = flb_input_chunk_append_raw(ins, FLB_INPUT_TRACES, 0, - tag, tag_len, out_buf, out_size); - - ctr_encode_msgpack_destroy(out_buf); - - return ret; -} - -/* Take a CTrace context and enqueue it as a Trace chunk */ -int flb_input_trace_append(struct flb_input_instance *ins, - const char *tag, size_t tag_len, - struct ctrace *ctr) -{ - return input_trace_append(ins, - 0, - tag, tag_len, - ctr); -} - -/* Take a CTrace context and enqueue it as a Trace chunk */ -int flb_input_trace_append_skip_processor_stages( - struct flb_input_instance *ins, - size_t processor_starting_stage, - const char *tag, size_t tag_len, - struct ctrace *ctr) -{ - return input_trace_append(ins, - processor_starting_stage, - tag, tag_len, - ctr); -} diff --git a/fluent-bit/src/flb_io.c b/fluent-bit/src/flb_io.c deleted file mode 100644 index 81303459c..000000000 --- a/fluent-bit/src/flb_io.c +++ /dev/null @@ -1,749 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * FLB_IO - * ====== - * This interface is used by the output plugins which needs to write over - * the network in plain communication or through the TLS support. When dealing - * with network operation there are a few things to keep in mind: - * - * - TCP hosts can be down. - * - Network can be slow. - * - If the amount of data to flush requires multiple 'write' operations, we - * should not block the main thread, instead use event-driven mechanism to - * write when is possible. - * - * Output plugins that flag themselves with FLB_OUTPUT_TCP or FLB_OUTPUT_TLS - * can take advantage of this interface. - * - * The workflow to use this is the following: - * - * - A connection and data flow requires an flb_io_upstream context. - * - We write/read data through the flb_io_write()/flb_io_read() interfaces. - * - * Note that Upstreams context may define how network operations will work, - * basically synchronous or asynchronous (non-blocking). - */ - -#include <stdio.h> -#include <stdlib.h> -#include <limits.h> -#include <assert.h> - -#include <monkey/mk_core.h> -#include <fluent-bit/flb_info.h> -#include <fluent-bit/flb_config.h> -#include <fluent-bit/flb_io.h> -#include <fluent-bit/tls/flb_tls.h> -#include <fluent-bit/flb_socket.h> -#include <fluent-bit/flb_upstream.h> -#include <fluent-bit/flb_downstream.h> - -#include <fluent-bit/flb_utils.h> -#include <fluent-bit/flb_macros.h> -#include <fluent-bit/flb_network.h> -#include <fluent-bit/flb_engine.h> -#include <fluent-bit/flb_coro.h> -#include <fluent-bit/flb_http_client.h> - -int flb_io_net_accept(struct flb_connection *connection, - struct flb_coro *coro) -{ - int ret; - - if (connection->fd != FLB_INVALID_SOCKET) { - flb_socket_close(connection->fd); - - connection->fd = FLB_INVALID_SOCKET; - connection->event.fd = FLB_INVALID_SOCKET; - } - - /* Accept the new connection */ - connection->fd = flb_net_accept(connection->downstream->server_fd); - - if (connection->fd == -1) { - connection->fd = FLB_INVALID_SOCKET; - - return -1; - } - -#ifdef FLB_HAVE_TLS - /* Check if TLS was enabled, if so perform the handshake */ - if (flb_stream_is_secure(connection->stream) && - connection->stream->tls_context != NULL) { - ret = flb_tls_session_create(connection->stream->tls_context, - connection, - coro); - - if (ret != 0) { - return -1; - } - } -#endif - - flb_trace("[io] connection OK"); - - return 0; -} - -int flb_io_net_connect(struct flb_connection *connection, - struct flb_coro *coro) -{ - int ret; - int async = FLB_FALSE; - flb_sockfd_t fd = -1; - // struct flb_upstream *u = u_conn->u; - - if (connection->fd > 0) { - flb_socket_close(connection->fd); - - connection->fd = -1; - connection->event.fd = -1; - } - - /* Check which connection mode must be done */ - if (coro) { - async = flb_upstream_is_async(connection->upstream); - } - else { - async = FLB_FALSE; - } - - /* Perform TCP connection */ - fd = flb_net_tcp_connect(connection->upstream->tcp_host, - connection->upstream->tcp_port, - connection->stream->net.source_address, - connection->stream->net.connect_timeout, - async, coro, connection); - if (fd == -1) { - return -1; - } - - if (connection->upstream->proxied_host) { - ret = flb_http_client_proxy_connect(connection); - - if (ret == -1) { - flb_debug("[http_client] flb_http_client_proxy_connect connection #%i failed to %s:%i.", - connection->fd, - connection->upstream->tcp_host, - connection->upstream->tcp_port); - - flb_socket_close(fd); - - return -1; - } - flb_debug("[http_client] flb_http_client_proxy_connect connection #%i connected to %s:%i.", - connection->fd, - connection->upstream->tcp_host, - connection->upstream->tcp_port); - } - -#ifdef FLB_HAVE_TLS - /* Check if TLS was enabled, if so perform the handshake */ - if (flb_stream_is_secure(connection->stream) && - connection->stream->tls_context != NULL) { - ret = flb_tls_session_create(connection->stream->tls_context, - connection, - coro); - - if (ret != 0) { - return -1; - } - } -#endif - - flb_trace("[io] connection OK"); - - return 0; -} - -static void net_io_propagate_critical_error( - struct flb_connection *connection) -{ - switch (errno) { - case EBADF: - case ECONNRESET: - case EDESTADDRREQ: - case ENOTCONN: - case EPIPE: - case EACCES: - case ENOTTY: - case ENETDOWN: - case ENETUNREACH: - connection->net_error = errno; - } -} - -static int fd_io_write(int fd, struct sockaddr_storage *address, - const void *data, size_t len, size_t *out_len); -static int net_io_write(struct flb_connection *connection, - const void *data, size_t len, size_t *out_len) -{ - struct sockaddr_storage *address; - int ret; - - if (connection->fd <= 0) { - if (connection->type != FLB_UPSTREAM_CONNECTION) { - return -1; - } - - ret = flb_io_net_connect((struct flb_connection *) connection, - flb_coro_get()); - - if (ret == -1) { - return -1; - } - } - - address = NULL; - - if (connection->type == FLB_DOWNSTREAM_CONNECTION) { - if (connection->stream->transport == FLB_TRANSPORT_UDP || - connection->stream->transport == FLB_TRANSPORT_UNIX_DGRAM) { - address = &connection->raw_remote_host; - } - } - - ret = fd_io_write(connection->fd, address, data, len, out_len); - - if (ret == -1) { - net_io_propagate_critical_error(connection); - } - - return ret; -} - -static int fd_io_write(int fd, struct sockaddr_storage *address, - const void *data, size_t len, size_t *out_len) -{ - int ret; - int tries = 0; - size_t total = 0; - - while (total < len) { - if (address != NULL) { - ret = sendto(fd, (char *) data + total, len - total, 0, - (struct sockaddr *) address, - flb_network_address_size(address)); - } - else { - ret = send(fd, (char *) data + total, len - total, 0); - } - - if (ret == -1) { - if (FLB_WOULDBLOCK()) { - /* - * FIXME: for now we are handling this in a very lazy way, - * just sleep for a second and retry (for a max of 30 tries). - */ - sleep(1); - tries++; - - if (tries == 30) { - /* Since we're aborting after 30 failures we want the - * caller to know how much data we were able to send - */ - - *out_len = total; - - return -1; - } - - continue; - } - - return -1; - } - - tries = 0; - total += ret; - } - - *out_len = total; - - return total; -} - -static FLB_INLINE void net_io_backup_event(struct flb_connection *connection, - struct mk_event *backup) -{ - if (connection != NULL && backup != NULL) { - memcpy(backup, &connection->event, sizeof(struct mk_event)); - } -} - -static FLB_INLINE void net_io_restore_event(struct flb_connection *connection, - struct mk_event *backup) -{ - int result; - - if (connection != NULL && backup != NULL) { - if (MK_EVENT_IS_REGISTERED((&connection->event))) { - result = mk_event_del(connection->evl, &connection->event); - - assert(result == 0); - } - - if (MK_EVENT_IS_REGISTERED(backup)) { - connection->event.priority = backup->priority; - connection->event.handler = backup->handler; - - result = mk_event_add(connection->evl, - connection->fd, - backup->type, - backup->mask, - &connection->event); - - assert(result == 0); - } - } -} - -/* - * Perform Async socket write(2) operations. This function depends on a main - * event-loop and the co-routines interface to yield/resume once sockets are - * ready to continue. - * - * Intentionally we register/de-register the socket file descriptor from - * the event loop each time when we require to do some work. - */ -static FLB_INLINE int net_io_write_async(struct flb_coro *co, - struct flb_connection *connection, - const void *data, size_t len, size_t *out_len) -{ - int ret = 0; - int error; - uint32_t mask; - ssize_t bytes; - size_t total = 0; - size_t to_send; - char so_error_buf[256]; - struct mk_event event_backup; - int event_restore_needed; - - event_restore_needed = FLB_FALSE; - - net_io_backup_event(connection, &event_backup); - -retry: - error = 0; - - if (len - total > 524288) { - to_send = 524288; - } - else { - to_send = (len - total); - } - - bytes = send(connection->fd, (char *) data + total, to_send, 0); - -#ifdef FLB_HAVE_TRACE - if (bytes > 0) { - flb_trace("[io coro=%p] [fd %i] write_async(2)=%d (%lu/%lu)", - co, connection->fd, bytes, total + bytes, len); - } - else { - flb_trace("[io coro=%p] [fd %i] write_async(2)=%d (%lu/%lu)", - co, connection->fd, bytes, total, len); - } -#endif - - if (bytes == -1) { - if (FLB_WOULDBLOCK()) { - event_restore_needed = FLB_TRUE; - - ret = mk_event_add(connection->evl, - connection->fd, - FLB_ENGINE_EV_THREAD, - MK_EVENT_WRITE, - &connection->event); - - connection->event.priority = FLB_ENGINE_PRIORITY_SEND_RECV; - - if (ret == -1) { - /* - * If we failed here there no much that we can do, just - * let the caller we failed - */ - *out_len = total; - - net_io_restore_event(connection, &event_backup); - - return -1; - } - - connection->coroutine = co; - - /* - * Return the control to the parent caller, we need to wait for - * the event loop to get back to us. - */ - flb_coro_yield(co, FLB_FALSE); - - /* We want this field to hold NULL at all times unless we are explicitly - * waiting to be resumed. - */ - connection->coroutine = NULL; - - /* Save events mask since mk_event_del() will reset it */ - mask = connection->event.mask; - - /* We got a notification, remove the event registered */ - ret = mk_event_del(connection->evl, &connection->event); - - if (ret == -1) { - *out_len = total; - - net_io_restore_event(connection, &event_backup); - - return -1; - } - - /* Check the connection status */ - if (mask & MK_EVENT_WRITE) { - error = flb_socket_error(connection->fd); - - if (error != 0) { - /* Connection is broken, not much to do here */ - strerror_r(error, so_error_buf, sizeof(so_error_buf) - 1); - - flb_error("[io fd=%i] error sending data to: %s (%s)", - connection->fd, - flb_connection_get_remote_address(connection), - so_error_buf); - - *out_len = total; - - net_io_restore_event(connection, &event_backup); - - return -1; - } - - MK_EVENT_NEW(&connection->event); - - goto retry; - } - else { - *out_len = total; - - net_io_restore_event(connection, &event_backup); - - return -1; - } - - } - else { - *out_len = total; - - net_io_restore_event(connection, &event_backup); - net_io_propagate_critical_error(connection); - - return -1; - } - } - - /* Update counters */ - total += bytes; - if (total < len) { - if ((connection->event.mask & MK_EVENT_WRITE) == 0) { - ret = mk_event_add(connection->evl, - connection->fd, - FLB_ENGINE_EV_THREAD, - MK_EVENT_WRITE, - &connection->event); - - connection->event.priority = FLB_ENGINE_PRIORITY_SEND_RECV; - - if (ret == -1) { - /* - * If we failed here there no much that we can do, just - * let the caller we failed - */ - *out_len = total; - - net_io_restore_event(connection, &event_backup); - - return -1; - } - } - - connection->coroutine = co; - - flb_coro_yield(co, MK_FALSE); - - /* We want this field to hold NULL at all times unless we are explicitly - * waiting to be resumed. - */ - connection->coroutine = NULL; - - goto retry; - } - - if (event_restore_needed) { - /* If we enter here it means we registered this connection - * in the event loop, in which case we need to unregister it - * and restore the original registration if there was one. - * - * We do it conditionally because in those cases in which - * send succeeds on the first try we don't touch the event - * and it wouldn't make sense to unregister and register for - * the same event. - */ - - net_io_restore_event(connection, &event_backup); - } - - *out_len = total; - - return bytes; -} - -static ssize_t fd_io_read(int fd, struct sockaddr_storage *address, - void *buf, size_t len); -static ssize_t net_io_read(struct flb_connection *connection, - void *buf, size_t len) -{ - struct sockaddr_storage *address; - int ret; - - address = NULL; - - if (connection->type == FLB_DOWNSTREAM_CONNECTION) { - if (connection->stream->transport == FLB_TRANSPORT_UDP || - connection->stream->transport == FLB_TRANSPORT_UNIX_DGRAM) { - address = &connection->raw_remote_host; - } - } - - ret = fd_io_read(connection->fd, address, buf, len); - - if (ret == -1) { - ret = FLB_WOULDBLOCK(); - if (ret) { - /* timeout caused error */ - flb_warn("[net] sync io_read #%i timeout after %i seconds from: %s", - connection->fd, - connection->net->io_timeout, - flb_connection_get_remote_address(connection)); - } - else { - net_io_propagate_critical_error(connection); - } - - return -1; - } - - return ret; -} - -static ssize_t fd_io_read(int fd, struct sockaddr_storage *address, - void *buf, size_t len) -{ - socklen_t address_size; - int ret; - - if (address != NULL) { - address_size = sizeof(struct sockaddr_storage); - ret = recvfrom(fd, buf, len, 0, - (struct sockaddr *) address, - &address_size); - } - else { - ret = recv(fd, buf, len, 0); - } - - if (ret == -1) { - return -1; - } - - return ret; -} - -static FLB_INLINE ssize_t net_io_read_async(struct flb_coro *co, - struct flb_connection *connection, - void *buf, size_t len) -{ - struct mk_event event_backup; - int event_restore_needed; - int ret; - - event_restore_needed = FLB_FALSE; - - net_io_backup_event(connection, &event_backup); - - retry_read: - ret = recv(connection->fd, buf, len, 0); - - if (ret == -1) { - if (FLB_WOULDBLOCK()) { - event_restore_needed = FLB_TRUE; - - ret = mk_event_add(connection->evl, - connection->fd, - FLB_ENGINE_EV_THREAD, - MK_EVENT_READ, - &connection->event); - - connection->event.priority = FLB_ENGINE_PRIORITY_SEND_RECV; - - if (ret == -1) { - /* - * If we failed here there no much that we can do, just - * let the caller we failed - */ - net_io_restore_event(connection, &event_backup); - - return -1; - } - - connection->coroutine = co; - - flb_coro_yield(co, MK_FALSE); - - /* We want this field to hold NULL at all times unless we are explicitly - * waiting to be resumed. - */ - connection->coroutine = NULL; - - goto retry_read; - } - else { - net_io_propagate_critical_error(connection); - } - - ret = -1; - } - else if (ret <= 0) { - ret = -1; - } - - if (event_restore_needed) { - /* If we enter here it means we registered this connection - * in the event loop, in which case we need to unregister it - * and restore the original registration if there was one. - * - * We do it conditionally because in those cases in which - * send succeeds on the first try we don't touch the event - * and it wouldn't make sense to unregister and register for - * the same event. - */ - - net_io_restore_event(connection, &event_backup); - } - - return ret; -} - -/* Write data to fd. For unix socket. */ -int flb_io_fd_write(int fd, const void *data, size_t len, size_t *out_len) -{ - /* TODO: support async mode */ - return fd_io_write(fd, NULL, data, len, out_len); -} - -/* Write data to an upstream connection/server */ -int flb_io_net_write(struct flb_connection *connection, const void *data, - size_t len, size_t *out_len) -{ - int flags; - struct flb_coro *coro; - int ret; - - ret = -1; - coro = flb_coro_get(); - flags = flb_connection_get_flags(connection); - - flb_trace("[io coro=%p] [net_write] trying %zd bytes", coro, len); - - if (connection->tls_session == NULL) { - if (flags & FLB_IO_ASYNC) { - ret = net_io_write_async(coro, connection, data, len, out_len); - } - else { - ret = net_io_write(connection, data, len, out_len); - } - } -#ifdef FLB_HAVE_TLS - else if (flags & FLB_IO_TLS) { - if (flags & FLB_IO_ASYNC) { - ret = flb_tls_net_write_async(coro, connection->tls_session, data, len, out_len); - } - else { - ret = flb_tls_net_write(connection->tls_session, data, len, out_len); - } - } -#endif - - if (ret > 0) { - flb_connection_reset_io_timeout(connection); - } - - flb_trace("[io coro=%p] [net_write] ret=%i total=%lu/%lu", - coro, ret, *out_len, len); - - return ret; -} - -ssize_t flb_io_fd_read(int fd, void *buf, size_t len) -{ - /* TODO: support async mode */ - return fd_io_read(fd, NULL, buf, len); -} - -ssize_t flb_io_net_read(struct flb_connection *connection, void *buf, size_t len) -{ - int ret; - int flags; - struct flb_coro *coro; - - ret = -1; - coro = flb_coro_get(); - - flb_trace("[io coro=%p] [net_read] try up to %zd bytes", coro, len); - - flags = flb_connection_get_flags(connection); - - if (!connection->tls_session) { - if (flags & FLB_IO_ASYNC) { - ret = net_io_read_async(coro, connection, buf, len); - } - else { - ret = net_io_read(connection, buf, len); - } - } -#ifdef FLB_HAVE_TLS - else if (flags & FLB_IO_TLS) { - if (flags & FLB_IO_ASYNC) { - ret = flb_tls_net_read_async(coro, connection->tls_session, buf, len); - } - else { - ret = flb_tls_net_read(connection->tls_session, buf, len); - } - } -#endif - - if (ret > 0) { - flb_connection_reset_io_timeout(connection); - } - - flb_trace("[io coro=%p] [net_read] ret=%i", coro, ret); - - return ret; -} diff --git a/fluent-bit/src/flb_kafka.c b/fluent-bit/src/flb_kafka.c deleted file mode 100644 index a3b69a9c8..000000000 --- a/fluent-bit/src/flb_kafka.c +++ /dev/null @@ -1,216 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2019-2021 The Fluent Bit Authors - * Copyright (C) 2015-2018 Treasure Data Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "fluent-bit/flb_config.h" -#include "fluent-bit/flb_mem.h" -#include "fluent-bit/flb_str.h" -#include "fluent-bit/flb_utils.h" -#include "monkey/mk_core/mk_list.h" -#include <fluent-bit/flb_kafka.h> -#include <fluent-bit/flb_kv.h> - -#include <rdkafka.h> - -rd_kafka_conf_t *flb_kafka_conf_create(struct flb_kafka *kafka, - struct mk_list *properties, - int with_group_id) -{ - struct mk_list *head; - struct flb_kv *kv; - const char *conf; - rd_kafka_conf_t *kafka_cfg; - char errstr[512]; - - kafka_cfg = rd_kafka_conf_new(); - if (!kafka_cfg) { - flb_error("[flb_kafka] Could not initialize kafka config object"); - goto err; - } - - conf = flb_config_prop_get("client_id", properties); - if (!conf) { - conf = "fluent-bit"; - } - if (rd_kafka_conf_set(kafka_cfg, "client.id", conf, - errstr, sizeof(errstr)) != RD_KAFKA_CONF_OK) { - flb_error("[flb_kafka] cannot configure client id: %s", errstr); - } - - if (with_group_id) { - conf = flb_config_prop_get("group_id", properties); - if (!conf) { - conf = "fluent-bit"; - } - if (rd_kafka_conf_set(kafka_cfg, "group.id", conf, - errstr, sizeof(errstr)) != RD_KAFKA_CONF_OK) { - flb_error("[flb_kafka] cannot configure group id: %s", errstr); - } - } - - conf = flb_config_prop_get("brokers", properties); - if (conf) { - if (rd_kafka_conf_set(kafka_cfg, "bootstrap.servers", conf, - errstr, sizeof(errstr)) != RD_KAFKA_CONF_OK) { - flb_error("[flb_kafka] failed to configure brokers: %s", errstr); - goto err; - } - kafka->brokers = flb_strdup(conf); - } - else { - flb_error("config: no brokers defined"); - goto err; - } - - /* Iterate custom rdkafka properties */ - mk_list_foreach(head, properties) { - kv = mk_list_entry(head, struct flb_kv, _head); - if (strncasecmp(kv->key, "rdkafka.", 8) == 0 && - flb_sds_len(kv->key) > 8) { - if (rd_kafka_conf_set(kafka_cfg, kv->key + 8, kv->val, - errstr, sizeof(errstr)) != RD_KAFKA_CONF_OK) { - flb_error("[flb_kafka] cannot configure '%s' property", kv->key + 8); - } - } - } - - return kafka_cfg; - -err: - if (kafka_cfg) { - flb_free(kafka_cfg); - } - return NULL; -} - -static int add_topic_partitions(rd_kafka_topic_partition_list_t *list, - const char *topic_str, - const char *partitions_str) -{ - int ret = -1; - struct mk_list *split; - char *str, *end; - int start, stop; - size_t len; - split = flb_utils_split(partitions_str, '-', -1); - if (!split) { - flb_error("[flb_kafka] Failed to split partitions string"); - goto end; - } - - len = mk_list_size(split); - if (len == 1) { - str = mk_list_entry(split->next, struct flb_split_entry, _head)->value; - start = strtol(str, &end, 10); - if (end == str || *end != '\0') { - flb_error("[flb_kafka] invalid partition \"%s\"", str); - goto end; - } - // single partition - rd_kafka_topic_partition_list_add(list, topic_str, start); - } else if (len == 2) { - str = mk_list_entry(split->next, struct flb_split_entry, _head)->value; - start = strtol(str, &end, 10); - if (end == str || *end != '\0') { - flb_error("[flb_kafka] invalid partition \"%s\"", str); - goto end; - } - str = mk_list_entry(split->next->next, struct flb_split_entry, _head)->value; - stop = strtol(str, &end, 10); - if (end == str || *end != '\0') { - flb_error("[flb_kafka] invalid partition \"%s\"", str); - goto end; - } - rd_kafka_topic_partition_list_add_range(list, topic_str, start, stop); - } else { - flb_error("[flb_kafka] invalid partition range string \"%s\"", partitions_str); - goto end; - } - - ret = 0; - -end: - if (split) { - flb_utils_split_free(split); - } - return ret; -} - -rd_kafka_topic_partition_list_t *flb_kafka_parse_topics(const char *topics_str) -{ - rd_kafka_topic_partition_list_t *ret; - struct mk_list *split = NULL; - struct mk_list *partitions = NULL; - struct mk_list *curr; - struct flb_split_entry *entry; - struct flb_split_entry *topic_entry; - struct flb_split_entry *partitions_entry; - size_t len; - - ret = rd_kafka_topic_partition_list_new(1); - if (!ret) { - flb_error("[flb_kafka] Failed to allocate topic list"); - goto err; - } - - split = flb_utils_split(topics_str, ',', -1); - if (!split) { - flb_error("[flb_kafka] Failed to split topics string"); - goto err; - } - - mk_list_foreach(curr, split) { - entry = mk_list_entry(curr, struct flb_split_entry, _head); - partitions = flb_utils_split(entry->value, ':', -1); - if (!partitions) { - flb_error("[flb_kafka] Failed to split topic string"); - goto err; - } - len = mk_list_size(partitions); - if (len == 1) { - rd_kafka_topic_partition_list_add(ret, entry->value, 0); - } else if (len == 2) { - topic_entry = mk_list_entry( - partitions->next, struct flb_split_entry, _head); - partitions_entry = mk_list_entry( - partitions->next->next, struct flb_split_entry, _head); - if (add_topic_partitions(ret, topic_entry->value, partitions_entry->value)) { - goto err; - } - } else { - flb_error("[flb_kafka] Failed to parse topic/partition string"); - goto err; - } - flb_utils_split_free(partitions); - } - flb_utils_split_free(split); - return ret; - -err: - if (ret) { - rd_kafka_topic_partition_list_destroy(ret); - } - if (split) { - flb_utils_split_free(split); - } - if (partitions) { - flb_utils_split_free(partitions); - } - return NULL; -} diff --git a/fluent-bit/src/flb_kernel.c b/fluent-bit/src/flb_kernel.c deleted file mode 100644 index d29ba922f..000000000 --- a/fluent-bit/src/flb_kernel.c +++ /dev/null @@ -1,161 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <fluent-bit/flb_info.h> -#include <fluent-bit/flb_mem.h> -#include <fluent-bit/flb_kernel.h> -#include <fluent-bit/flb_utils.h> - -#ifdef _WIN32 - -/* Dummy function for Windows environment */ -struct flb_kernel *flb_kernel_info() -{ - int len; - struct flb_kernel *kernel; - - kernel = flb_malloc(sizeof(struct flb_kernel)); - if (!kernel) { - flb_errno(); - return NULL; - } - - kernel->minor = 0; - kernel->major = 0; - kernel->patch = 0; - kernel->s_version.data = flb_malloc(16); - - if (!kernel->s_version.data) { - flb_errno(); - flb_free(kernel); - return NULL; - } - - - len = snprintf(kernel->s_version.data, 16, "0.0.0"); - if (len == -1) { - perror("snprintf"); - return NULL; - } - kernel->s_version.len = len; - kernel->n_version = 0; - - return kernel; -} -#else - -#include <ctype.h> -#include <sys/utsname.h> - -/* - * Routine taken from Monkey Project, Eduardo says it's ok ;) - */ -struct flb_kernel *flb_kernel_info() -{ - - int a, b, c; - int len; - int pos; - char *p, *t; - char *tmp; - struct utsname uts; - struct flb_kernel *kernel; - - if (uname(&uts) == -1) { - flb_errno(); - return NULL; - } - len = strlen(uts.release); - - /* Fixme: this don't support Linux Kernel 10.x.x :P */ - a = (*uts.release - '0'); - - /* Second number */ - p = (uts.release) + 2; - pos = mk_string_char_search(p, '.', len - 2); - if (pos <= 0) { - /* Some Debian systems uses a different notation, e.g: 3.14-2-amd64 */ - pos = mk_string_char_search(p, '-', len - 2); - if (pos <= 0) { - return NULL; - } - } - - tmp = mk_string_copy_substr(p, 0, pos); - if (!tmp) { - return NULL; - } - b = atoi(tmp); - mk_mem_free(tmp); - - /* Last number (it needs filtering) */ - t = p = p + pos + 1; - do { - t++; - } while (isdigit(*t)); - - tmp = mk_string_copy_substr(p, 0, t - p); - if (!tmp) { - return NULL; - } - c = atoi(tmp); - mk_mem_free(tmp); - - kernel = flb_malloc(sizeof(struct flb_kernel)); - if (!kernel) { - flb_errno(); - return NULL; - } - kernel->minor = a; - kernel->major = b; - kernel->patch = c; - kernel->s_version.data = flb_malloc(16); - - if (!kernel->s_version.data) { - flb_errno(); - flb_free(kernel); - return NULL; - } - - len = snprintf(kernel->s_version.data, 16, "%i.%i.%i", a, b, c); - if (len == -1) { - flb_errno(); - flb_free(kernel->s_version.data); - flb_free(kernel); - return NULL; - } - kernel->s_version.len = len; - kernel->n_version = FLB_KERNEL_VERSION(a, b, c); - - return kernel; -} - -#endif - -void flb_kernel_destroy(struct flb_kernel *kernel) -{ - if (kernel == NULL) { - return; - } - - if (kernel->s_version.data) { - flb_free(kernel->s_version.data); - } - flb_free(kernel); -} diff --git a/fluent-bit/src/flb_kv.c b/fluent-bit/src/flb_kv.c deleted file mode 100644 index 9be276685..000000000 --- a/fluent-bit/src/flb_kv.c +++ /dev/null @@ -1,132 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <fluent-bit/flb_info.h> -#include <fluent-bit/flb_kv.h> -#include <fluent-bit/flb_log.h> -#include <fluent-bit/flb_mem.h> - -void flb_kv_init(struct mk_list *list) -{ - mk_list_init(list); -} - -struct flb_kv *flb_kv_item_create_len(struct mk_list *list, - char *k_buf, size_t k_len, - char *v_buf, size_t v_len) -{ - struct flb_kv *kv; - - kv = flb_calloc(1, sizeof(struct flb_kv)); - if (!kv) { - flb_errno(); - return NULL; - } - - kv->key = flb_sds_create_len(k_buf, k_len); - if (!kv->key) { - flb_free(kv); - return NULL; - } - - if (v_len > 0) { - kv->val = flb_sds_create_len(v_buf, v_len); - if (!kv->val) { - flb_sds_destroy(kv->key); - flb_free(kv); - return NULL; - } - } - - mk_list_add(&kv->_head, list); - return kv; -} - -struct flb_kv *flb_kv_item_create(struct mk_list *list, - char *k_buf, char *v_buf) -{ - int k_len; - int v_len = 0; - - if (!k_buf) { - return NULL; - } - k_len = strlen(k_buf); - - if (v_buf) { - v_len = strlen(v_buf); - } - - return flb_kv_item_create_len(list, k_buf, k_len, v_buf, v_len); -} - -void flb_kv_item_destroy(struct flb_kv *kv) -{ - if (kv->key) { - flb_sds_destroy(kv->key); - } - - if (kv->val) { - flb_sds_destroy(kv->val); - } - - mk_list_del(&kv->_head); - flb_free(kv); -} - -void flb_kv_release(struct mk_list *list) -{ - struct mk_list *tmp; - struct mk_list *head; - struct flb_kv *kv; - - mk_list_foreach_safe(head, tmp, list) { - kv = mk_list_entry(head, struct flb_kv, _head); - flb_kv_item_destroy(kv); - } -} - -const char *flb_kv_get_key_value(const char *key, struct mk_list *list) -{ - int len; - struct mk_list *head; - struct flb_kv *kv; - - if (!key) { - return NULL; - } - - len = strlen(key); - if (len == 0) { - return NULL; - } - - mk_list_foreach(head, list) { - kv = mk_list_entry(head, struct flb_kv, _head); - if (flb_sds_len(kv->key) != len) { - continue; - } - - if (strncasecmp(kv->key, key, len) == 0) { - return kv->val; - } - } - - return NULL; -} diff --git a/fluent-bit/src/flb_lib.c b/fluent-bit/src/flb_lib.c deleted file mode 100644 index 882faa547..000000000 --- a/fluent-bit/src/flb_lib.c +++ /dev/null @@ -1,815 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit Demo - * =============== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -#include <fluent-bit/flb_lib.h> -#include <fluent-bit/flb_mem.h> -#include <fluent-bit/flb_pipe.h> -#include <fluent-bit/flb_engine.h> -#include <fluent-bit/flb_input.h> -#include <fluent-bit/flb_output.h> -#include <fluent-bit/flb_filter.h> -#include <fluent-bit/flb_utils.h> -#include <fluent-bit/flb_time.h> -#include <fluent-bit/flb_coro.h> -#include <fluent-bit/flb_callback.h> -#include <fluent-bit/flb_kv.h> -#include <fluent-bit/flb_metrics.h> -#include <fluent-bit/flb_upstream.h> -#include <fluent-bit/flb_downstream.h> -#include <fluent-bit/tls/flb_tls.h> - -#include <signal.h> -#include <stdarg.h> - -#ifdef FLB_HAVE_MTRACE -#include <mcheck.h> -#endif - -#ifdef FLB_HAVE_AWS_ERROR_REPORTER -#include <fluent-bit/aws/flb_aws_error_reporter.h> - -struct flb_aws_error_reporter *error_reporter; -#endif - -/* thread initializator */ -static pthread_once_t flb_lib_once = PTHREAD_ONCE_INIT; - -/* reference to the last 'flb_lib_ctx' context started through flb_start() */ -FLB_TLS_DEFINE(flb_ctx_t, flb_lib_active_context); - -/* reference to the last 'flb_cf' context started through flb_start() */ -FLB_TLS_DEFINE(struct flb_cf, flb_lib_active_cf_context); - -#ifdef FLB_SYSTEM_WINDOWS -static inline int flb_socket_init_win32(void) -{ - WSADATA wsaData; - int err; - - err = WSAStartup(MAKEWORD(2, 2), &wsaData); - if (err != 0) { - fprintf(stderr, "WSAStartup failed with error: %d\n", err); - return err; - } - return 0; -} -#endif - -static inline struct flb_input_instance *in_instance_get(flb_ctx_t *ctx, - int ffd) -{ - struct mk_list *head; - struct flb_input_instance *i_ins; - - mk_list_foreach(head, &ctx->config->inputs) { - i_ins = mk_list_entry(head, struct flb_input_instance, _head); - if (i_ins->id == ffd) { - return i_ins; - } - } - - return NULL; -} - -static inline struct flb_output_instance *out_instance_get(flb_ctx_t *ctx, - int ffd) -{ - struct mk_list *head; - struct flb_output_instance *o_ins; - - mk_list_foreach(head, &ctx->config->outputs) { - o_ins = mk_list_entry(head, struct flb_output_instance, _head); - if (o_ins->id == ffd) { - return o_ins; - } - } - - return NULL; -} - -static inline struct flb_filter_instance *filter_instance_get(flb_ctx_t *ctx, - int ffd) -{ - struct mk_list *head; - struct flb_filter_instance *f_ins; - - mk_list_foreach(head, &ctx->config->filters) { - f_ins = mk_list_entry(head, struct flb_filter_instance, _head); - if (f_ins->id == ffd) { - return f_ins; - } - } - - return NULL; -} - -void flb_init_env() -{ - flb_tls_init(); - flb_coro_init(); - flb_upstream_init(); - flb_downstream_init(); - flb_output_prepare(); - - FLB_TLS_INIT(flb_lib_active_context); - FLB_TLS_INIT(flb_lib_active_cf_context); - - /* libraries */ - cmt_initialize(); -} - -flb_ctx_t *flb_create() -{ - int ret; - flb_ctx_t *ctx; - struct flb_config *config; - -#ifdef FLB_HAVE_MTRACE - /* Start tracing malloc and free */ - mtrace(); -#endif - -#ifdef FLB_SYSTEM_WINDOWS - /* Ensure we initialized Windows Sockets */ - if (flb_socket_init_win32()) { - return NULL; - } -#endif - - ctx = flb_calloc(1, sizeof(flb_ctx_t)); - if (!ctx) { - perror("malloc"); - return NULL; - } - - config = flb_config_init(); - if (!config) { - flb_free(ctx); - return NULL; - } - ctx->config = config; - ctx->status = FLB_LIB_NONE; - - /* - * Initialize our pipe to send data to our worker, used - * by 'lib' input plugin. - */ - ret = flb_pipe_create(config->ch_data); - if (ret == -1) { - perror("pipe"); - flb_config_exit(ctx->config); - flb_free(ctx); - return NULL; - } - - /* Create the event loop to receive notifications */ - ctx->event_loop = mk_event_loop_create(256); - if (!ctx->event_loop) { - flb_config_exit(ctx->config); - flb_free(ctx); - return NULL; - } - config->ch_evl = ctx->event_loop; - - /* Prepare the notification channels */ - ctx->event_channel = flb_calloc(1, sizeof(struct mk_event)); - if (!ctx->event_channel) { - perror("calloc"); - flb_config_exit(ctx->config); - flb_free(ctx); - return NULL; - } - - MK_EVENT_ZERO(ctx->event_channel); - - ret = mk_event_channel_create(config->ch_evl, - &config->ch_notif[0], - &config->ch_notif[1], - ctx->event_channel); - if (ret != 0) { - flb_error("[lib] could not create notification channels"); - flb_stop(ctx); - flb_destroy(ctx); - return NULL; - } - - #ifdef FLB_HAVE_AWS_ERROR_REPORTER - if (is_error_reporting_enabled()) { - error_reporter = flb_aws_error_reporter_create(); - } - #endif - - return ctx; -} - -/* Release resources associated to the library context */ -void flb_destroy(flb_ctx_t *ctx) -{ - if (!ctx) { - return; - } - - if (ctx->event_channel) { - mk_event_del(ctx->event_loop, ctx->event_channel); - flb_free(ctx->event_channel); - } - - /* Remove resources from the event loop */ - mk_event_loop_destroy(ctx->event_loop); - - /* cfg->is_running is set to false when flb_engine_shutdown has been invoked (event loop) */ - if (ctx->config) { - if (ctx->config->is_running == FLB_TRUE) { - flb_engine_shutdown(ctx->config); - } - flb_config_exit(ctx->config); - } - - #ifdef FLB_HAVE_AWS_ERROR_REPORTER - if (is_error_reporting_enabled()) { - flb_aws_error_reporter_destroy(error_reporter); - } - #endif - - flb_free(ctx); - ctx = NULL; - -#ifdef FLB_HAVE_MTRACE - /* Stop tracing malloc and free */ - muntrace(); -#endif -} - -/* Defines a new input instance */ -int flb_input(flb_ctx_t *ctx, const char *input, void *data) -{ - struct flb_input_instance *i_ins; - - i_ins = flb_input_new(ctx->config, input, data, FLB_TRUE); - if (!i_ins) { - return -1; - } - - return i_ins->id; -} - -/* Defines a new output instance */ -int flb_output(flb_ctx_t *ctx, const char *output, struct flb_lib_out_cb *cb) -{ - struct flb_output_instance *o_ins; - - o_ins = flb_output_new(ctx->config, output, cb, FLB_TRUE); - if (!o_ins) { - return -1; - } - - return o_ins->id; -} - -/* Defines a new filter instance */ -int flb_filter(flb_ctx_t *ctx, const char *filter, void *data) -{ - struct flb_filter_instance *f_ins; - - f_ins = flb_filter_new(ctx->config, filter, data); - if (!f_ins) { - return -1; - } - - return f_ins->id; -} - -/* Set an input interface property */ -int flb_input_set(flb_ctx_t *ctx, int ffd, ...) -{ - int ret; - char *key; - char *value; - va_list va; - struct flb_input_instance *i_ins; - - i_ins = in_instance_get(ctx, ffd); - if (!i_ins) { - return -1; - } - - va_start(va, ffd); - while ((key = va_arg(va, char *))) { - value = va_arg(va, char *); - if (!value) { - /* Wrong parameter */ - va_end(va); - return -1; - } - ret = flb_input_set_property(i_ins, key, value); - if (ret != 0) { - va_end(va); - return -1; - } - } - - va_end(va); - return 0; -} - -static inline int flb_config_map_property_check(char *plugin_name, struct mk_list *config_map, char *key, char *val) -{ - struct flb_kv *kv; - struct mk_list properties; - int r; - - mk_list_init(&properties); - - kv = flb_kv_item_create(&properties, (char *) key, (char *) val); - if (!kv) { - return FLB_LIB_ERROR; - } - - r = flb_config_map_properties_check(plugin_name, &properties, config_map); - flb_kv_item_destroy(kv); - return r; -} - -/* Check if a given k, v is a valid config directive for the given output plugin */ -int flb_output_property_check(flb_ctx_t *ctx, int ffd, char *key, char *val) -{ - struct flb_output_instance *o_ins; - struct mk_list *config_map; - struct flb_output_plugin *p; - int r; - - o_ins = out_instance_get(ctx, ffd); - if (!o_ins) { - return FLB_LIB_ERROR; - } - - p = o_ins->p; - if (!p->config_map) { - return FLB_LIB_NO_CONFIG_MAP; - } - - config_map = flb_config_map_create(ctx->config, p->config_map); - if (!config_map) { - return FLB_LIB_ERROR; - } - - r = flb_config_map_property_check(p->name, config_map, key, val); - flb_config_map_destroy(config_map); - return r; -} - -/* Check if a given k, v is a valid config directive for the given input plugin */ -int flb_input_property_check(flb_ctx_t *ctx, int ffd, char *key, char *val) -{ - struct flb_input_instance *i_ins; - struct flb_input_plugin *p; - struct mk_list *config_map; - int r; - - i_ins = in_instance_get(ctx, ffd); - if (!i_ins) { - return FLB_LIB_ERROR; - } - - p = i_ins->p; - if (!p->config_map) { - return FLB_LIB_NO_CONFIG_MAP; - } - - config_map = flb_config_map_create(ctx->config, p->config_map); - if (!config_map) { - return FLB_LIB_ERROR; - } - - r = flb_config_map_property_check(p->name, config_map, key, val); - flb_config_map_destroy(config_map); - return r; -} - -/* Check if a given k, v is a valid config directive for the given filter plugin */ -int flb_filter_property_check(flb_ctx_t *ctx, int ffd, char *key, char *val) -{ - struct flb_filter_instance *f_ins; - struct flb_filter_plugin *p; - struct mk_list *config_map; - int r; - - f_ins = filter_instance_get(ctx, ffd); - if (!f_ins) { - return FLB_LIB_ERROR; - } - - p = f_ins->p; - if (!p->config_map) { - return FLB_LIB_NO_CONFIG_MAP; - } - - config_map = flb_config_map_create(ctx->config, p->config_map); - if (!config_map) { - return FLB_LIB_ERROR; - } - - r = flb_config_map_property_check(p->name, config_map, key, val); - flb_config_map_destroy(config_map); - return r; -} - -/* Set an output interface property */ -int flb_output_set(flb_ctx_t *ctx, int ffd, ...) -{ - int ret; - char *key; - char *value; - va_list va; - struct flb_output_instance *o_ins; - - o_ins = out_instance_get(ctx, ffd); - if (!o_ins) { - return -1; - } - - va_start(va, ffd); - while ((key = va_arg(va, char *))) { - value = va_arg(va, char *); - if (!value) { - /* Wrong parameter */ - va_end(va); - return -1; - } - - ret = flb_output_set_property(o_ins, key, value); - if (ret != 0) { - va_end(va); - return -1; - } - } - - va_end(va); - return 0; -} - -int flb_output_set_callback(flb_ctx_t *ctx, int ffd, char *name, - void (*cb)(char *, void *, void *)) -{ - struct flb_output_instance *o_ins; - - o_ins = out_instance_get(ctx, ffd); - if (!o_ins) { - return -1; - } - - return flb_callback_set(o_ins->callback, name, cb); -} - -int flb_output_set_test(flb_ctx_t *ctx, int ffd, char *test_name, - void (*out_callback) (void *, int, int, void *, size_t, void *), - void *out_callback_data, - void *test_ctx) -{ - struct flb_output_instance *o_ins; - - o_ins = out_instance_get(ctx, ffd); - if (!o_ins) { - return -1; - } - - /* - * Enabling a test, set the output instance in 'test' mode, so no real - * flush callback is invoked, only the desired implemented test. - */ - - /* Formatter test */ - if (strcmp(test_name, "formatter") == 0) { - o_ins->test_mode = FLB_TRUE; - o_ins->test_formatter.rt_ctx = ctx; - o_ins->test_formatter.rt_ffd = ffd; - o_ins->test_formatter.rt_out_callback = out_callback; - o_ins->test_formatter.rt_data = out_callback_data; - o_ins->test_formatter.flush_ctx = test_ctx; - } - else { - return -1; - } - - return 0; -} - -/* Set an filter interface property */ -int flb_filter_set(flb_ctx_t *ctx, int ffd, ...) -{ - int ret; - char *key; - char *value; - va_list va; - struct flb_filter_instance *f_ins; - - f_ins = filter_instance_get(ctx, ffd); - if (!f_ins) { - return -1; - } - - va_start(va, ffd); - while ((key = va_arg(va, char *))) { - value = va_arg(va, char *); - if (!value) { - /* Wrong parameter */ - va_end(va); - return -1; - } - - ret = flb_filter_set_property(f_ins, key, value); - if (ret != 0) { - va_end(va); - return -1; - } - } - - va_end(va); - return 0; -} - -/* Set a service property */ -int flb_service_set(flb_ctx_t *ctx, ...) -{ - int ret; - char *key; - char *value; - va_list va; - - va_start(va, ctx); - - while ((key = va_arg(va, char *))) { - value = va_arg(va, char *); - if (!value) { - /* Wrong parameter */ - va_end(va); - return -1; - } - - ret = flb_config_set_property(ctx->config, key, value); - if (ret != 0) { - va_end(va); - return -1; - } - } - - va_end(va); - return 0; -} - -/* Load a configuration file that may be used by the input or output plugin */ -int flb_lib_config_file(struct flb_lib_ctx *ctx, const char *path) -{ - if (access(path, R_OK) != 0) { - perror("access"); - return -1; - } - - ctx->config->file = mk_rconf_open(path); - if (!ctx->config->file) { - fprintf(stderr, "Error reading configuration file: %s\n", path); - return -1; - } - - return 0; -} - -/* This is a wrapper to release a buffer which comes from out_lib_flush() */ -int flb_lib_free(void* data) -{ - if (data == NULL) { - return -1; - } - flb_free(data); - return 0; -} - - -/* Push some data into the Engine */ -int flb_lib_push(flb_ctx_t *ctx, int ffd, const void *data, size_t len) -{ - int ret; - struct flb_input_instance *i_ins; - - if (ctx->status == FLB_LIB_NONE || ctx->status == FLB_LIB_ERROR) { - flb_error("[lib] cannot push data, engine is not running"); - return -1; - } - - i_ins = in_instance_get(ctx, ffd); - if (!i_ins) { - return -1; - } - - ret = flb_pipe_w(i_ins->channel[1], data, len); - if (ret == -1) { - flb_errno(); - return -1; - } - return ret; -} - -static void flb_lib_worker(void *data) -{ - int ret; - flb_ctx_t *ctx = data; - struct flb_config *config; - - config = ctx->config; - flb_context_set(ctx); - mk_utils_worker_rename("flb-pipeline"); - ret = flb_engine_start(config); - if (ret == -1) { - flb_engine_failed(config); - flb_engine_shutdown(config); - } - config->exit_status_code = ret; - ctx->status = FLB_LIB_NONE; -} - -/* Return the current time to be used by lib callers */ -double flb_time_now() -{ - struct flb_time t; - - flb_time_get(&t); - return flb_time_to_double(&t); -} - -int static do_start(flb_ctx_t *ctx) -{ - int fd; - int bytes; - int ret; - uint64_t val; - pthread_t tid; - struct mk_event *event; - struct flb_config *config; - - pthread_once(&flb_lib_once, flb_init_env); - - flb_debug("[lib] context set: %p", ctx); - - /* set context as the last active one */ - - /* spawn worker thread */ - config = ctx->config; - ret = mk_utils_worker_spawn(flb_lib_worker, ctx, &tid); - if (ret == -1) { - return -1; - } - config->worker = tid; - - /* Wait for the started signal so we can return to the caller */ - mk_event_wait(config->ch_evl); - mk_event_foreach(event, config->ch_evl) { - fd = event->fd; - bytes = flb_pipe_r(fd, &val, sizeof(uint64_t)); - if (bytes <= 0) { -#if defined(FLB_SYSTEM_MACOS) - pthread_cancel(tid); -#endif - pthread_join(tid, NULL); - ctx->status = FLB_LIB_ERROR; - return -1; - } - - if (val == FLB_ENGINE_STARTED) { - flb_debug("[lib] backend started"); - ctx->status = FLB_LIB_OK; - break; - } - else if (val == FLB_ENGINE_FAILED) { - flb_error("[lib] backend failed"); -#if defined(FLB_SYSTEM_MACOS) - pthread_cancel(tid); -#endif - pthread_join(tid, NULL); - ctx->status = FLB_LIB_ERROR; - return -1; - } - else { - flb_error("[lib] other error"); - } - } - - return 0; -} - -/* Start the engine */ -int flb_start(flb_ctx_t *ctx) -{ - int ret; - - ret = do_start(ctx); - if (ret == 0) { - /* set context as the last active one */ - flb_context_set(ctx); - } - - return ret; -} - -/* Start the engine without setting the global context */ -int flb_start_trace(flb_ctx_t *ctx) -{ - return do_start(ctx); -} - -int flb_loop(flb_ctx_t *ctx) -{ - while (ctx->status == FLB_LIB_OK) { - sleep(1); - } - return 0; -} - -/* Stop the engine */ -int flb_stop(flb_ctx_t *ctx) -{ - int ret; - pthread_t tid; - - flb_debug("[lib] ctx stop address: %p, config context=%p\n", ctx, ctx->config); - - tid = ctx->config->worker; - - if (ctx->status == FLB_LIB_NONE || ctx->status == FLB_LIB_ERROR) { - /* - * There is a chance the worker thread is still active while - * the service exited for some reason (plugin action). Always - * wait and double check that the child thread is not running. - */ -#if defined(FLB_SYSTEM_MACOS) - pthread_cancel(tid); -#endif - pthread_join(tid, NULL); - return 0; - } - - if (!ctx->config) { - return 0; - } - - if (ctx->config->file) { - mk_rconf_free(ctx->config->file); - } - - flb_debug("[lib] sending STOP signal to the engine"); - - flb_engine_exit(ctx->config); -#if defined(FLB_SYSTEM_MACOS) - pthread_cancel(tid); -#endif - ret = pthread_join(tid, NULL); - if (ret != 0) { - flb_errno(); - } - flb_debug("[lib] Fluent Bit engine stopped"); - - return ret; -} - - -void flb_context_set(flb_ctx_t *ctx) -{ - FLB_TLS_SET(flb_lib_active_context, ctx); -} - -flb_ctx_t *flb_context_get() -{ - flb_ctx_t *ctx; - - ctx = FLB_TLS_GET(flb_lib_active_context); - return ctx; -} - -void flb_cf_context_set(struct flb_cf *cf) -{ - FLB_TLS_SET(flb_lib_active_cf_context, cf); -} - -struct flb_cf *flb_cf_context_get() -{ - struct flb_cf *cf; - - cf = FLB_TLS_GET(flb_lib_active_cf_context); - return cf; -} diff --git a/fluent-bit/src/flb_log.c b/fluent-bit/src/flb_log.c deleted file mode 100644 index d004af8af..000000000 --- a/fluent-bit/src/flb_log.c +++ /dev/null @@ -1,696 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <stdio.h> -#include <stdlib.h> -#include <signal.h> -#include <stdarg.h> -#include <inttypes.h> -#include <sys/types.h> -#include <sys/stat.h> -#include <fcntl.h> - -#include <monkey/mk_core.h> -#include <fluent-bit/flb_log.h> -#include <fluent-bit/flb_pipe.h> -#include <fluent-bit/flb_config.h> -#include <fluent-bit/flb_worker.h> -#include <fluent-bit/flb_mem.h> - -#ifdef FLB_HAVE_AWS_ERROR_REPORTER -#include <fluent-bit/aws/flb_aws_error_reporter.h> - -extern struct flb_aws_error_reporter *error_reporter; -#endif - -FLB_TLS_DEFINE(struct flb_log, flb_log_ctx) - -/* Simple structure to dispatch messages to the log collector */ -struct log_message { - size_t size; - char msg[4096 - sizeof(size_t)]; -}; - -static inline int consume_byte(flb_pipefd_t fd) -{ - int ret; - uint64_t val; - - /* We need to consume the byte */ - ret = flb_pipe_r(fd, &val, sizeof(val)); - if (ret <= 0) { - flb_errno(); - return -1; - } - - return 0; -} - -static inline int log_push(struct log_message *msg, struct flb_log *log) -{ - int fd; - int ret = -1; - - if (log->type == FLB_LOG_STDERR) { - return write(STDERR_FILENO, msg->msg, msg->size); - } - else if (log->type == FLB_LOG_FILE) { - fd = open(log->out, O_CREAT | O_WRONLY | O_APPEND, 0666); - if (fd == -1) { - fprintf(stderr, "[log] error opening log file %s. Using stderr.\n", - log->out); - return write(STDERR_FILENO, msg->msg, msg->size); - } - ret = write(fd, msg->msg, msg->size); - close(fd); - } - - return ret; -} - -static inline int log_read(flb_pipefd_t fd, struct flb_log *log) -{ - int bytes; - struct log_message msg; - - /* - * Since write operations to the pipe are always atomic 'if' they are - * under the PIPE_BUF limit (4KB on Linux) and our messages are always 1KB, - * we can trust we will always get a full message on each read(2). - */ - bytes = flb_pipe_read_all(fd, &msg, sizeof(struct log_message)); - if (bytes <= 0) { - flb_errno(); - return -1; - } - if (msg.size > sizeof(msg.msg)) { - fprintf(stderr, "[log] message too long: %zi > %zi", - msg.size, sizeof(msg.msg)); - return -1; - } - log_push(&msg, log); - - return bytes; -} - -/* Central collector of messages */ -static void log_worker_collector(void *data) -{ - int run = FLB_TRUE; - struct mk_event *event = NULL; - struct flb_log *log = data; - - FLB_TLS_INIT(flb_log_ctx); - FLB_TLS_SET(flb_log_ctx, log); - - mk_utils_worker_rename("flb-logger"); - - /* Signal the caller */ - pthread_mutex_lock(&log->pth_mutex); - log->pth_init = FLB_TRUE; - pthread_cond_signal(&log->pth_cond); - pthread_mutex_unlock(&log->pth_mutex); - - while (run) { - mk_event_wait(log->evl); - mk_event_foreach(event, log->evl) { - if (event->type == FLB_LOG_EVENT) { - log_read(event->fd, log); - } - else if (event->type == FLB_LOG_MNG) { - consume_byte(event->fd); - run = FLB_FALSE; - } - } - } - - pthread_exit(NULL); -} - -struct flb_log_cache *flb_log_cache_create(int timeout_seconds, int size) -{ - int i; - struct flb_log_cache *cache; - struct flb_log_cache_entry *entry; - - if (size <= 0) { - return NULL; - } - - cache = flb_calloc(1, sizeof(struct flb_log_cache)); - if (!cache) { - flb_errno(); - return NULL; - } - cache->timeout = timeout_seconds; - mk_list_init(&cache->entries); - - for (i = 0; i < size; i++) { - entry = flb_calloc(1, sizeof(struct flb_log_cache_entry)); - if (!entry) { - flb_errno(); - flb_log_cache_destroy(cache); - return NULL; - } - - entry->buf = flb_sds_create_size(FLB_LOG_CACHE_TEXT_BUF_SIZE); - if (!entry->buf) { - flb_errno(); - flb_log_cache_destroy(cache); - } - entry->timestamp = 0; /* unset for now */ - mk_list_add(&entry->_head, &cache->entries); - } - - return cache; -} - -void flb_log_cache_destroy(struct flb_log_cache *cache) -{ - struct mk_list *tmp; - struct mk_list *head; - struct flb_log_cache_entry *entry; - - if (!cache) { - return; - } - - mk_list_foreach_safe(head, tmp, &cache->entries) { - entry = mk_list_entry(head, struct flb_log_cache_entry, _head); - flb_sds_destroy(entry->buf); - mk_list_del(&entry->_head); - flb_free(entry); - } - flb_free(cache); -} - -struct flb_log_cache_entry *flb_log_cache_exists(struct flb_log_cache *cache, char *msg_buf, size_t msg_size) -{ - size_t size; - struct mk_list *head; - struct flb_log_cache_entry *entry; - - if (msg_size <= 1) { - return NULL; - } - - /* number of bytes to compare */ - size = msg_size / 2; - - mk_list_foreach(head, &cache->entries) { - entry = mk_list_entry(head, struct flb_log_cache_entry, _head); - if (entry->timestamp == 0) { - continue; - } - - if (flb_sds_len(entry->buf) < size) { - continue; - } - - if (strncmp(entry->buf, msg_buf, size) == 0) { - return entry; - } - } - - return NULL; -} - - -/* returns an unused entry or the oldest one */ -struct flb_log_cache_entry *flb_log_cache_get_target(struct flb_log_cache *cache, uint64_t ts) -{ - struct mk_list *head; - struct flb_log_cache_entry *entry; - struct flb_log_cache_entry *target = NULL; - - mk_list_foreach(head, &cache->entries) { - entry = mk_list_entry(head, struct flb_log_cache_entry, _head); - - /* unused entry */ - if (entry->timestamp == 0) { - return entry; - } - - /* expired entry */ - if (entry->timestamp + cache->timeout < ts) { - return entry; - } - - /* keep a reference to the oldest entry to sacrifice it */ - if (!target || entry->timestamp < target->timestamp) { - target = entry; - } - } - - return target; -} - -/* - * should the incoming message to be suppressed because already one similar exists in - * the cache ? - * - * if no similar message exists, then the incoming message is added to the cache. - */ -int flb_log_cache_check_suppress(struct flb_log_cache *cache, char *msg_buf, size_t msg_size) -{ - uint64_t now = 0; - struct flb_log_cache_entry *entry; - - now = time(NULL); - entry = flb_log_cache_exists(cache, msg_buf, msg_size); - - /* if no similar message found, add the incoming message to the cache */ - if (!entry) { - /* look for an unused entry or the oldest one */ - entry = flb_log_cache_get_target(cache, now); - - /* if no target entry is available just return, do not suppress the message */ - if (!entry) { - return FLB_FALSE; - } - - /* add the message to the cache */ - flb_sds_len_set(entry->buf, 0); - entry->buf = flb_sds_copy(entry->buf, msg_buf, msg_size); - entry->timestamp = now; - return FLB_FALSE; - } - else { - if (entry->timestamp + cache->timeout > now) { - return FLB_TRUE; - } - else { - entry->timestamp = now; - return FLB_FALSE; - } - } - return FLB_TRUE; -} - -int flb_log_worker_init(struct flb_worker *worker) -{ - int ret; - struct flb_config *config = worker->config; - struct flb_log *log = config->log; - struct flb_log_cache *cache; - - /* Pipe to communicate Thread with worker log-collector */ - ret = flb_pipe_create(worker->log); - if (ret == -1) { - flb_errno(); - return -1; - } - - /* Register the read-end of the pipe (log[0]) into the event loop */ - ret = mk_event_add(log->evl, worker->log[0], - FLB_LOG_EVENT, MK_EVENT_READ, &worker->event); - if (ret == -1) { - close(worker->log[0]); - close(worker->log[1]); - return -1; - } - - /* Log cache to reduce noise */ - cache = flb_log_cache_create(10, FLB_LOG_CACHE_ENTRIES); - if (!cache) { - close(worker->log[0]); - close(worker->log[1]); - return -1; - } - worker->log_cache = cache; - return 0; -} - -int flb_log_set_level(struct flb_config *config, int level) -{ - config->log->level = level; - return 0; -} - -int flb_log_get_level_str(char *str) -{ - if (strcasecmp(str, "off") == 0) { - return FLB_LOG_OFF; - } - else if (strcasecmp(str, "error") == 0) { - return FLB_LOG_ERROR; - } - else if (strcasecmp(str, "warn") == 0 || strcasecmp(str, "warning") == 0) { - return FLB_LOG_WARN; - } - else if (strcasecmp(str, "info") == 0) { - return FLB_LOG_INFO; - } - else if (strcasecmp(str, "debug") == 0) { - return FLB_LOG_DEBUG; - } - else if (strcasecmp(str, "trace") == 0) { - return FLB_LOG_TRACE; - } - - return -1; -} - -int flb_log_set_file(struct flb_config *config, char *out) -{ - struct flb_log *log = config->log; - - if (out) { - log->type = FLB_LOG_FILE; - log->out = out; - } - else { - log->type = FLB_LOG_STDERR; - log->out = NULL; - } - - return 0; -} - -struct flb_log *flb_log_create(struct flb_config *config, int type, - int level, char *out) -{ - int ret; - struct flb_log *log; - struct flb_worker *worker; - struct mk_event_loop *evl; - - log = flb_calloc(1, sizeof(struct flb_log)); - if (!log) { - flb_errno(); - return NULL; - } - config->log = log; - - /* Create event loop to be used by the collector worker */ - evl = mk_event_loop_create(32); - if (!evl) { - fprintf(stderr, "[log] could not create event loop\n"); - flb_free(log); - config->log = NULL; - return NULL; - } - - /* Prepare logging context */ - log->type = type; - log->level = level; - log->out = out; - log->evl = evl; - log->tid = 0; - - ret = flb_pipe_create(log->ch_mng); - if (ret == -1) { - fprintf(stderr, "[log] could not create pipe(2)"); - mk_event_loop_destroy(log->evl); - flb_free(log); - config->log = NULL; - return NULL; - } - MK_EVENT_ZERO(&log->event); - - /* Register channel manager into the event loop */ - ret = mk_event_add(log->evl, log->ch_mng[0], - FLB_LOG_MNG, MK_EVENT_READ, &log->event); - if (ret == -1) { - fprintf(stderr, "[log] could not register event\n"); - mk_event_loop_destroy(log->evl); - flb_free(log); - config->log = NULL; - return NULL; - } - - /* - * Since the main process/thread might want to write log messages, - * it will need a 'worker-like' context, here we create a fake worker - * context just for messaging purposes. - */ - worker = flb_worker_context_create(NULL, NULL, config); - if (!worker) { - flb_errno(); - mk_event_loop_destroy(log->evl); - flb_free(log); - config->log = NULL; - } - - /* Set the worker context global */ - FLB_TLS_INIT(flb_worker_ctx); - FLB_TLS_SET(flb_worker_ctx, worker); - - ret = flb_log_worker_init(worker); - if (ret == -1) { - flb_errno(); - mk_event_loop_destroy(log->evl); - flb_free(log); - config->log = NULL; - flb_free(worker); - return NULL; - } - log->worker = worker; - - /* - * This lock is used for the 'pth_cond' conditional. Once the worker - * thread is ready will signal the condition. - */ - pthread_mutex_init(&log->pth_mutex, NULL); - pthread_cond_init(&log->pth_cond, NULL); - log->pth_init = FLB_FALSE; - - pthread_mutex_lock(&log->pth_mutex); - - ret = flb_worker_create(log_worker_collector, log, &log->tid, config); - if (ret == -1) { - pthread_mutex_unlock(&log->pth_mutex); - mk_event_loop_destroy(log->evl); - flb_free(log->worker); - flb_free(log); - config->log = NULL; - return NULL; - } - - /* Block until the child thread is ready */ - while (!log->pth_init) { - pthread_cond_wait(&log->pth_cond, &log->pth_mutex); - } - pthread_mutex_unlock(&log->pth_mutex); - - return log; -} - -int flb_log_construct(struct log_message *msg, int *ret_len, - int type, const char *file, int line, const char *fmt, va_list *args) -{ - int body_size; - int ret; - int len; - int total; - time_t now; - const char *header_color = NULL; - const char *header_title = NULL; - const char *bold_color = ANSI_BOLD; - const char *reset_color = ANSI_RESET; - struct tm result; - struct tm *current; - - switch (type) { - case FLB_LOG_HELP: - header_title = "help"; - header_color = ANSI_CYAN; - break; - case FLB_LOG_INFO: - header_title = "info"; - header_color = ANSI_GREEN; - break; - case FLB_LOG_WARN: - header_title = "warn"; - header_color = ANSI_YELLOW; - break; - case FLB_LOG_ERROR: - header_title = "error"; - header_color = ANSI_RED; - break; - case FLB_LOG_DEBUG: - header_title = "debug"; - header_color = ANSI_YELLOW; - break; - case FLB_LOG_IDEBUG: - header_title = "debug"; - header_color = ANSI_CYAN; - break; - case FLB_LOG_TRACE: - header_title = "trace"; - header_color = ANSI_BLUE; - break; - } - - #ifdef FLB_LOG_NO_CONTROL_CHARS - header_color = ""; - bold_color = ""; - reset_color = ""; - #else - /* Only print colors to a terminal */ - if (!isatty(STDOUT_FILENO)) { - header_color = ""; - bold_color = ""; - reset_color = ""; - } - #endif // FLB_LOG_NO_CONTROL_CHARS - - now = time(NULL); - current = localtime_r(&now, &result); - - if (current == NULL) { - return -1; - } - - len = snprintf(msg->msg, sizeof(msg->msg) - 1, - "%s[%s%i/%02i/%02i %02i:%02i:%02i%s]%s [%s%5s%s] ", - /* time */ /* type */ - - /* time variables */ - bold_color, reset_color, - current->tm_year + 1900, - current->tm_mon + 1, - current->tm_mday, - current->tm_hour, - current->tm_min, - current->tm_sec, - bold_color, reset_color, - - /* type format */ - header_color, header_title, reset_color); - - body_size = (sizeof(msg->msg) - 2) - len; - total = vsnprintf(msg->msg + len, - body_size, - fmt, *args); - if (total < 0) { - return -1; - } - ret = total; /* ret means a buffer size need to save log body */ - - total = strlen(msg->msg + len) + len; - msg->msg[total++] = '\n'; - msg->msg[total] = '\0'; - msg->size = total; - - *ret_len = len; - - if (ret >= body_size) { - /* log is truncated */ - return ret - body_size; - } - - return 0; -} - -/** - * flb_log_is_truncated tries to construct log and returns that the log is truncated. - * - * @param same as flb_log_print - * @return 0: log is not truncated. -1: some error occurs. - * positive number: truncated log size. - * - */ -int flb_log_is_truncated(int type, const char *file, int line, const char *fmt, ...) -{ - int ret; - int len; - struct log_message msg = {0}; - va_list args; - - va_start(args, fmt); - ret = flb_log_construct(&msg, &len, type, file, line, fmt, &args); - va_end(args); - - if (ret < 0) { - return -1; - } - - return ret; -} - -void flb_log_print(int type, const char *file, int line, const char *fmt, ...) -{ - int n; - int len; - int ret; - struct log_message msg = {0}; - va_list args; - - struct flb_worker *w; - - va_start(args, fmt); - ret = flb_log_construct(&msg, &len, type, file, line, fmt, &args); - va_end(args); - - if (ret < 0) { - return; - } - - w = flb_worker_get(); - if (w) { - n = flb_pipe_write_all(w->log[1], &msg, sizeof(msg)); - if (n == -1) { - fprintf(stderr, "%s", (char *) msg.msg); - perror("write"); - } - } - else { - fprintf(stderr, "%s", (char *) msg.msg); - } - - #ifdef FLB_HAVE_AWS_ERROR_REPORTER - if (is_error_reporting_enabled()) { - if (type == FLB_LOG_ERROR) { - flb_aws_error_reporter_write(error_reporter, msg.msg + len); - } - - flb_aws_error_reporter_clean(error_reporter); - } - #endif -} - -int flb_errno_print(int errnum, const char *file, int line) -{ - char buf[256]; - - strerror_r(errnum, buf, sizeof(buf) - 1); - flb_error("[%s:%i errno=%i] %s", file, line, errnum, buf); - return 0; -} - -int flb_log_destroy(struct flb_log *log, struct flb_config *config) -{ - uint64_t val = FLB_TRUE; - - /* Signal the child worker, stop working */ - flb_pipe_w(log->ch_mng[1], &val, sizeof(val)); - pthread_join(log->tid, NULL); - - /* Release resources */ - mk_event_loop_destroy(log->evl); - flb_pipe_destroy(log->ch_mng); - if (log->worker->log_cache) { - flb_log_cache_destroy(log->worker->log_cache); - } - flb_free(log->worker); - flb_free(log); - - return 0; -} diff --git a/fluent-bit/src/flb_log_event_decoder.c b/fluent-bit/src/flb_log_event_decoder.c deleted file mode 100644 index 00a778e3d..000000000 --- a/fluent-bit/src/flb_log_event_decoder.c +++ /dev/null @@ -1,383 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <fluent-bit/flb_log_event_decoder.h> -#include <fluent-bit/flb_byteswap.h> - -static int create_empty_map(struct flb_log_event_decoder *context) { - msgpack_packer packer; - msgpack_sbuffer buffer; - int result; - size_t offset; - - result = FLB_EVENT_DECODER_SUCCESS; - - context->empty_map = NULL; - - msgpack_sbuffer_init(&buffer); - msgpack_packer_init(&packer, &buffer, msgpack_sbuffer_write); - - result = msgpack_pack_map(&packer, 0); - - if (result != 0) { - result = FLB_EVENT_DECODER_ERROR_INITIALIZATION_FAILURE; - } - else { - offset = 0; - - msgpack_unpacked_init(&context->unpacked_empty_map); - - result = msgpack_unpack_next(&context->unpacked_empty_map, - buffer.data, - buffer.size, - &offset); - - if (result != MSGPACK_UNPACK_SUCCESS) { - result = FLB_EVENT_DECODER_ERROR_INITIALIZATION_FAILURE; - } - else { - context->empty_map = &context->unpacked_empty_map.data; - - result = FLB_EVENT_DECODER_SUCCESS; - } - } - - msgpack_sbuffer_destroy(&buffer); - - return result; -} - -void flb_log_event_decoder_reset(struct flb_log_event_decoder *context, - char *input_buffer, - size_t input_length) -{ - context->offset = 0; - context->buffer = input_buffer; - context->length = input_length; - context->last_result = FLB_EVENT_DECODER_ERROR_INSUFFICIENT_DATA; - - msgpack_unpacked_destroy(&context->unpacked_event); - msgpack_unpacked_init(&context->unpacked_event); - -} - -int flb_log_event_decoder_init(struct flb_log_event_decoder *context, - char *input_buffer, - size_t input_length) -{ - if (context == NULL) { - return FLB_EVENT_DECODER_ERROR_INVALID_CONTEXT; - } - - memset(context, 0, sizeof(struct flb_log_event_decoder)); - - context->dynamically_allocated = FLB_FALSE; - context->initialized = FLB_TRUE; - - flb_log_event_decoder_reset(context, input_buffer, input_length); - - return create_empty_map(context); -} - -struct flb_log_event_decoder *flb_log_event_decoder_create( - char *input_buffer, - size_t input_length) -{ - struct flb_log_event_decoder *context; - int result; - - context = (struct flb_log_event_decoder *) \ - flb_calloc(1, sizeof(struct flb_log_event_decoder)); - - result = flb_log_event_decoder_init(context, - input_buffer, - input_length); - - if (context != NULL) { - context->dynamically_allocated = FLB_TRUE; - - if (result != FLB_EVENT_DECODER_SUCCESS) { - flb_log_event_decoder_destroy(context); - - context = NULL; - } - } - - return context; -} - -void flb_log_event_decoder_destroy(struct flb_log_event_decoder *context) -{ - int dynamically_allocated; - - if (context != NULL) { - if (context->initialized) { - msgpack_unpacked_destroy(&context->unpacked_empty_map); - msgpack_unpacked_destroy(&context->unpacked_event); - } - - dynamically_allocated = context->dynamically_allocated; - - memset(context, 0, sizeof(struct flb_log_event_decoder)); - - /* This might look silly and with most of the codebase including - * this module as context it might be but just in case we choose - * to stray away from the assumption of FLB_FALSE being zero and - * FLB_TRUE being one in favor of explicitly comparing variables to - * the the constants I will leave this here. - */ - context->initialized = FLB_FALSE; - - if (dynamically_allocated) { - flb_free(context); - } - } -} - -int flb_log_event_decoder_decode_timestamp(msgpack_object *input, - struct flb_time *output) -{ - flb_time_zero(output); - - if (input->type == MSGPACK_OBJECT_POSITIVE_INTEGER) { - output->tm.tv_sec = input->via.u64; - } - else if(input->type == MSGPACK_OBJECT_FLOAT) { - output->tm.tv_sec = input->via.f64; - output->tm.tv_nsec = ((input->via.f64 - output->tm.tv_sec) * 1000000000); - } - else if(input->type == MSGPACK_OBJECT_EXT) { - if (input->via.ext.type != 0 || input->via.ext.size != 8) { - return FLB_EVENT_DECODER_ERROR_WRONG_TIMESTAMP_TYPE; - } - - output->tm.tv_sec = FLB_BSWAP_32(*((uint32_t *) &input->via.ext.ptr[0])); - output->tm.tv_nsec = FLB_BSWAP_32(*((uint32_t *) &input->via.ext.ptr[4])); - } - else { - return FLB_EVENT_DECODER_ERROR_WRONG_TIMESTAMP_TYPE; - } - - return FLB_EVENT_DECODER_SUCCESS; -} - -int flb_event_decoder_decode_object(struct flb_log_event_decoder *context, - struct flb_log_event *event, - msgpack_object *input) -{ - msgpack_object *timestamp; - msgpack_object *metadata; - int result; - int format; - msgpack_object *header; - msgpack_object *body; - msgpack_object *root; - - memset(event, 0, sizeof(struct flb_log_event)); - - /* Ensure that the root element is a 2 element array*/ - root = input; - - if (root->type != MSGPACK_OBJECT_ARRAY) { - return FLB_EVENT_DECODER_ERROR_WRONG_ROOT_TYPE; - } - - if (root->via.array.size != \ - FLB_LOG_EVENT_EXPECTED_ROOT_ELEMENT_COUNT) { - return FLB_EVENT_DECODER_ERROR_WRONG_ROOT_SIZE; - } - - header = &root->via.array.ptr[0]; - - /* Determine if the first element is the header or - * a legacy timestamp (int, float or ext). - */ - if (header->type == MSGPACK_OBJECT_ARRAY) { - if (header->via.array.size != \ - FLB_LOG_EVENT_EXPECTED_HEADER_ELEMENT_COUNT) { - return FLB_EVENT_DECODER_ERROR_WRONG_HEADER_SIZE; - } - - timestamp = &header->via.array.ptr[0]; - metadata = &header->via.array.ptr[1]; - - format = FLB_LOG_EVENT_FORMAT_FLUENT_BIT_V2; - } - else { - header = NULL; - timestamp = &root->via.array.ptr[0]; - metadata = context->empty_map; - - format = FLB_LOG_EVENT_FORMAT_FORWARD; - } - - if (timestamp->type != MSGPACK_OBJECT_POSITIVE_INTEGER && - timestamp->type != MSGPACK_OBJECT_FLOAT && - timestamp->type != MSGPACK_OBJECT_EXT) { - return FLB_EVENT_DECODER_ERROR_WRONG_TIMESTAMP_TYPE; - } - - if (metadata->type != MSGPACK_OBJECT_MAP) { - return FLB_EVENT_DECODER_ERROR_WRONG_METADATA_TYPE; - } - - body = &root->via.array.ptr[1]; - - if (body->type != MSGPACK_OBJECT_MAP) { - return FLB_EVENT_DECODER_ERROR_WRONG_BODY_TYPE; - } - - result = flb_log_event_decoder_decode_timestamp(timestamp, &event->timestamp); - - if (result != FLB_EVENT_DECODER_SUCCESS) { - return result; - } - - event->raw_timestamp = timestamp; - event->metadata = metadata; - event->format = format; - event->body = body; - event->root = root; - - context->record_base = \ - (const char *) &context->buffer[context->previous_offset]; - context->record_length = context->offset - context->previous_offset; - - return FLB_EVENT_DECODER_SUCCESS; -} - -int flb_log_event_decoder_get_last_result(struct flb_log_event_decoder *context) -{ - if (context->last_result == FLB_EVENT_DECODER_ERROR_INSUFFICIENT_DATA && - context->offset == context->length) { - context->last_result = FLB_EVENT_DECODER_SUCCESS; - } - - return context->last_result; -} - -int flb_log_event_decoder_next(struct flb_log_event_decoder *context, - struct flb_log_event *event) -{ - size_t previous_offset; - int result; - - if (context == NULL) { - return FLB_EVENT_DECODER_ERROR_INVALID_CONTEXT; - } - if (context->length == 0) { - context->last_result = FLB_EVENT_DECODER_ERROR_INSUFFICIENT_DATA; - return context->last_result; - } - - context->record_base = NULL; - context->record_length = 0; - - if (event == NULL) { - context->last_result = FLB_EVENT_DECODER_ERROR_INVALID_ARGUMENT; - return context->last_result; - } - - memset(event, 0, sizeof(struct flb_log_event)); - - previous_offset = context->offset; - - result = msgpack_unpack_next(&context->unpacked_event, - context->buffer, - context->length, - &context->offset); - - if (result == MSGPACK_UNPACK_CONTINUE) { - context->last_result = FLB_EVENT_DECODER_ERROR_INSUFFICIENT_DATA; - return context->last_result; - } - else if (result != MSGPACK_UNPACK_SUCCESS) { - context->last_result = FLB_EVENT_DECODER_ERROR_DESERIALIZATION_FAILURE; - return context->last_result; - } - - context->previous_offset = previous_offset; - context->last_result = flb_event_decoder_decode_object(context, - event, - &context->unpacked_event.data); - return context->last_result; -} - -const char *flb_log_event_decoder_get_error_description(int error_code) -{ - const char *ret; - - switch (error_code) { - case FLB_EVENT_DECODER_SUCCESS: - ret = "Success"; - break; - - case FLB_EVENT_DECODER_ERROR_INITIALIZATION_FAILURE: - ret = "Initialization failure"; - break; - - case FLB_EVENT_DECODER_ERROR_INVALID_CONTEXT: - ret = "Invalid context"; - break; - - case FLB_EVENT_DECODER_ERROR_INVALID_ARGUMENT: - ret = "Invalid argument"; - break; - - case FLB_EVENT_DECODER_ERROR_WRONG_ROOT_TYPE: - ret = "Wrong root type"; - break; - - case FLB_EVENT_DECODER_ERROR_WRONG_ROOT_SIZE: - ret = "Wrong root size"; - break; - - case FLB_EVENT_DECODER_ERROR_WRONG_HEADER_TYPE: - ret = "Wrong header type"; - break; - - case FLB_EVENT_DECODER_ERROR_WRONG_HEADER_SIZE: - ret = "Wrong header size"; - break; - - case FLB_EVENT_DECODER_ERROR_WRONG_TIMESTAMP_TYPE: - ret = "Wrong timestamp type"; - break; - - case FLB_EVENT_DECODER_ERROR_WRONG_METADATA_TYPE: - ret = "Wrong metadata type"; - break; - - case FLB_EVENT_DECODER_ERROR_WRONG_BODY_TYPE: - ret = "Wrong body type"; - break; - - case FLB_EVENT_DECODER_ERROR_DESERIALIZATION_FAILURE: - ret = "Deserialization failure"; - break; - - case FLB_EVENT_DECODER_ERROR_INSUFFICIENT_DATA: - ret = "Insufficient data"; - break; - - default: - ret = "Unknown error"; - } - return ret; -} diff --git a/fluent-bit/src/flb_log_event_encoder.c b/fluent-bit/src/flb_log_event_encoder.c deleted file mode 100644 index bb6507b8c..000000000 --- a/fluent-bit/src/flb_log_event_encoder.c +++ /dev/null @@ -1,391 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <fluent-bit/flb_log_event_encoder.h> -#include <fluent-bit/flb_log_event_encoder_primitives.h> -#include <stdarg.h> - -void static inline flb_log_event_encoder_update_internal_state( - struct flb_log_event_encoder *context) -{ - context->output_buffer = context->buffer.data; - context->output_length = context->buffer.size; -} - -void flb_log_event_encoder_reset(struct flb_log_event_encoder *context) -{ - flb_log_event_encoder_dynamic_field_reset(&context->metadata); - flb_log_event_encoder_dynamic_field_reset(&context->body); - flb_log_event_encoder_dynamic_field_reset(&context->root); - - msgpack_sbuffer_clear(&context->buffer); - - flb_log_event_encoder_update_internal_state(context); -} - -int flb_log_event_encoder_init(struct flb_log_event_encoder *context, int format) -{ - if (context == NULL) { - return FLB_EVENT_ENCODER_ERROR_INVALID_CONTEXT; - } - - if (format < FLB_LOG_EVENT_FORMAT_FORWARD || - format > FLB_LOG_EVENT_FORMAT_FLUENT_BIT_V2) { - return FLB_EVENT_ENCODER_ERROR_INVALID_ARGUMENT; - } - - memset(context, 0, sizeof(struct flb_log_event_encoder)); - - context->dynamically_allocated = FLB_FALSE; - context->initialized = FLB_TRUE; - context->format = format; - - msgpack_sbuffer_init(&context->buffer); - msgpack_packer_init(&context->packer, - &context->buffer, - msgpack_sbuffer_write); - - flb_log_event_encoder_dynamic_field_init(&context->metadata, - MSGPACK_OBJECT_MAP); - - flb_log_event_encoder_dynamic_field_init(&context->body, - MSGPACK_OBJECT_MAP); - - flb_log_event_encoder_dynamic_field_init(&context->root, - MSGPACK_OBJECT_ARRAY); - - return FLB_EVENT_ENCODER_SUCCESS; -} - -struct flb_log_event_encoder *flb_log_event_encoder_create(int format) -{ - struct flb_log_event_encoder *context; - int result; - - context = (struct flb_log_event_encoder *) \ - flb_calloc(1, sizeof(struct flb_log_event_encoder)); - - result = flb_log_event_encoder_init(context, format); - - if (context != NULL) { - context->dynamically_allocated = FLB_TRUE; - - if (result != FLB_EVENT_ENCODER_SUCCESS) { - flb_log_event_encoder_destroy(context); - - context = NULL; - } - } - - return context; -} - -void flb_log_event_encoder_destroy(struct flb_log_event_encoder *context) -{ - if (context != NULL) { - if (context->initialized) { - flb_log_event_encoder_dynamic_field_destroy(&context->metadata); - flb_log_event_encoder_dynamic_field_destroy(&context->body); - flb_log_event_encoder_dynamic_field_destroy(&context->root); - - msgpack_sbuffer_destroy(&context->buffer); - - context->initialized = FLB_FALSE; - } - - if (context->dynamically_allocated) { - flb_free(context); - } - } -} - -void flb_log_event_encoder_claim_internal_buffer_ownership( - struct flb_log_event_encoder *context) -{ - if (context != NULL) { - msgpack_sbuffer_release(&context->buffer); - } -} - -int flb_log_event_encoder_emit_raw_record(struct flb_log_event_encoder *context, - const char *buffer, - size_t length) -{ - int result; - - result = msgpack_pack_str_body(&context->packer, buffer, length); - - if (result != 0) { - result = FLB_EVENT_ENCODER_ERROR_SERIALIZATION_FAILURE; - } - else { - result = FLB_EVENT_ENCODER_SUCCESS; - } - - flb_log_event_encoder_update_internal_state(context); - flb_log_event_encoder_reset_record(context); - - return result; -} - -int flb_log_event_encoder_emit_record(struct flb_log_event_encoder *context) -{ - int result; - - if (context == NULL) { - return FLB_EVENT_ENCODER_ERROR_INVALID_CONTEXT; - } - - result = FLB_EVENT_ENCODER_SUCCESS; - - /* This function needs to be improved and optimized to avoid excessive - * memory copying operations. - */ - - /* This conditional accounts for external raw record emission as - * performed by some filters using either - * flb_log_event_encoder_set_root_from_raw_msgpack - * or - * flb_log_event_encoder_set_root_from_msgpack_object - */ - if (context->root.size == 0) { - result = flb_log_event_encoder_root_begin_array(context); - - if (context->format == FLB_LOG_EVENT_FORMAT_FLUENT_BIT_V2) { - if (result == FLB_EVENT_ENCODER_SUCCESS) { - result = flb_log_event_encoder_root_begin_array(context); - } - } - - if (result == FLB_EVENT_ENCODER_SUCCESS) { - result = flb_log_event_encoder_append_root_timestamp( - context, &context->timestamp); - } - - if (context->format == FLB_LOG_EVENT_FORMAT_FLUENT_BIT_V2) { - if (result == FLB_EVENT_ENCODER_SUCCESS) { - result = flb_log_event_encoder_append_root_raw_msgpack( - context, - context->metadata.data, - context->metadata.size); - } - - /* We need to explicitly commit the current array (which - * holds the timestamp and metadata elements so we leave - * that scope and go back to the root scope where we can - * append the body element. - */ - if (result == FLB_EVENT_ENCODER_SUCCESS) { - result = flb_log_event_encoder_root_commit_array(context); - } - } - - if (result == FLB_EVENT_ENCODER_SUCCESS) { - result = flb_log_event_encoder_append_root_raw_msgpack( - context, - context->body.data, - context->body.size); - } - - if (result == FLB_EVENT_ENCODER_SUCCESS) { - result = flb_log_event_encoder_dynamic_field_flush(&context->root); - } - } - - if (result == FLB_EVENT_ENCODER_SUCCESS) { - result = msgpack_pack_str_body(&context->packer, - context->root.data, - context->root.size); - - if (result != 0) { - result = FLB_EVENT_ENCODER_ERROR_SERIALIZATION_FAILURE; - } - else { - result = FLB_EVENT_ENCODER_SUCCESS; - } - } - - flb_log_event_encoder_update_internal_state(context); - flb_log_event_encoder_reset_record(context); - - return result; -} - -int flb_log_event_encoder_reset_record(struct flb_log_event_encoder *context) -{ - flb_log_event_encoder_dynamic_field_reset(&context->metadata); - flb_log_event_encoder_dynamic_field_reset(&context->body); - flb_log_event_encoder_dynamic_field_reset(&context->root); - - flb_time_zero(&context->timestamp); - - return FLB_EVENT_ENCODER_SUCCESS; -} - -int flb_log_event_encoder_rollback_record(struct flb_log_event_encoder *context) -{ - return flb_log_event_encoder_reset_record(context); -} - -int flb_log_event_encoder_begin_record(struct flb_log_event_encoder *context) -{ - flb_log_event_encoder_reset_record(context); - - flb_log_event_encoder_metadata_begin_map(context); - flb_log_event_encoder_body_begin_map(context); - - return FLB_EVENT_ENCODER_SUCCESS; -} - -int flb_log_event_encoder_commit_record(struct flb_log_event_encoder *context) -{ - int result; - - result = flb_log_event_encoder_dynamic_field_flush(&context->metadata); - - if (result == FLB_EVENT_ENCODER_SUCCESS) { - result = flb_log_event_encoder_dynamic_field_flush(&context->body); - } - - if (result == FLB_EVENT_ENCODER_SUCCESS) { - result = flb_log_event_encoder_emit_record(context); - } - else { - flb_log_event_encoder_reset_record(context); - } - - return result; -} - -int flb_log_event_encoder_set_timestamp( - struct flb_log_event_encoder *context, - struct flb_time *timestamp) -{ - if (timestamp != NULL) { - flb_time_copy(&context->timestamp, timestamp); - } - else { - flb_time_get(&context->timestamp); - } - - return FLB_EVENT_ENCODER_SUCCESS; -} - -int flb_log_event_encoder_set_current_timestamp( - struct flb_log_event_encoder *context) -{ - return flb_log_event_encoder_set_timestamp(context, NULL); -} - -int flb_log_event_encoder_append_metadata_values_unsafe( - struct flb_log_event_encoder *context, - ...) -{ - va_list arguments; - int result; - - va_start(arguments, context); - - result = flb_log_event_encoder_append_values_unsafe( - context, - FLB_LOG_EVENT_METADATA, - arguments); - - va_end(arguments); - - return result; -} - -int flb_log_event_encoder_append_body_values_unsafe( - struct flb_log_event_encoder *context, - ...) -{ - va_list arguments; - int result; - - va_start(arguments, context); - - result = flb_log_event_encoder_append_values_unsafe( - context, - FLB_LOG_EVENT_BODY, - arguments); - - va_end(arguments); - - return result; -} - -int flb_log_event_encoder_append_root_values_unsafe( - struct flb_log_event_encoder *context, - ...) -{ - va_list arguments; - int result; - - va_start(arguments, context); - - result = flb_log_event_encoder_append_values_unsafe( - context, - FLB_LOG_EVENT_ROOT, - arguments); - - va_end(arguments); - - return result; -} - -const char *flb_log_event_encoder_get_error_description(int error_code) -{ - const char *ret; - - switch (error_code) { - case FLB_EVENT_ENCODER_SUCCESS: - ret = "Success"; - break; - - case FLB_EVENT_ENCODER_ERROR_UNSPECIFIED: - ret = "Unspecified"; - break; - - case FLB_EVENT_ENCODER_ERROR_ALLOCATION_ERROR: - ret = "Allocation error"; - break; - - case FLB_EVENT_ENCODER_ERROR_INVALID_CONTEXT: - ret = "Invalid context"; - break; - - case FLB_EVENT_ENCODER_ERROR_INVALID_ARGUMENT: - ret = "Invalid argument"; - break; - - case FLB_EVENT_ENCODER_ERROR_SERIALIZATION_FAILURE: - ret = "Serialization failure"; - break; - - case FLB_EVENT_ENCODER_ERROR_INVALID_VALUE_TYPE: - ret = "Invalid value type"; - break; - - default: - ret = "Unknown error"; - } - - return ret; -} diff --git a/fluent-bit/src/flb_log_event_encoder_dynamic_field.c b/fluent-bit/src/flb_log_event_encoder_dynamic_field.c deleted file mode 100644 index fd8be44a7..000000000 --- a/fluent-bit/src/flb_log_event_encoder_dynamic_field.c +++ /dev/null @@ -1,272 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <fluent-bit/flb_log_event_encoder.h> -#include <fluent-bit/flb_log_event_encoder_dynamic_field.h> - -struct flb_log_event_encoder_dynamic_field_scope * - flb_log_event_encoder_dynamic_field_scope_current( - struct flb_log_event_encoder_dynamic_field *field) -{ - if (cfl_list_is_empty(&field->scopes)) { - return NULL; - } - - return cfl_list_entry_first( - &field->scopes, - struct flb_log_event_encoder_dynamic_field_scope, - _head); -} - -int flb_log_event_encoder_dynamic_field_scope_enter( - struct flb_log_event_encoder_dynamic_field *field, - int type) -{ - int result; - struct flb_log_event_encoder_dynamic_field_scope *scope; - - if (type != MSGPACK_OBJECT_MAP && - type != MSGPACK_OBJECT_ARRAY) { - return FLB_EVENT_ENCODER_ERROR_INVALID_ARGUMENT; - } - - result = flb_log_event_encoder_dynamic_field_append(field); - - if (result != FLB_EVENT_ENCODER_SUCCESS) { - return result; - } - - scope = flb_calloc(1, - sizeof(struct flb_log_event_encoder_dynamic_field_scope)); - - if (scope == NULL) { - return FLB_EVENT_ENCODER_ERROR_ALLOCATION_ERROR; - } - - cfl_list_entry_init(&scope->_head); - - scope->type = type; - scope->offset = field->buffer.size; - - cfl_list_prepend(&scope->_head, &field->scopes); - - if (type == MSGPACK_OBJECT_MAP) { - flb_mp_map_header_init(&scope->header, &field->packer); - } - else if (type == MSGPACK_OBJECT_ARRAY) { - flb_mp_array_header_init(&scope->header, &field->packer); - } - - return FLB_EVENT_ENCODER_SUCCESS; -} - -int flb_log_event_encoder_dynamic_field_scope_leave( - struct flb_log_event_encoder_dynamic_field *field, - struct flb_log_event_encoder_dynamic_field_scope *scope, - int commit) -{ - if (scope == NULL) { - return FLB_EVENT_ENCODER_ERROR_INVALID_ARGUMENT; - } - - if (commit) { - /* We increment the entry count on each append because - * we don't discriminate based on the scope type so - * we need to divide the entry count by two for maps - * to ensure the entry count matches the kv pair count - */ - - if (scope->type == MSGPACK_OBJECT_MAP) { - scope->header.entries /= 2; - flb_mp_map_header_end(&scope->header); - } - else { - flb_mp_array_header_end(&scope->header); - } - } - else { - field->buffer.size = scope->offset; - } - - cfl_list_del(&scope->_head); - - flb_free(scope); - - return FLB_EVENT_ENCODER_SUCCESS; -} - -int flb_log_event_encoder_dynamic_field_begin_map( - struct flb_log_event_encoder_dynamic_field *field) -{ - return flb_log_event_encoder_dynamic_field_scope_enter(field, - MSGPACK_OBJECT_MAP); -} - -int flb_log_event_encoder_dynamic_field_begin_array( - struct flb_log_event_encoder_dynamic_field *field) -{ - return flb_log_event_encoder_dynamic_field_scope_enter(field, - MSGPACK_OBJECT_ARRAY); -} - -int flb_log_event_encoder_dynamic_field_commit_map( - struct flb_log_event_encoder_dynamic_field *field) -{ - struct flb_log_event_encoder_dynamic_field_scope *scope; - - scope = flb_log_event_encoder_dynamic_field_scope_current(field); - - return flb_log_event_encoder_dynamic_field_scope_leave(field, - scope, - FLB_TRUE); -} - -int flb_log_event_encoder_dynamic_field_commit_array( - struct flb_log_event_encoder_dynamic_field *field) -{ - struct flb_log_event_encoder_dynamic_field_scope *scope; - - scope = flb_log_event_encoder_dynamic_field_scope_current(field); - - return flb_log_event_encoder_dynamic_field_scope_leave(field, - scope, - FLB_TRUE); -} - -int flb_log_event_encoder_dynamic_field_rollback_map( - struct flb_log_event_encoder_dynamic_field *field) -{ - struct flb_log_event_encoder_dynamic_field_scope *scope; - - scope = flb_log_event_encoder_dynamic_field_scope_current(field); - - return flb_log_event_encoder_dynamic_field_scope_leave(field, - scope, - FLB_FALSE); -} - -int flb_log_event_encoder_dynamic_field_rollback_array( - struct flb_log_event_encoder_dynamic_field *field) -{ - struct flb_log_event_encoder_dynamic_field_scope *scope; - - scope = flb_log_event_encoder_dynamic_field_scope_current(field); - - return flb_log_event_encoder_dynamic_field_scope_leave(field, - scope, - FLB_TRUE); -} - -int flb_log_event_encoder_dynamic_field_append( - struct flb_log_event_encoder_dynamic_field *field) -{ - struct flb_log_event_encoder_dynamic_field_scope *scope; - - scope = flb_log_event_encoder_dynamic_field_scope_current(field); - - if (scope == NULL) { - if (cfl_list_is_empty(&field->scopes)) { - return FLB_EVENT_ENCODER_SUCCESS; - } - - return FLB_EVENT_ENCODER_ERROR_INVALID_ARGUMENT; - } - - flb_mp_map_header_append(&scope->header); - - return FLB_EVENT_ENCODER_SUCCESS; -} - - -static int flb_log_event_encoder_dynamic_field_flush_scopes( - struct flb_log_event_encoder_dynamic_field *field, - int commit) -{ - int result; - struct flb_log_event_encoder_dynamic_field_scope *scope; - - result = FLB_EVENT_ENCODER_SUCCESS; - - do { - scope = flb_log_event_encoder_dynamic_field_scope_current(field); - - if (scope != NULL) { - result = flb_log_event_encoder_dynamic_field_scope_leave(field, - scope, - commit); - } - } while (scope != NULL && - result == FLB_EVENT_ENCODER_SUCCESS); - - return result; -} - -int flb_log_event_encoder_dynamic_field_flush( - struct flb_log_event_encoder_dynamic_field *field) -{ - int result; - - result = flb_log_event_encoder_dynamic_field_flush_scopes(field, FLB_TRUE); - - if (result == FLB_EVENT_ENCODER_SUCCESS) { - field->data = field->buffer.data; - field->size = field->buffer.size; - } - - return result; -} - -int flb_log_event_encoder_dynamic_field_reset( - struct flb_log_event_encoder_dynamic_field *field) -{ - msgpack_sbuffer_clear(&field->buffer); - - flb_log_event_encoder_dynamic_field_flush_scopes(field, FLB_FALSE); - - field->data = NULL; - field->size = 0; - - return FLB_EVENT_ENCODER_SUCCESS; -} - -int flb_log_event_encoder_dynamic_field_init( - struct flb_log_event_encoder_dynamic_field *field, - int type) -{ - msgpack_sbuffer_init(&field->buffer); - msgpack_packer_init(&field->packer, - &field->buffer, - msgpack_sbuffer_write); - - field->initialized = FLB_TRUE; - field->type = type; - - cfl_list_init(&field->scopes); - flb_log_event_encoder_dynamic_field_reset(field); - - return FLB_EVENT_ENCODER_SUCCESS; -} - -void flb_log_event_encoder_dynamic_field_destroy( - struct flb_log_event_encoder_dynamic_field *field) -{ - msgpack_sbuffer_destroy(&field->buffer); - - field->initialized = FLB_FALSE; -} diff --git a/fluent-bit/src/flb_log_event_encoder_primitives.c b/fluent-bit/src/flb_log_event_encoder_primitives.c deleted file mode 100644 index ca395e390..000000000 --- a/fluent-bit/src/flb_log_event_encoder_primitives.c +++ /dev/null @@ -1,721 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <fluent-bit/flb_log_event_encoder.h> -#include <fluent-bit/flb_log_event_encoder_primitives.h> -#include <fluent-bit/flb_byteswap.h> -#include <stdarg.h> - -static inline \ -int translate_msgpack_encoder_result(int value) -{ - if (value != 0) { - return FLB_EVENT_ENCODER_ERROR_SERIALIZATION_FAILURE; - } - - return FLB_EVENT_ENCODER_SUCCESS; -} - -int flb_log_event_encoder_append_value( - struct flb_log_event_encoder *context, - int target_field, - int increment_entry_count, - int value_type, - char *value_buffer, - size_t value_length) -{ - int result; - struct flb_log_event_encoder_dynamic_field *field; - - if (value_type < FLB_LOG_EVENT_STRING_MIN_VALUE_TYPE || - value_type > FLB_LOG_EVENT_STRING_MAX_VALUE_TYPE) { - return FLB_EVENT_ENCODER_ERROR_INVALID_ARGUMENT; - } - - result = flb_log_event_encoder_get_field(context, target_field, &field); - - if (result == FLB_EVENT_ENCODER_SUCCESS) { - if (increment_entry_count) { - result = flb_log_event_encoder_dynamic_field_append(field); - } - - if (result == FLB_EVENT_ENCODER_SUCCESS) { - if (value_type == FLB_LOG_EVENT_STRING_LENGTH_VALUE_TYPE) { - result = msgpack_pack_str(&field->packer, value_length); - } - else if (value_type == FLB_LOG_EVENT_BINARY_LENGTH_VALUE_TYPE) { - result = msgpack_pack_bin(&field->packer, value_length); - } - else if (value_type == FLB_LOG_EVENT_EXT_LENGTH_VALUE_TYPE) { - result = msgpack_pack_ext(&field->packer, value_length, - *((int8_t *) value_buffer)); - } - else if (value_type == FLB_LOG_EVENT_NULL_VALUE_TYPE) { - result = msgpack_pack_nil(&field->packer); - } - else { - if (value_buffer == NULL) { - return FLB_EVENT_ENCODER_ERROR_INVALID_ARGUMENT; - } - - if (value_type == FLB_LOG_EVENT_STRING_BODY_VALUE_TYPE) { - result = msgpack_pack_str_body(&field->packer, - value_buffer, - value_length); - } - else if (value_type == FLB_LOG_EVENT_BINARY_BODY_VALUE_TYPE) { - result = msgpack_pack_bin_body(&field->packer, - value_buffer, - value_length); - } - else if (value_type == FLB_LOG_EVENT_EXT_BODY_VALUE_TYPE) { - result = msgpack_pack_ext_body(&field->packer, - value_buffer, - value_length); - } - else if (value_type == FLB_LOG_EVENT_CHAR_VALUE_TYPE) { - result = msgpack_pack_char(&field->packer, - *((char *) value_buffer)); - } - else if (value_type == FLB_LOG_EVENT_INT8_VALUE_TYPE) { - result = msgpack_pack_int8(&field->packer, - *((int8_t *) value_buffer)); - } - else if (value_type == FLB_LOG_EVENT_INT16_VALUE_TYPE) { - result = msgpack_pack_int16(&field->packer, - *((int16_t *) value_buffer)); - } - else if (value_type == FLB_LOG_EVENT_INT32_VALUE_TYPE) { - result = msgpack_pack_int32(&field->packer, - *((int32_t *) value_buffer)); - } - else if (value_type == FLB_LOG_EVENT_INT64_VALUE_TYPE) { - result = msgpack_pack_int64(&field->packer, - *((int64_t *) value_buffer)); - } - else if (value_type == FLB_LOG_EVENT_UINT8_VALUE_TYPE) { - result = msgpack_pack_uint8(&field->packer, - *((uint8_t *) value_buffer)); - } - else if (value_type == FLB_LOG_EVENT_UINT16_VALUE_TYPE) { - result = msgpack_pack_uint16(&field->packer, - *((uint16_t *) value_buffer)); - } - else if (value_type == FLB_LOG_EVENT_UINT32_VALUE_TYPE) { - result = msgpack_pack_uint32(&field->packer, - *((uint32_t *) value_buffer)); - } - else if (value_type == FLB_LOG_EVENT_UINT64_VALUE_TYPE) { - result = msgpack_pack_uint64(&field->packer, - *((uint64_t *) value_buffer)); - } - else if (value_type == FLB_LOG_EVENT_DOUBLE_VALUE_TYPE) { - result = msgpack_pack_double(&field->packer, - *((double *) value_buffer)); - } - else if (value_type == FLB_LOG_EVENT_BOOLEAN_VALUE_TYPE) { - if (*((int *) value_buffer)) { - result = msgpack_pack_true(&field->packer); - } - else { - result = msgpack_pack_false(&field->packer); - } - } - else if (value_type == FLB_LOG_EVENT_MSGPACK_OBJECT_VALUE_TYPE) { - result = msgpack_pack_object( - &field->packer, - *((msgpack_object *) value_buffer)); - } - else if (value_type == FLB_LOG_EVENT_MSGPACK_RAW_VALUE_TYPE) { - result = msgpack_pack_str_body(&field->packer, - value_buffer, - value_length); - } - else { - return FLB_EVENT_ENCODER_ERROR_INVALID_CONTEXT; - } - - result = translate_msgpack_encoder_result(result); - } - } - } - - return result; -} - -int flb_log_event_encoder_append_binary_length( - struct flb_log_event_encoder *context, - int target_field, - size_t length) -{ - return flb_log_event_encoder_append_value( - context, target_field, FLB_TRUE, - FLB_LOG_EVENT_BINARY_LENGTH_VALUE_TYPE, - NULL, length); -} - -int flb_log_event_encoder_append_binary_body( - struct flb_log_event_encoder *context, - int target_field, - char *value, - size_t length) -{ - return flb_log_event_encoder_append_value( - context, target_field, FLB_FALSE, - FLB_LOG_EVENT_BINARY_BODY_VALUE_TYPE, - value, length); -} - -int flb_log_event_encoder_append_ext_length( - struct flb_log_event_encoder *context, - int target_field, - int8_t type, - size_t length) -{ - return flb_log_event_encoder_append_value( - context, target_field, FLB_TRUE, - FLB_LOG_EVENT_EXT_LENGTH_VALUE_TYPE, - (char *) &type, length); -} - -int flb_log_event_encoder_append_ext_body( - struct flb_log_event_encoder *context, - int target_field, - char *value, - size_t length) -{ - return flb_log_event_encoder_append_value( - context, target_field, FLB_FALSE, - FLB_LOG_EVENT_EXT_BODY_VALUE_TYPE, - value, length); -} - -int flb_log_event_encoder_append_string_length( - struct flb_log_event_encoder *context, - int target_field, - size_t length) -{ - return flb_log_event_encoder_append_value( - context, target_field, FLB_TRUE, - FLB_LOG_EVENT_STRING_LENGTH_VALUE_TYPE, - NULL, length); -} - -int flb_log_event_encoder_append_string_body( - struct flb_log_event_encoder *context, - int target_field, - char *value, - size_t length) -{ - return flb_log_event_encoder_append_value( - context, target_field, FLB_FALSE, - FLB_LOG_EVENT_STRING_BODY_VALUE_TYPE, - value, length); -} -int flb_log_event_encoder_append_int8( - struct flb_log_event_encoder *context, - int target_field, - int8_t value) -{ - return flb_log_event_encoder_append_value( - context, target_field, FLB_TRUE, - FLB_LOG_EVENT_INT8_VALUE_TYPE, - (char *) &value, 0); -} - -int flb_log_event_encoder_append_int16( - struct flb_log_event_encoder *context, - int target_field, - int16_t value) -{ - return flb_log_event_encoder_append_value( - context, target_field, FLB_TRUE, - FLB_LOG_EVENT_INT16_VALUE_TYPE, - (char *) &value, 0); -} - -int flb_log_event_encoder_append_int32( - struct flb_log_event_encoder *context, - int target_field, - int32_t value) -{ - return flb_log_event_encoder_append_value( - context, target_field, FLB_TRUE, - FLB_LOG_EVENT_INT32_VALUE_TYPE, - (char *) &value, 0); -} - -int flb_log_event_encoder_append_int64( - struct flb_log_event_encoder *context, - int target_field, - int64_t value) -{ - return flb_log_event_encoder_append_value( - context, target_field, FLB_TRUE, - FLB_LOG_EVENT_INT64_VALUE_TYPE, - (char *) &value, 0); -} - -int flb_log_event_encoder_append_uint8( - struct flb_log_event_encoder *context, - int target_field, - uint8_t value) -{ - return flb_log_event_encoder_append_value( - context, target_field, FLB_TRUE, - FLB_LOG_EVENT_UINT8_VALUE_TYPE, - (char *) &value, 0); -} - -int flb_log_event_encoder_append_uint16( - struct flb_log_event_encoder *context, - int target_field, - uint16_t value) -{ - return flb_log_event_encoder_append_value( - context, target_field, FLB_TRUE, - FLB_LOG_EVENT_UINT16_VALUE_TYPE, - (char *) &value, 0); -} - -int flb_log_event_encoder_append_uint32( - struct flb_log_event_encoder *context, - int target_field, - uint32_t value) -{ - return flb_log_event_encoder_append_value( - context, target_field, FLB_TRUE, - FLB_LOG_EVENT_UINT32_VALUE_TYPE, - (char *) &value, 0); -} - -int flb_log_event_encoder_append_uint64( - struct flb_log_event_encoder *context, - int target_field, - uint64_t value) -{ - return flb_log_event_encoder_append_value( - context, target_field, FLB_TRUE, - FLB_LOG_EVENT_UINT64_VALUE_TYPE, - (char *) &value, 0); -} - -int flb_log_event_encoder_append_double( - struct flb_log_event_encoder *context, - int target_field, - double value) -{ - return flb_log_event_encoder_append_value( - context, target_field, FLB_TRUE, - FLB_LOG_EVENT_DOUBLE_VALUE_TYPE, - (char *) &value, 0); -} - -int flb_log_event_encoder_append_boolean( - struct flb_log_event_encoder *context, - int target_field, - int value) -{ - return flb_log_event_encoder_append_value( - context, target_field, FLB_TRUE, - FLB_LOG_EVENT_BOOLEAN_VALUE_TYPE, - (char *) &value, 0); -} - -int flb_log_event_encoder_append_null( - struct flb_log_event_encoder *context, - int target_field) -{ - return flb_log_event_encoder_append_value( - context, target_field, FLB_TRUE, - FLB_LOG_EVENT_NULL_VALUE_TYPE, - NULL, 0); -} - -int flb_log_event_encoder_append_character( - struct flb_log_event_encoder *context, - int target_field, - char value) -{ - return flb_log_event_encoder_append_value( - context, target_field, FLB_TRUE, - FLB_LOG_EVENT_CHAR_VALUE_TYPE, - (char *) &value, 0); -} - -int flb_log_event_encoder_append_binary( - struct flb_log_event_encoder *context, - int target_field, - char *value, - size_t length) -{ - int result; - - result = flb_log_event_encoder_append_binary_length( - context, - target_field, - length); - - if (result == FLB_EVENT_ENCODER_SUCCESS) { - result = flb_log_event_encoder_append_binary_body( - context, - target_field, - value, - length); - } - - return result; -} - -int flb_log_event_encoder_append_string( - struct flb_log_event_encoder *context, - int target_field, - char *value, - size_t length) -{ - int result; - - result = flb_log_event_encoder_append_string_length( - context, - target_field, - length); - - if (result == FLB_EVENT_ENCODER_SUCCESS) { - result = flb_log_event_encoder_append_string_body( - context, - target_field, - value, - length); - } - - return result; -} - -int flb_log_event_encoder_append_ext( - struct flb_log_event_encoder *context, - int target_field, - int8_t type, - char *value, - size_t length) -{ - int result; - - result = flb_log_event_encoder_append_ext_length( - context, - target_field, - type, - length); - - if (result == FLB_EVENT_ENCODER_SUCCESS) { - result = flb_log_event_encoder_append_ext_body( - context, - target_field, - value, - length); - } - - return result; -} - -int flb_log_event_encoder_append_cstring( - struct flb_log_event_encoder *context, - int target_field, - char *value) -{ - return flb_log_event_encoder_append_string( - context, - target_field, - value, - strlen(value)); -} - -int flb_log_event_encoder_append_msgpack_object( - struct flb_log_event_encoder *context, - int target_field, - msgpack_object *value) -{ - const int value_type = FLB_LOG_EVENT_MSGPACK_OBJECT_VALUE_TYPE; - - return flb_log_event_encoder_append_value(context, target_field, - FLB_TRUE, value_type, - (char *) value, 0); -} - -int flb_log_event_encoder_append_raw_msgpack( - struct flb_log_event_encoder *context, - int target_field, - char *value_buffer, - size_t value_size) -{ - const int value_type = FLB_LOG_EVENT_MSGPACK_RAW_VALUE_TYPE; - - return flb_log_event_encoder_append_value(context, target_field, - FLB_TRUE, value_type, - value_buffer, value_size); -} - - -int flb_log_event_encoder_append_timestamp( - struct flb_log_event_encoder *context, - int target_field, - struct flb_time *value) -{ - if (context->format == FLB_LOG_EVENT_FORMAT_FORWARD_LEGACY) { - return flb_log_event_encoder_append_legacy_timestamp( - context, target_field, value); - } - else if (context->format == FLB_LOG_EVENT_FORMAT_FORWARD) { - return flb_log_event_encoder_append_forward_v1_timestamp( - context, target_field, value); - } - else if (context->format == FLB_LOG_EVENT_FORMAT_FLUENT_BIT_V1) { - return flb_log_event_encoder_append_fluent_bit_v1_timestamp( - context, target_field, value); - } - else if (context->format == FLB_LOG_EVENT_FORMAT_FLUENT_BIT_V2) { - return flb_log_event_encoder_append_fluent_bit_v2_timestamp( - context, target_field, value); - } - else { - return FLB_EVENT_ENCODER_ERROR_INVALID_ARGUMENT; - } -} - -int flb_log_event_encoder_append_legacy_timestamp( - struct flb_log_event_encoder *context, - int target_field, - struct flb_time *value) -{ - const int value_type = FLB_LOG_EVENT_UINT64_VALUE_TYPE; - uint64_t timestamp; - - timestamp = value->tm.tv_sec; - - return flb_log_event_encoder_append_value(context, target_field, - FLB_TRUE, value_type, - (char *) ×tamp, 0); -} - -int flb_log_event_encoder_append_forward_v1_timestamp( - struct flb_log_event_encoder *context, - int target_field, - struct flb_time *timestamp) -{ - uint32_t value[2]; - - value[0] = FLB_BSWAP_32((uint32_t) timestamp->tm.tv_sec); - value[1] = FLB_BSWAP_32((uint32_t) timestamp->tm.tv_nsec); - - return flb_log_event_encoder_append_ext(context, target_field, - 0, (char *) value, 8); -} - -int flb_log_event_encoder_append_fluent_bit_v1_timestamp( - struct flb_log_event_encoder *context, - int target_field, - struct flb_time *value) -{ - return flb_log_event_encoder_append_forward_v1_timestamp(context, - target_field, - value); -} - -int flb_log_event_encoder_append_fluent_bit_v2_timestamp( - struct flb_log_event_encoder *context, - int target_field, - struct flb_time *value) -{ - return flb_log_event_encoder_append_fluent_bit_v1_timestamp(context, - target_field, - value); -} - -int flb_log_event_encoder_append_values_unsafe( - struct flb_log_event_encoder *context, - int target_field, - va_list arguments) -{ - int8_t current_ext_type; - size_t processed_values; - char *buffer_address; - int value_type; - int result; - - processed_values = 0; - result = FLB_EVENT_ENCODER_SUCCESS; - - for (processed_values = 0 ; - processed_values < FLB_EVENT_ENCODER_VALUE_LIMIT && - result == FLB_EVENT_ENCODER_SUCCESS ; - processed_values++) { - value_type = va_arg(arguments, int); - - if (value_type == FLB_LOG_EVENT_APPEND_TERMINATOR_VALUE_TYPE) { - break; - } - else if (value_type == FLB_LOG_EVENT_STRING_LENGTH_VALUE_TYPE) { - result = flb_log_event_encoder_append_string_length(context, - target_field, - va_arg(arguments, size_t)); - } - else if (value_type == FLB_LOG_EVENT_STRING_BODY_VALUE_TYPE) { - buffer_address = va_arg(arguments, char *); - - result = flb_log_event_encoder_append_string_body(context, - target_field, - buffer_address, - va_arg(arguments, size_t)); - } - else if (value_type == FLB_LOG_EVENT_BINARY_LENGTH_VALUE_TYPE) { - result = flb_log_event_encoder_append_binary_length(context, - target_field, - va_arg(arguments, size_t)); - } - else if (value_type == FLB_LOG_EVENT_BINARY_BODY_VALUE_TYPE) { - buffer_address = va_arg(arguments, char *); - - result = flb_log_event_encoder_append_binary_body(context, - target_field, - buffer_address, - va_arg(arguments, size_t)); - } - else if (value_type == FLB_LOG_EVENT_EXT_LENGTH_VALUE_TYPE) { - current_ext_type = (int8_t) va_arg(arguments, int); - - result = flb_log_event_encoder_append_ext_length(context, - target_field, - current_ext_type, - va_arg(arguments, size_t)); - } - else if (value_type == FLB_LOG_EVENT_EXT_BODY_VALUE_TYPE) { - buffer_address = va_arg(arguments, char *); - - result = flb_log_event_encoder_append_ext_body(context, - target_field, - buffer_address, - va_arg(arguments, size_t)); - } - else if (value_type == FLB_LOG_EVENT_NULL_VALUE_TYPE) { - result = flb_log_event_encoder_append_null(context, - target_field); - } - else if (value_type == FLB_LOG_EVENT_CHAR_VALUE_TYPE) { - result = flb_log_event_encoder_append_character(context, - target_field, - (char) va_arg(arguments, int)); - } - else if (value_type == FLB_LOG_EVENT_INT8_VALUE_TYPE) { - result = flb_log_event_encoder_append_int8(context, - target_field, - (int8_t) va_arg(arguments, int)); - } - else if (value_type == FLB_LOG_EVENT_INT16_VALUE_TYPE) { - result = flb_log_event_encoder_append_int16(context, - target_field, - (int16_t) va_arg(arguments, int)); - } - else if (value_type == FLB_LOG_EVENT_INT32_VALUE_TYPE) { - result = flb_log_event_encoder_append_int32(context, - target_field, - va_arg(arguments, int32_t)); - } - else if (value_type == FLB_LOG_EVENT_INT64_VALUE_TYPE) { - result = flb_log_event_encoder_append_int64(context, - target_field, - va_arg(arguments, int64_t)); - } - else if (value_type == FLB_LOG_EVENT_UINT8_VALUE_TYPE) { - result = flb_log_event_encoder_append_uint8(context, - target_field, - (uint8_t) va_arg(arguments, unsigned int)); - } - else if (value_type == FLB_LOG_EVENT_UINT16_VALUE_TYPE) { - result = flb_log_event_encoder_append_uint16(context, - target_field, - (uint16_t) va_arg(arguments, unsigned int)); - } - else if (value_type == FLB_LOG_EVENT_UINT32_VALUE_TYPE) { - result = flb_log_event_encoder_append_uint32(context, - target_field, - va_arg(arguments, uint32_t)); - } - else if (value_type == FLB_LOG_EVENT_UINT64_VALUE_TYPE) { - result = flb_log_event_encoder_append_uint64(context, - target_field, - va_arg(arguments, uint64_t)); - } - else if (value_type == FLB_LOG_EVENT_DOUBLE_VALUE_TYPE) { - result = flb_log_event_encoder_append_double(context, - target_field, - va_arg(arguments, double)); - } - else if (value_type == FLB_LOG_EVENT_BOOLEAN_VALUE_TYPE) { - result = flb_log_event_encoder_append_boolean(context, - target_field, - va_arg(arguments, int)); - } - else if (value_type == FLB_LOG_EVENT_MSGPACK_OBJECT_VALUE_TYPE) { - result = flb_log_event_encoder_append_msgpack_object(context, - target_field, - va_arg(arguments, msgpack_object *)); - } - else if (value_type == FLB_LOG_EVENT_MSGPACK_RAW_VALUE_TYPE) { - buffer_address = va_arg(arguments, char *); - - result = flb_log_event_encoder_append_raw_msgpack(context, - target_field, - buffer_address, - va_arg(arguments, size_t)); - } - else if (value_type == FLB_LOG_EVENT_TIMESTAMP_VALUE_TYPE) { - result = flb_log_event_encoder_append_timestamp(context, - target_field, - va_arg(arguments, struct flb_time *)); - } - else if (value_type == FLB_LOG_EVENT_LEGACY_TIMESTAMP_VALUE_TYPE) { - result = flb_log_event_encoder_append_legacy_timestamp(context, - target_field, - va_arg(arguments, struct flb_time *)); - } - else if (value_type == FLB_LOG_EVENT_FORWARD_V1_TIMESTAMP_VALUE_TYPE) { - result = flb_log_event_encoder_append_forward_v1_timestamp(context, - target_field, - va_arg(arguments, struct flb_time *)); - } - else if (value_type == FLB_LOG_EVENT_FLUENT_BIT_V1_TIMESTAMP_VALUE_TYPE) { - result = flb_log_event_encoder_append_fluent_bit_v1_timestamp(context, - target_field, - va_arg(arguments, struct flb_time *)); - } - else if (value_type == FLB_LOG_EVENT_FLUENT_BIT_V2_TIMESTAMP_VALUE_TYPE) { - result = flb_log_event_encoder_append_fluent_bit_v2_timestamp(context, - target_field, - va_arg(arguments, struct flb_time *)); - } - else { - result = FLB_EVENT_ENCODER_ERROR_INVALID_VALUE_TYPE; - } - } - - if (processed_values >= FLB_EVENT_ENCODER_VALUE_LIMIT) { - flb_error("Log event encoder : value count limit exceeded"); - } - - return result; -} diff --git a/fluent-bit/src/flb_lua.c b/fluent-bit/src/flb_lua.c deleted file mode 100644 index e4df896c5..000000000 --- a/fluent-bit/src/flb_lua.c +++ /dev/null @@ -1,841 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "lua.h" -#include "mpack/mpack.h" -#include "msgpack/unpack.h" -#include <fluent-bit/flb_info.h> -#include <fluent-bit/flb_time.h> -#include <fluent-bit/flb_lua.h> -#include <stdint.h> - -int flb_lua_enable_flb_null(lua_State *l) -{ - /* set flb.null */ - lua_pushlightuserdata(l, NULL); - - flb_info("[%s] set %s", __FUNCTION__, FLB_LUA_VAR_FLB_NULL); - lua_setglobal(l, FLB_LUA_VAR_FLB_NULL); - - return 0; -} - -void flb_lua_pushtimetable(lua_State *l, struct flb_time *tm) -{ - lua_createtable(l, 0, 2); - - /* seconds */ - lua_pushlstring(l, "sec", 3); - lua_pushinteger(l, tm->tm.tv_sec); - lua_settable(l, -3); - - /* nanoseconds */ - lua_pushlstring(l, "nsec", 4); - lua_pushinteger(l, tm->tm.tv_nsec); - lua_settable(l, -3); -} - -int flb_lua_is_valid_func(lua_State *lua, flb_sds_t func) -{ - int ret = FLB_FALSE; - - lua_getglobal(lua, func); - if (lua_isfunction(lua, -1)) { - ret = FLB_TRUE; - } - lua_pop(lua, -1); /* discard return value of isfunction */ - - return ret; -} - -static int flb_lua_setmetatable(lua_State *l, struct flb_lua_metadata *meta, int index) -{ - int abs_index; - - if (meta->initialized != FLB_TRUE) { - return -1; - } - abs_index = flb_lua_absindex(l, index); - - lua_createtable(l, 0, 1); - - /* data type */ - lua_pushlstring(l, "type", 4); - lua_pushinteger(l, meta->data_type); - lua_settable(l, -3); /* point created table */ - - lua_setmetatable(l, abs_index); - - return 0; -} - -int flb_lua_pushmpack(lua_State *l, mpack_reader_t *reader) -{ - int ret = 0; - mpack_tag_t tag; - uint32_t length; - uint32_t i; - int index; - struct flb_lua_metadata meta; - - tag = mpack_read_tag(reader); - switch (mpack_tag_type(&tag)) { - case mpack_type_nil: - lua_getglobal(l, FLB_LUA_VAR_FLB_NULL); - break; - case mpack_type_bool: - lua_pushboolean(l, mpack_tag_bool_value(&tag)); - break; - case mpack_type_int: - lua_pushinteger(l, mpack_tag_int_value(&tag)); - break; - case mpack_type_uint: - lua_pushinteger(l, mpack_tag_uint_value(&tag)); - break; - case mpack_type_float: - lua_pushnumber(l, mpack_tag_float_value(&tag)); - break; - case mpack_type_double: - lua_pushnumber(l, mpack_tag_double_value(&tag)); - break; - case mpack_type_str: - case mpack_type_bin: - case mpack_type_ext: - length = mpack_tag_bytes(&tag); - lua_pushlstring(l, reader->data, length); - reader->data += length; - break; - case mpack_type_array: - flb_lua_metadata_init(&meta); - meta.data_type = FLB_LUA_L2C_TYPE_ARRAY; - - length = mpack_tag_array_count(&tag); - lua_createtable(l, length, 0); - index = lua_gettop(l); /* save index of created table */ - for (i = 0; i < length; i++) { - ret = flb_lua_pushmpack(l, reader); - if (ret) { - return ret; - } - lua_rawseti(l, -2, i+1); - } - flb_lua_setmetatable(l, &meta, index); - - break; - case mpack_type_map: - flb_lua_metadata_init(&meta); - meta.data_type = FLB_LUA_L2C_TYPE_MAP; - - length = mpack_tag_map_count(&tag); - lua_createtable(l, length, 0); - index = lua_gettop(l); /* save index of created table */ - for (i = 0; i < length; i++) { - ret = flb_lua_pushmpack(l, reader); - if (ret) { - return ret; - } - ret = flb_lua_pushmpack(l, reader); - if (ret) { - return ret; - } - lua_settable(l, -3); - } - flb_lua_setmetatable(l, &meta, index); - - break; - default: - return -1; - } - return 0; -} - -void flb_lua_pushmsgpack(lua_State *l, msgpack_object *o) -{ - int i; - int size; - int index; - struct flb_lua_metadata meta; - - lua_checkstack(l, 3); - - switch(o->type) { - case MSGPACK_OBJECT_NIL: - lua_getglobal(l, FLB_LUA_VAR_FLB_NULL); - break; - - case MSGPACK_OBJECT_BOOLEAN: - lua_pushboolean(l, o->via.boolean); - break; - - case MSGPACK_OBJECT_POSITIVE_INTEGER: - lua_pushinteger(l, (double) o->via.u64); - break; - - case MSGPACK_OBJECT_NEGATIVE_INTEGER: - lua_pushinteger(l, (double) o->via.i64); - break; - - case MSGPACK_OBJECT_FLOAT32: - case MSGPACK_OBJECT_FLOAT64: - lua_pushnumber(l, (double) o->via.f64); - break; - - case MSGPACK_OBJECT_STR: - lua_pushlstring(l, o->via.str.ptr, o->via.str.size); - break; - - case MSGPACK_OBJECT_BIN: - lua_pushlstring(l, o->via.bin.ptr, o->via.bin.size); - break; - - case MSGPACK_OBJECT_EXT: - lua_pushlstring(l, o->via.ext.ptr, o->via.ext.size); - break; - - case MSGPACK_OBJECT_ARRAY: - flb_lua_metadata_init(&meta); - meta.data_type = FLB_LUA_L2C_TYPE_ARRAY; - - size = o->via.array.size; - lua_createtable(l, size, 0); - index = lua_gettop(l); /* save index of created table */ - if (size != 0) { - msgpack_object *p = o->via.array.ptr; - for (i = 0; i < size; i++) { - flb_lua_pushmsgpack(l, p+i); - lua_rawseti (l, index, i+1); - } - } - flb_lua_setmetatable(l, &meta, index); - break; - - case MSGPACK_OBJECT_MAP: - flb_lua_metadata_init(&meta); - meta.data_type = FLB_LUA_L2C_TYPE_MAP; - - size = o->via.map.size; - lua_createtable(l, 0, size); - index = lua_gettop(l); /* save index of created table */ - if (size != 0) { - msgpack_object_kv *p = o->via.map.ptr; - for (i = 0; i < size; i++) { - flb_lua_pushmsgpack(l, &(p+i)->key); - flb_lua_pushmsgpack(l, &(p+i)->val); - lua_settable(l, index); - } - } - flb_lua_setmetatable(l, &meta, index); - break; - } -} - -static int lua_isinteger(lua_State *L, int index) -{ - lua_Number n; - lua_Integer i; - - if (lua_type(L, index) == LUA_TNUMBER) { - n = lua_tonumber(L, index); - i = lua_tointeger(L, index); - - if (i == n) { - return 1; - } - } - return 0; -} - -/* - * This function is to call lua function table.maxn. - * CAUTION: table.maxn is removed from Lua 5.2. - * If we update luajit which is based Lua 5.2+, - * this function should be removed. -*/ -static int lua_table_maxn(lua_State *l, int index) -{ -#if defined(LUA_VERSION_NUM) && LUA_VERSION_NUM < 520 - int ret = -1; - if (lua_type(l, index) != LUA_TTABLE) { - return -1; - } - - lua_getglobal(l, "table"); - lua_getfield(l, -1, "maxn"); - lua_remove(l, -2); /* remove table (lua_getglobal(L, "table")) */ - lua_pushvalue(l, index); /* copy record to top of stack */ - ret = lua_pcall(l, 1, 1, 0); - if (ret < 0) { - flb_error("[filter_lua] failed to exec table.maxn ret=%d", ret); - return -1; - } - if (lua_type(l, -1) != LUA_TNUMBER) { - flb_error("[filter_lua] not LUA_TNUMBER"); - lua_pop(l, 1); - return -1; - } - - if (lua_isinteger(l, -1)) { - ret = lua_tointeger(l, -1); - } - lua_pop(l, 1); - - return ret; -#else - return (int)lua_rawlen(l, index); -#endif -} - -int flb_lua_arraylength(lua_State *l, int index) -{ - lua_Integer n; - int count = 0; - int max = 0; - int ret = 0; - - index = flb_lua_absindex(l, index); - - ret = lua_table_maxn(l, index); - if (ret > 0) { - return ret; - } - - lua_pushnil(l); - while (lua_next(l, index) != 0) { - if (lua_type(l, -2) == LUA_TNUMBER) { - n = lua_tonumber(l, -2); - if (n > 0) { - max = n > max ? n : max; - count++; - lua_pop(l, 1); - continue; - } - } - lua_pop(l, 2); - return -1; - } - if (max != count) { - return -1; - } - return max; -} - -static void lua_toarray_msgpack(lua_State *l, - msgpack_packer *pck, - int index, - struct flb_lua_l2c_config *l2cc) -{ - int len; - int i; - - lua_pushnumber(l, (lua_Number)lua_objlen(l, -1)); // lua_len - len = (int)lua_tointeger(l, -1); - lua_pop(l, 1); - - msgpack_pack_array(pck, len); - for (i = 1; i <= len; i++) { - lua_rawgeti(l, -1, i); - flb_lua_tomsgpack(l, pck, 0, l2cc); - lua_pop(l, 1); - } -} - -static void lua_toarray_mpack(lua_State *l, - mpack_writer_t *writer, - int index, - struct flb_lua_l2c_config *l2cc) -{ - int len; - int i; - - lua_pushnumber(l, (lua_Number)lua_objlen(l, -1)); // lua_len - len = (int)lua_tointeger(l, -1); - lua_pop(l, 1); - - mpack_write_tag(writer, mpack_tag_array(len)); - for (i = 1; i <= len; i++) { - lua_rawgeti(l, -1, i); - flb_lua_tompack(l, writer, 0, l2cc); - lua_pop(l, 1); - } -} - -static void try_to_convert_data_type(lua_State *l, - msgpack_packer *pck, - struct flb_lua_l2c_config *l2cc) -{ - size_t len; - const char *tmp = NULL; - - struct mk_list *tmp_list = NULL; - struct mk_list *head = NULL; - struct flb_lua_l2c_type *l2c = NULL; - - // convert to int - if ((lua_type(l, -2) == LUA_TSTRING) - && lua_type(l, -1) == LUA_TNUMBER){ - tmp = lua_tolstring(l, -2, &len); - - mk_list_foreach_safe(head, tmp_list, &l2cc->l2c_types) { - l2c = mk_list_entry(head, struct flb_lua_l2c_type, _head); - if (!strncmp(l2c->key, tmp, len) && l2c->type == FLB_LUA_L2C_TYPE_INT) { - flb_lua_tomsgpack(l, pck, -1, l2cc); - msgpack_pack_int64(pck, (int64_t)lua_tonumber(l, -1)); - return; - } - } - } - else if ((lua_type(l, -2) == LUA_TSTRING) - && lua_type(l, -1) == LUA_TTABLE){ - tmp = lua_tolstring(l, -2, &len); - - mk_list_foreach_safe(head, tmp_list, &l2cc->l2c_types) { - l2c = mk_list_entry(head, struct flb_lua_l2c_type, _head); - if (!strncmp(l2c->key, tmp, len) && l2c->type == FLB_LUA_L2C_TYPE_ARRAY) { - flb_lua_tomsgpack(l, pck, -1, l2cc); - lua_toarray_msgpack(l, pck, 0, l2cc); - return; - } - } - } - - /* not matched */ - flb_lua_tomsgpack(l, pck, -1, l2cc); - flb_lua_tomsgpack(l, pck, 0, l2cc); -} - -static void try_to_convert_data_type_mpack(lua_State *l, - mpack_writer_t *writer, - struct flb_lua_l2c_config *l2cc) -{ - size_t len; - const char *tmp = NULL; - - struct mk_list *tmp_list = NULL; - struct mk_list *head = NULL; - struct flb_lua_l2c_type *l2c = NULL; - - // convert to int - if ((lua_type(l, -2) == LUA_TSTRING) - && lua_type(l, -1) == LUA_TNUMBER){ - tmp = lua_tolstring(l, -2, &len); - - mk_list_foreach_safe(head, tmp_list, &l2cc->l2c_types) { - l2c = mk_list_entry(head, struct flb_lua_l2c_type, _head); - if (!strncmp(l2c->key, tmp, len) && l2c->type == FLB_LUA_L2C_TYPE_INT) { - flb_lua_tompack(l, writer, -1, l2cc); - mpack_write_int(writer, (int64_t)lua_tonumber(l, -1)); - return; - } - } - } - else if ((lua_type(l, -2) == LUA_TSTRING) - && lua_type(l, -1) == LUA_TTABLE){ - tmp = lua_tolstring(l, -2, &len); - - mk_list_foreach_safe(head, tmp_list, &l2cc->l2c_types) { - l2c = mk_list_entry(head, struct flb_lua_l2c_type, _head); - if (!strncmp(l2c->key, tmp, len) && l2c->type == FLB_LUA_L2C_TYPE_ARRAY) { - flb_lua_tompack(l, writer, -1, l2cc); - lua_toarray_mpack(l, writer, 0, l2cc); - return; - } - } - } - - /* not matched */ - flb_lua_tompack(l, writer, -1, l2cc); - flb_lua_tompack(l, writer, 0, l2cc); -} - -static int flb_lua_getmetatable(lua_State *l, int index, struct flb_lua_metadata *meta) -{ - int lua_ret; - int abs_index; - const char *str; - size_t len; - - if (meta->initialized != FLB_TRUE) { - return -1; - } - - lua_ret = lua_getmetatable(l, index); - if (lua_ret == 0) { - /* no metadata */ - return -1; - } - else if (lua_type(l, -1) != LUA_TTABLE) { - /* invalid metatable? */ - lua_pop(l, 1); - return -1; - } - - lua_pushnil(l); - abs_index = flb_lua_absindex(l, -2); - while (lua_next(l, abs_index) != 0) { - if (lua_type(l, -2) != LUA_TSTRING) { - /* key is not a string */ - flb_debug("key is not a string"); - lua_pop(l, 1); - continue; - } - - str = lua_tolstring(l, -2, &len); /* key */ - - if (len == 4 && strncmp(str, "type", 4) == 0) { - /* data_type */ - if (lua_type(l, -1) != LUA_TNUMBER) { - /* value is not data type */ - flb_debug("type is not num. type=%s", lua_typename(l, lua_type(l, -1))); - lua_pop(l, 1); - continue; - } - meta->data_type = (int)lua_tointeger(l, -1); - } - lua_pop(l, 1); - } - lua_pop(l, 1); /* pop metatable */ - - return 0; -} - -static void lua_tomap_mpack(lua_State *l, - mpack_writer_t *writer, - int index, - struct flb_lua_l2c_config *l2cc) -{ - int len; - - len = 0; - lua_pushnil(l); - while (lua_next(l, -2) != 0) { - lua_pop(l, 1); - len++; - } - mpack_write_tag(writer, mpack_tag_map(len)); - - lua_pushnil(l); - - if (l2cc->l2c_types_num > 0) { - /* type conversion */ - while (lua_next(l, -2) != 0) { - try_to_convert_data_type_mpack(l, writer, l2cc); - lua_pop(l, 1); - } - } else { - while (lua_next(l, -2) != 0) { - flb_lua_tompack(l, writer, -1, l2cc); - flb_lua_tompack(l, writer, 0, l2cc); - lua_pop(l, 1); - } - } -} - -void flb_lua_tompack(lua_State *l, - mpack_writer_t *writer, - int index, - struct flb_lua_l2c_config *l2cc) -{ - int len; - int i; - int use_metatable = FLB_FALSE; - struct flb_lua_metadata meta; - - switch (lua_type(l, -1 + index)) { - case LUA_TSTRING: - { - const char *str; - size_t len; - - str = lua_tolstring(l, -1 + index, &len); - - mpack_write_str(writer, str, len); - } - break; - case LUA_TNUMBER: - { - if (lua_isinteger(l, -1 + index)) { - int64_t num = lua_tointeger(l, -1 + index); - mpack_write_int(writer, num); - } - else { - double num = lua_tonumber(l, -1 + index); - mpack_write_double(writer, num); - } - } - break; - case LUA_TBOOLEAN: - if (lua_toboolean(l, -1 + index)) - mpack_write_true(writer); - else - mpack_write_false(writer); - break; - case LUA_TTABLE: - flb_lua_metadata_init(&meta); - if (flb_lua_getmetatable(l, -1 + index, &meta) == 0 && - meta.data_type >= 0) { - use_metatable = FLB_TRUE; - } - if (use_metatable) { - if (meta.data_type == FLB_LUA_L2C_TYPE_ARRAY) { - /* array */ - lua_toarray_mpack(l, writer, 0, l2cc); - } - else { - /* map */ - lua_tomap_mpack(l, writer, -1 + index, l2cc); - } - break; - } - - len = flb_lua_arraylength(l, -1 + index); - if (len > 0) { - mpack_write_tag(writer, mpack_tag_array(len)); - for (i = 1; i <= len; i++) { - lua_rawgeti(l, -1, i); - flb_lua_tompack(l, writer, 0, l2cc); - lua_pop(l, 1); - } - } - else { - lua_tomap_mpack(l, writer, -1 + index, l2cc); - } - break; - case LUA_TNIL: - mpack_write_nil(writer); - break; - - case LUA_TLIGHTUSERDATA: - if (lua_touserdata(l, -1 + index) == NULL) { - mpack_write_nil(writer); - break; - } - case LUA_TFUNCTION: - case LUA_TUSERDATA: - case LUA_TTHREAD: - /* cannot serialize */ - break; - } -} - -static inline void lua_tomap_msgpack(lua_State *l, - msgpack_packer *pck, - int index, - struct flb_lua_l2c_config *l2cc) -{ - int len; - int abs_index; - - abs_index = flb_lua_absindex(l, index); - - len = 0; - lua_pushnil(l); - while (lua_next(l, abs_index) != 0) { - lua_pop(l, 1); - len++; - } - msgpack_pack_map(pck, len); - - lua_pushnil(l); - - if (l2cc->l2c_types_num > 0) { - /* type conversion */ - while (lua_next(l, abs_index) != 0) { - try_to_convert_data_type(l, pck, l2cc); - lua_pop(l, 1); - } - } else { - while (lua_next(l, abs_index) != 0) { - flb_lua_tomsgpack(l, pck, -1, l2cc); - flb_lua_tomsgpack(l, pck, 0, l2cc); - lua_pop(l, 1); - } - } -} - -void flb_lua_tomsgpack(lua_State *l, - msgpack_packer *pck, - int index, - struct flb_lua_l2c_config *l2cc) -{ - int len; - int i; - int use_metatable = FLB_FALSE; - struct flb_lua_metadata meta; - - switch (lua_type(l, -1 + index)) { - case LUA_TSTRING: - { - const char *str; - size_t len; - - str = lua_tolstring(l, -1 + index, &len); - - msgpack_pack_str(pck, len); - msgpack_pack_str_body(pck, str, len); - } - break; - case LUA_TNUMBER: - { - if (lua_isinteger(l, -1 + index)) { - int64_t num = lua_tointeger(l, -1 + index); - msgpack_pack_int64(pck, num); - } - else { - double num = lua_tonumber(l, -1 + index); - msgpack_pack_double(pck, num); - } - } - break; - case LUA_TBOOLEAN: - if (lua_toboolean(l, -1 + index)) - msgpack_pack_true(pck); - else - msgpack_pack_false(pck); - break; - case LUA_TTABLE: - flb_lua_metadata_init(&meta); - if (flb_lua_getmetatable(l, -1 + index, &meta) == 0 && - meta.data_type >= 0) { - use_metatable = FLB_TRUE; - } - if (use_metatable) { - if (meta.data_type == FLB_LUA_L2C_TYPE_ARRAY) { - /* array */ - lua_toarray_msgpack(l, pck, 0, l2cc); - } - else { - /* map */ - lua_tomap_msgpack(l, pck, -1 + index, l2cc); - } - break; - } - - len = flb_lua_arraylength(l, -1 + index); - if (len > 0) { - msgpack_pack_array(pck, len); - for (i = 1; i <= len; i++) { - lua_rawgeti(l, -1, i); - flb_lua_tomsgpack(l, pck, 0, l2cc); - lua_pop(l, 1); - } - } - else { - lua_tomap_msgpack(l, pck, -1 + index, l2cc); - } - break; - case LUA_TNIL: - msgpack_pack_nil(pck); - break; - - case LUA_TLIGHTUSERDATA: - if (lua_touserdata(l, -1 + index) == NULL) { - msgpack_pack_nil(pck); - break; - } - case LUA_TFUNCTION: - case LUA_TUSERDATA: - case LUA_TTHREAD: - /* cannot serialize */ - break; - } -} - -static void print_lua_value(FILE *out, lua_State *l, int index, int depth) -{ - int i; - int i_depth; - int type; - size_t len_s; - double val_d; - int64_t val_i; - int len_t; - - index = flb_lua_absindex(l, index); - - type = lua_type(l, index); - fprintf(out, "%s:", lua_typename(l, type)); - switch(type){ - case LUA_TSTRING: - fprintf(out, " %s\n", lua_tolstring(l,index, &len_s)); - break; - case LUA_TBOOLEAN: - fprintf(out, " %s\n", lua_toboolean(l, index) ? "true":"false"); - break; - case LUA_TNUMBER: - val_i = lua_tointeger(l, index); - val_d = lua_tonumber(l, index); - fprintf(out, " d=%lf i=%ld\n", val_d, val_i); - break; - case LUA_TTABLE: - len_t = flb_lua_arraylength(l, index); - fprintf(out, " size=%d ", len_t); - if (len_t > 0) { - fprintf(out, "array\n"); - for (i=1; i<=len_t; i++) { - for (i_depth=0; i_depth<depth; i_depth++) { - fputc(' ', stdout); - } - fprintf(out, "%03d: ", i); - lua_rawgeti(l, index, i); - print_lua_value(out, l, -1, depth+2); - lua_pop(l, 1); - } - fprintf(out, "\n"); - break; - } - - lua_pushnil(l); - fprintf(out, "map\n"); - while (lua_next(l, index) != 0) { - for (i_depth=0; i_depth<depth; i_depth++) { - fputc(' ', stdout); - } - fprintf(out, "val: "); - print_lua_value(out, l,-1, depth+2); /* val */ - for (i_depth=0; i_depth<depth; i_depth++) { - fputc(' ', stdout); - } - fprintf(out, "key: "); - print_lua_value(out, l,-2, depth+2); /* key */ - lua_pop(l, 1); /* pop value */ - } - - break; - default: - fprintf(out, " (not supported value)\n"); - } -} - -void flb_lua_dump_stack(FILE *out, lua_State *l) -{ - int top; - int i; - - top = lua_gettop(l); - if (top == 0) { - fprintf(out, "stack is empty\n"); - return; - } - fprintf(out, "top index =%d ======\n", top); - for (i=top; i>=1; i--) { - fprintf(out, "%03d: ", i); - print_lua_value(out, l, i, 2); - } - fprintf(out, "======\n"); -} diff --git a/fluent-bit/src/flb_luajit.c b/fluent-bit/src/flb_luajit.c deleted file mode 100644 index 9c8372f87..000000000 --- a/fluent-bit/src/flb_luajit.c +++ /dev/null @@ -1,97 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <fluent-bit/flb_info.h> -#include <fluent-bit/flb_mem.h> -#include <fluent-bit/flb_log.h> -#include <fluent-bit/flb_luajit.h> - -struct flb_luajit *flb_luajit_create(struct flb_config *config) -{ - struct flb_luajit *lj; - - lj = flb_malloc(sizeof(struct flb_luajit)); - if (!lj) { - flb_errno(); - return NULL; - } - - lj->state = luaL_newstate(); - if (!lj->state) { - flb_error("[luajit] error creating new context"); - flb_free(lj); - return NULL; - } - luaL_openlibs(lj->state); - lj->config = config; - mk_list_add(&lj->_head, &config->luajit_list); - - return lj; -} - -int flb_luajit_load_script(struct flb_luajit *lj, char *script) -{ - int ret; - - ret = luaL_loadfile(lj->state, script); - if (ret != 0) { - flb_error("[luajit] error loading script: %s", - lua_tostring(lj->state, -1)); - return -1; - } - - return 0; -} - -int flb_luajit_load_buffer(struct flb_luajit *lj, char *string, size_t len, char *name) -{ - int ret; - - ret = luaL_loadbuffer(lj->state, string, len, name); - if (ret != 0) { - flb_error("[luajit] error loading buffer: %s", - lua_tostring(lj->state, -1)); - return -1; - } - - return 0; -} - -void flb_luajit_destroy(struct flb_luajit *lj) -{ - lua_close(lj->state); - mk_list_del(&lj->_head); - flb_free(lj); -} - -int flb_luajit_destroy_all(struct flb_config *ctx) -{ - int c = 0; - struct mk_list *tmp; - struct mk_list *head; - struct flb_luajit *lj; - - mk_list_foreach_safe(head, tmp, &ctx->luajit_list) { - lj = mk_list_entry(head, struct flb_luajit, _head); - flb_luajit_destroy(lj); - c++; - } - - return c; -} diff --git a/fluent-bit/src/flb_meta.c b/fluent-bit/src/flb_meta.c deleted file mode 100644 index ed3f52ffc..000000000 --- a/fluent-bit/src/flb_meta.c +++ /dev/null @@ -1,79 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <fluent-bit/flb_info.h> -#include <fluent-bit/flb_env.h> -#include <fluent-bit/flb_mem.h> -#include <fluent-bit/flb_meta.h> - -/* - * A meta is a way to extend the configuration through specific commands, e.g: - * - * @SET a=b - * - * the meta command is prefixed with @, the command it self is 'SET' and have - * the parameters 'a=b'. - * - * Each command have their own handler function: meta_cmd_ABC(). - */ - -/* @SET command: register a key/value as a configuration variable */ -static int meta_cmd_set(struct flb_config *ctx, const char *params) -{ - int ret; - int len; - char *p; - char *key; - char *val; - - p = strchr(params, '='); - if (!p) { - fprintf(stderr, "[meta SET] invalid parameter '%s'\n", params); - return -1; - } - - len = strlen(params); - key = mk_string_copy_substr(params, 0, p - params); - if (!key) { - return -1; - } - - val = mk_string_copy_substr(params, (p - params) + 1, len); - if (!val) { - flb_free(key); - return -1; - } - - /* Set the variable in our local environment */ - ret = flb_env_set(ctx->env, key, val); - flb_free(key); - flb_free(val); - - return ret; -} - -/* Run a specific command */ -int flb_meta_run(struct flb_config *ctx, const char *cmd, const char *params) -{ - if (strcasecmp(cmd, "SET") == 0) { - return meta_cmd_set(ctx, params); - } - - return -1; -} diff --git a/fluent-bit/src/flb_metrics.c b/fluent-bit/src/flb_metrics.c deleted file mode 100644 index 9a9e9c7e7..000000000 --- a/fluent-bit/src/flb_metrics.c +++ /dev/null @@ -1,365 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * Metrics interface is a helper to gather general metrics from the core or - * plugins at runtime. - */ - -#include <fluent-bit/flb_info.h> -#include <fluent-bit/flb_mem.h> -#include <fluent-bit/flb_version.h> -#include <fluent-bit/flb_utils.h> -#include <fluent-bit/flb_metrics.h> -#include <msgpack.h> - -static int id_exists(int id, struct flb_metrics *metrics) -{ - struct mk_list *head; - struct flb_metric *metric; - - mk_list_foreach(head, &metrics->list) { - metric = mk_list_entry(head, struct flb_metric, _head); - if (metric->id == id) { - return FLB_TRUE; - } - } - - return FLB_FALSE; -} - -static int id_get(struct flb_metrics *metrics) -{ - int id; - int ret = FLB_FALSE; - - /* Try to use 'count' as an id */ - id = metrics->count; - - while ((ret = id_exists(id, metrics)) == FLB_TRUE) { - id++; - } - - return id; -} - -struct flb_metric *flb_metrics_get_id(int id, struct flb_metrics *metrics) -{ - struct mk_list *head; - struct flb_metric *m; - - mk_list_foreach(head, &metrics->list) { - m = mk_list_entry(head, struct flb_metric, _head); - if (m->id == id) { - return m; - } - } - - return NULL; -} - -struct flb_metrics *flb_metrics_create(const char *title) -{ - int ret; - struct flb_metrics *metrics; - - /* Create a metrics parent context */ - metrics = flb_malloc(sizeof(struct flb_metrics)); - if (!metrics) { - flb_errno(); - return NULL; - } - metrics->count = 0; - - /* Set metrics title */ - ret = flb_metrics_title(title, metrics); - if (ret == -1) { - flb_free(metrics); - return NULL; - } - - /* List head for specific metrics under the context */ - mk_list_init(&metrics->list); - return metrics; -} - -int flb_metrics_title(const char *title, struct flb_metrics *metrics) -{ - int ret; - size_t size = sizeof(metrics->title) - 1; - - ret = snprintf(metrics->title, size, "%s", title); - if (ret == -1) { - flb_errno(); - return -1; - } - else if (ret >= size){ - flb_warn("[%s] title '%s' was truncated", __FUNCTION__, title); - } - metrics->title_len = strlen(metrics->title); - return 0; -} - -int flb_metrics_add(int id, const char *title, struct flb_metrics *metrics) -{ - int ret; - struct flb_metric *m; - size_t size; - - /* Create context */ - m = flb_malloc(sizeof(struct flb_metric)); - if (!m) { - flb_errno(); - return -1; - } - m->val = 0; - size = sizeof(m->title) - 1; - - /* Write title */ - ret = snprintf(m->title, size, "%s", title); - if (ret == -1) { - flb_errno(); - flb_free(m); - return -1; - } - else if (ret >= size) { - flb_warn("[%s] title '%s' was truncated", __FUNCTION__, title); - } - - m->title_len = strlen(m->title); - - /* Assign an ID */ - if (id >= 0) { - /* Check this new ID is available */ - if (id_exists(id, metrics) == FLB_TRUE) { - flb_error("[metrics] id=%i already exists for metric '%s'", - id, metrics->title); - flb_free(m); - return -1; - } - } - else { - id = id_get(metrics); - } - - /* Link to parent list */ - mk_list_add(&m->_head, &metrics->list); - m->id = id; - metrics->count++; - - return id; -} - -int flb_metrics_sum(int id, size_t val, struct flb_metrics *metrics) -{ - struct flb_metric *m; - - m = flb_metrics_get_id(id, metrics); - if (!m) { - return -1; - } - - m->val += val; - return 0; -} - -int flb_metrics_destroy(struct flb_metrics *metrics) -{ - int count = 0; - struct mk_list *tmp; - struct mk_list *head; - struct flb_metric *m; - - mk_list_foreach_safe(head, tmp, &metrics->list) { - m = mk_list_entry(head, struct flb_metric, _head); - mk_list_del(&m->_head); - flb_free(m); - count++; - } - - flb_free(metrics); - return count; -} - -int flb_metrics_print(struct flb_metrics *metrics) -{ - struct mk_list *head; - struct flb_metric *m; - - printf("[metric dump] title => '%s'", metrics->title); - - mk_list_foreach(head, &metrics->list) { - m = mk_list_entry(head, struct flb_metric, _head); - printf(", '%s' => %lu", m->title, m->val); - } - printf("\n"); - - return 0; -} - -/* Write metrics in messagepack format */ -int flb_metrics_dump_values(char **out_buf, size_t *out_size, - struct flb_metrics *me) -{ - struct mk_list *head; - struct flb_metric *m; - msgpack_sbuffer mp_sbuf; - msgpack_packer mp_pck; - - /* Prepare new outgoing buffer */ - msgpack_sbuffer_init(&mp_sbuf); - msgpack_packer_init(&mp_pck, &mp_sbuf, msgpack_sbuffer_write); - - msgpack_pack_map(&mp_pck, me->count); - - mk_list_foreach(head, &me->list) { - m = mk_list_entry(head, struct flb_metric, _head); - msgpack_pack_str(&mp_pck, m->title_len); - msgpack_pack_str_body(&mp_pck, m->title, m->title_len); - msgpack_pack_uint64(&mp_pck, m->val); - } - - *out_buf = mp_sbuf.data; - *out_size = mp_sbuf.size; - - return 0; -} - -static int attach_uptime(struct flb_config *ctx, struct cmt *cmt, - uint64_t ts, char *hostname) -{ - double uptime; - struct cmt_counter *c; - - /* uptime */ - c = cmt_counter_create(cmt, "fluentbit", "", "uptime", - "Number of seconds that Fluent Bit has been running.", - 1, (char *[]) {"hostname"}); - if (!c) { - return -1; - } - - uptime = time(NULL) - ctx->init_time; - - cmt_counter_set(c, ts, uptime, 1, (char *[]) {hostname}); - return 0; -} - -static int attach_process_start_time_seconds(struct flb_config *ctx, - struct cmt *cmt, - uint64_t ts, char *hostname) -{ - double val; - struct cmt_gauge *g; - - g = cmt_gauge_create(cmt, "fluentbit", "", "process_start_time_seconds", - "Start time of the process since unix epoch in seconds.", - 1, (char *[]) {"hostname"}); - if (!g) { - return -1; - } - - val = (double) ctx->init_time; - cmt_gauge_set(g, ts, val, 1, (char *[]) {hostname}); - return 0; -} - -static char *get_os_name() -{ -#ifdef _WIN64 - return "win64"; -#elif _WIN32 - return "win32"; -#elif __APPLE__ || __MACH__ - return "macos"; -#elif __linux__ - return "linux"; -#elif __FreeBSD__ - return "freebsd"; -#elif __unix || __unix__ - return "unix"; -#else - return "other"; -#endif -} - -static int attach_build_info(struct flb_config *ctx, struct cmt *cmt, uint64_t ts, - char *hostname) -{ - double val; - char *os; - struct cmt_gauge *g; - - g = cmt_gauge_create(cmt, "fluentbit", "build", "info", - "Build version information.", - 3, (char *[]) {"hostname", "version", "os"}); - if (!g) { - return -1; - } - - val = (double) ctx->init_time; - os = get_os_name(); - - cmt_gauge_set(g, ts, val, 3, (char *[]) {hostname, FLB_VERSION_STR, os}); - return 0; -} - -static int attach_hot_reload_info(struct flb_config *ctx, struct cmt *cmt, uint64_t ts, - char *hostname) -{ - double val; - struct cmt_gauge *g; - - g = cmt_gauge_create(cmt, "fluentbit", "", "hot_reloaded_times", - "Collect the count of hot reloaded times.", - 1, (char *[]) {"hostname"}); - if (!g) { - return -1; - } - - val = (double) ctx->hot_reloaded_count; - - cmt_gauge_set(g, ts, val, 1, (char *[]) {hostname}); - return 0; -} - -/* Append internal Fluent Bit metrics to context */ -int flb_metrics_fluentbit_add(struct flb_config *ctx, struct cmt *cmt) -{ - int ret; - size_t ts; - char hostname[128]; - - /* current timestamp */ - ts = cfl_time_now(); - - /* get hostname */ - ret = gethostname(hostname, sizeof(hostname) - 1); - if (ret == -1) { - strcpy(hostname, "unknown"); - } - - /* Attach metrics to cmetrics context */ - attach_uptime(ctx, cmt, ts, hostname); - attach_process_start_time_seconds(ctx, cmt, ts, hostname); - attach_build_info(ctx, cmt, ts, hostname); - attach_hot_reload_info(ctx, cmt, ts, hostname); - - return 0; -} diff --git a/fluent-bit/src/flb_metrics_exporter.c b/fluent-bit/src/flb_metrics_exporter.c deleted file mode 100644 index 8560fe6ee..000000000 --- a/fluent-bit/src/flb_metrics_exporter.c +++ /dev/null @@ -1,336 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * Metrics exporter go around each Fluent Bit subsystem and collect metrics - * in a fixed interval of time. This operation is atomic and happens as one - * event handled by the main event loop. - */ - -#include <fluent-bit/flb_info.h> -#include <fluent-bit/flb_mem.h> -#include <fluent-bit/flb_utils.h> -#include <fluent-bit/flb_config.h> -#include <fluent-bit/flb_input.h> -#include <fluent-bit/flb_output.h> -#include <fluent-bit/flb_pack.h> -#include <fluent-bit/flb_http_server.h> -#include <fluent-bit/flb_storage.h> -#include <fluent-bit/flb_metrics.h> -#include <fluent-bit/flb_metrics_exporter.h> - -static int collect_inputs(msgpack_sbuffer *mp_sbuf, msgpack_packer *mp_pck, - struct flb_config *ctx) -{ - int total = 0; - size_t s; - char *buf; - struct mk_list *head; - struct flb_input_instance *i; - - msgpack_pack_str(mp_pck, 5); - msgpack_pack_str_body(mp_pck, "input", 5); - - mk_list_foreach(head, &ctx->inputs) { - i = mk_list_entry(head, struct flb_input_instance, _head); - if (!i->metrics) { - continue; - } - total++; /* FIXME: keep total number in cache */ - } - - msgpack_pack_map(mp_pck, total); - mk_list_foreach(head, &ctx->inputs) { - i = mk_list_entry(head, struct flb_input_instance, _head); - if (!i->metrics) { - continue; - } - - flb_metrics_dump_values(&buf, &s, i->metrics); - msgpack_pack_str(mp_pck, i->metrics->title_len); - msgpack_pack_str_body(mp_pck, i->metrics->title, i->metrics->title_len); - msgpack_sbuffer_write(mp_sbuf, buf, s); - flb_free(buf); - } - - return 0; -} - -static int collect_filters(msgpack_sbuffer *mp_sbuf, msgpack_packer *mp_pck, - struct flb_config *ctx) -{ - int total = 0; - size_t s; - char *buf; - struct mk_list *head; - struct flb_filter_instance *i; - - msgpack_pack_str(mp_pck, 6); - msgpack_pack_str_body(mp_pck, "filter", 6); - - mk_list_foreach(head, &ctx->filters) { - i = mk_list_entry(head, struct flb_filter_instance, _head); - if (!i->metrics) { - continue; - } - total++; - } - - msgpack_pack_map(mp_pck, total); - mk_list_foreach(head, &ctx->filters) { - i = mk_list_entry(head, struct flb_filter_instance, _head); - if (!i->metrics) { - continue; - } - - flb_metrics_dump_values(&buf, &s, i->metrics); - msgpack_pack_str(mp_pck, i->metrics->title_len); - msgpack_pack_str_body(mp_pck, i->metrics->title, i->metrics->title_len); - msgpack_sbuffer_write(mp_sbuf, buf, s); - flb_free(buf); - } - - return 0; -} - -static int collect_outputs(msgpack_sbuffer *mp_sbuf, msgpack_packer *mp_pck, - struct flb_config *ctx) -{ - int total = 0; - size_t s; - char *buf; - struct mk_list *head; - struct flb_output_instance *i; - - msgpack_pack_str(mp_pck, 6); - msgpack_pack_str_body(mp_pck, "output", 6); - - mk_list_foreach(head, &ctx->outputs) { - i = mk_list_entry(head, struct flb_output_instance, _head); - if (!i->metrics) { - continue; - } - total++; /* FIXME: keep total number in cache */ - } - - msgpack_pack_map(mp_pck, total); - mk_list_foreach(head, &ctx->outputs) { - i = mk_list_entry(head, struct flb_output_instance, _head); - if (!i->metrics) { - continue; - } - - flb_metrics_dump_values(&buf, &s, i->metrics); - msgpack_pack_str(mp_pck, i->metrics->title_len); - msgpack_pack_str_body(mp_pck, i->metrics->title, i->metrics->title_len); - msgpack_sbuffer_write(mp_sbuf, buf, s); - flb_free(buf); - } - - return 0; -} - -static int collect_metrics(struct flb_me *me) -{ - int ret; - int keys; - char *buf_data; - size_t buf_size; - struct flb_config *ctx = me->config; - struct cmt *cmt; - - /* - * msgpack buffer for old-style /v1/metrics - * ---------------------------------------- - */ - msgpack_sbuffer mp_sbuf; - msgpack_packer mp_pck; - - /* Prepare new outgoing buffer */ - msgpack_sbuffer_init(&mp_sbuf); - msgpack_packer_init(&mp_pck, &mp_sbuf, msgpack_sbuffer_write); - - keys = 3; /* input, filter, output */ - msgpack_pack_map(&mp_pck, keys); - - /* Collect metrics from input instances */ - collect_inputs(&mp_sbuf, &mp_pck, me->config); - collect_filters(&mp_sbuf, &mp_pck, me->config); - collect_outputs(&mp_sbuf, &mp_pck, me->config); - - /* - * If the built-in HTTP server is enabled, push metrics and health checks - * --------------------------------------------------------------------- - */ - -#ifdef FLB_HAVE_HTTP_SERVER - if (ctx->http_server == FLB_TRUE) { - /* /v1/metrics (old) */ - flb_hs_push_pipeline_metrics(ctx->http_ctx, mp_sbuf.data, mp_sbuf.size); - - /* /v1/health */ - if (ctx->health_check == FLB_TRUE) { - flb_hs_push_health_metrics(ctx->http_ctx, mp_sbuf.data, mp_sbuf.size); - } - - /* /v2/metrics: retrieve a CMetrics context with internal metrics */ - cmt = flb_me_get_cmetrics(ctx); - if (cmt) { - /* encode context to msgpack */ - ret = cmt_encode_msgpack_create(cmt, &buf_data, &buf_size); - if (ret == 0) { - flb_hs_push_metrics(ctx->http_ctx, buf_data, buf_size); - cmt_encode_msgpack_destroy(buf_data); - } - cmt_destroy(cmt); - } - } -#endif - - /* destroy msgpack buffer for old-style /v1/metrics */ - msgpack_sbuffer_destroy(&mp_sbuf); - - - return 0; -} - -/* Create metrics exporter context */ -struct flb_me *flb_me_create(struct flb_config *ctx) -{ - int fd; - struct mk_event *event; - struct flb_me *me; - - /* Context */ - me = flb_calloc(1, sizeof(struct flb_me)); - if (!me) { - flb_errno(); - return NULL; - } - me->config = ctx; - - /* Initialize event loop context */ - event = &me->event; - MK_EVENT_ZERO(event); - - /* Run every one second */ - fd = mk_event_timeout_create(ctx->evl, 1, 0, &me->event); - if (fd == -1) { - flb_error("[metrics_exporter] registration failed"); - flb_free(me); - return NULL; - } - me->fd = fd; - - return me; - -} - -/* Handle the event loop notification: "it's time to collect metrics" */ -int flb_me_fd_event(int fd, struct flb_me *me) -{ - if (fd != me->fd) { - return -1; - } - - flb_utils_timer_consume(fd); - collect_metrics(me); - - return 0; -} - -int flb_me_destroy(struct flb_me *me) -{ - mk_event_timeout_destroy(me->config->evl, &me->event); - flb_free(me); - return 0; -} - -/* Export all metrics as CMetrics context */ -struct cmt *flb_me_get_cmetrics(struct flb_config *ctx) -{ - int ret; - struct mk_list *head; - struct flb_input_instance *i; /* inputs */ - struct flb_filter_instance *f; /* filter */ - struct flb_output_instance *o; /* output */ - struct cmt *cmt; - - cmt = cmt_create(); - if (!cmt) { - return NULL; - } - - /* Fluent Bit metrics */ - flb_metrics_fluentbit_add(ctx, cmt); - - if (ctx->storage_metrics == FLB_TRUE) { - /* - * Storage metrics are updated in two places: - * - * - global metrics: updated by using flb_storage_metrics_update() - * - input: flb_storage callback update the metrics automatically every 5 seconds - * - * In this part, we only take care about the global storage metrics. - */ - flb_storage_metrics_update(ctx, ctx->storage_metrics_ctx); - ret = cmt_cat(cmt, ctx->storage_metrics_ctx->cmt); - if (ret == -1) { - flb_error("[metrics exporter] could not append global storage_metrics"); - cmt_destroy(cmt); - return NULL; - } - } - - /* Pipeline metrics: input, filters, outputs */ - mk_list_foreach(head, &ctx->inputs) { - i = mk_list_entry(head, struct flb_input_instance, _head); - ret = cmt_cat(cmt, i->cmt); - if (ret == -1) { - flb_error("[metrics exporter] could not append metrics from %s", - flb_input_name(i)); - cmt_destroy(cmt); - return NULL; - } - } - - mk_list_foreach(head, &ctx->filters) { - f = mk_list_entry(head, struct flb_filter_instance, _head); - ret = cmt_cat(cmt, f->cmt); - if (ret == -1) { - flb_error("[metrics exporter] could not append metrics from %s", - flb_filter_name(f)); - cmt_destroy(cmt); - return NULL; - } - } - - mk_list_foreach(head, &ctx->outputs) { - o = mk_list_entry(head, struct flb_output_instance, _head); - ret = cmt_cat(cmt, o->cmt); - if (ret == -1) { - flb_error("[metrics exporter] could not append metrics from %s", - flb_output_name(o)); - cmt_destroy(cmt); - return NULL; - } - } - - return cmt; -} diff --git a/fluent-bit/src/flb_mp.c b/fluent-bit/src/flb_mp.c deleted file mode 100644 index 50cd251c8..000000000 --- a/fluent-bit/src/flb_mp.c +++ /dev/null @@ -1,645 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <fluent-bit/flb_info.h> -#include <fluent-bit/flb_utils.h> -#include <fluent-bit/flb_mem.h> -#include <fluent-bit/flb_log.h> -#include <fluent-bit/flb_mp.h> -#include <fluent-bit/flb_slist.h> -#include <fluent-bit/flb_record_accessor.h> -#include <fluent-bit/flb_metrics.h> - -#include <msgpack.h> -#include <mpack/mpack.h> - -/* don't do this at home */ -#define pack_uint16(buf, d) _msgpack_store16(buf, (uint16_t) d) -#define pack_uint32(buf, d) _msgpack_store32(buf, (uint32_t) d) - -/* Return the number of msgpack serialized events in the buffer */ -int flb_mp_count(const void *data, size_t bytes) -{ - return flb_mp_count_remaining(data, bytes, NULL); -} - -int flb_mp_count_remaining(const void *data, size_t bytes, size_t *remaining_bytes) -{ - size_t remaining; - int count = 0; - mpack_reader_t reader; - - mpack_reader_init_data(&reader, (const char *) data, bytes); - for (;;) { - remaining = mpack_reader_remaining(&reader, NULL); - if (!remaining) { - break; - } - mpack_discard(&reader); - if (mpack_reader_error(&reader)) { - break; - } - count++; - } - - if (remaining_bytes) { - *remaining_bytes = remaining; - } - mpack_reader_destroy(&reader); - return count; -} - -int flb_mp_validate_metric_chunk(const void *data, size_t bytes, - int *out_series, size_t *processed_bytes) -{ - int ret; - int ok = CMT_DECODE_MSGPACK_SUCCESS; - int count = 0; - size_t off = 0; - size_t pre_off = 0; - struct cmt *cmt; - - while ((ret = cmt_decode_msgpack_create(&cmt, - (char *) data, bytes, &off)) == ok) { - cmt_destroy(cmt); - count++; - pre_off = off; - } - - switch (ret) { - case CMT_DECODE_MSGPACK_INVALID_ARGUMENT_ERROR: - case CMT_DECODE_MSGPACK_CORRUPT_INPUT_DATA_ERROR: - case CMT_DECODE_MSGPACK_CONSUME_ERROR: - case CMT_DECODE_MSGPACK_ENGINE_ERROR: - case CMT_DECODE_MSGPACK_PENDING_MAP_ENTRIES: - case CMT_DECODE_MSGPACK_PENDING_ARRAY_ENTRIES: - case CMT_DECODE_MSGPACK_UNEXPECTED_KEY_ERROR: - case CMT_DECODE_MSGPACK_UNEXPECTED_DATA_TYPE_ERROR: - case CMT_DECODE_MSGPACK_DICTIONARY_LOOKUP_ERROR: - case CMT_DECODE_MSGPACK_VERSION_ERROR: - goto error; - } - - if (ret == CMT_DECODE_MSGPACK_INSUFFICIENT_DATA && off == bytes) { - *out_series = count; - *processed_bytes = pre_off; - return 0; - } - -error: - *out_series = count; - *processed_bytes = pre_off; - - return -1; -} - -int flb_mp_validate_log_chunk(const void *data, size_t bytes, - int *out_records, size_t *processed_bytes) -{ - int ret; - int count = 0; - size_t off = 0; - size_t pre_off = 0; - size_t ptr_size; - unsigned char *ptr; - msgpack_object array; - msgpack_object ts; - msgpack_object header; - msgpack_object record; - msgpack_object metadata; - msgpack_unpacked result; - - msgpack_unpacked_init(&result); - while (msgpack_unpack_next(&result, data, bytes, &off) == MSGPACK_UNPACK_SUCCESS) { - array = result.data; - - if (array.type != MSGPACK_OBJECT_ARRAY) { - /* - * Sometimes there is a special case: Chunks might have extra zero - * bytes at the end of a record, meaning: no more records. This is not - * an error and actually it happens if a previous run of Fluent Bit - * was stopped/killed before to adjust the file size. - * - * Just validate if all bytes are zero, if so, adjust counters - * and return zero. - */ - ptr = (unsigned char *) (data); - ptr += pre_off; - if (ptr[0] != 0) { - goto error; - } - - ptr_size = bytes - pre_off; - ret = memcmp(ptr, ptr + 1, ptr_size - 1); - if (ret == 0) { - /* - * The chunk is valid, just let the caller know the last processed - * valid byte. - */ - msgpack_unpacked_destroy(&result); - *out_records = count; - *processed_bytes = pre_off; - return 0; - } - goto error; - } - - if (array.via.array.size != 2) { - goto error; - } - - header = array.via.array.ptr[0]; - record = array.via.array.ptr[1]; - - if (header.type == MSGPACK_OBJECT_ARRAY) { - if (header.via.array.size != 2) { - goto error; - } - - ts = header.via.array.ptr[0]; - metadata = header.via.array.ptr[1]; - - if (metadata.type != MSGPACK_OBJECT_MAP) { - goto error; - } - } - else { - ts = header; - } - - if (ts.type != MSGPACK_OBJECT_POSITIVE_INTEGER && - ts.type != MSGPACK_OBJECT_FLOAT && - ts.type != MSGPACK_OBJECT_EXT) { - goto error; - } - - if (record.type != MSGPACK_OBJECT_MAP) { - goto error; - } - - count++; - pre_off = off; - } - - msgpack_unpacked_destroy(&result); - *out_records = count; - *processed_bytes = pre_off; - return 0; - - error: - msgpack_unpacked_destroy(&result); - *out_records = count; - *processed_bytes = pre_off; - - return -1; -} - -/* Adjust a mspack header buffer size */ -void flb_mp_set_map_header_size(char *buf, int size) -{ - uint8_t h; - char *tmp = buf; - - h = tmp[0]; - if (h >> 4 == 0x8) { /* 1000xxxx */ - *tmp = (uint8_t) 0x8 << 4 | ((uint8_t) size); - } - else if (h == 0xde) { - tmp++; - pack_uint16(tmp, size); - } - else if (h == 0xdf) { - tmp++; - pack_uint32(tmp, size); - } -} - -void flb_mp_set_array_header_size(char *buf, int size) -{ - uint8_t h; - char *tmp = buf; - - h = tmp[0]; - if (h >> 4 == 0x9) { /* 1001xxxx */ - *tmp = (uint8_t) 0x9 << 4 | ((uint8_t) size); - } - else if (h == 0xdc) { - tmp++; - pack_uint16(tmp, size); - } - else if (h == 0xdd) { - tmp++; - pack_uint32(tmp, size); - } -} - -/* - * msgpack-c requires to set the number of the entries in a map beforehand. For our - * use case this adds some complexity, having developers to count all possible - * entries that might be added. - * - * As a workaround and to avoid map's recomposition over and over, this simple API - * allows to initialize the array header, 'register' new entries (as counters) and - * finalize, upon finalization the proper array header size is adjusted. - * - * To make things easier, we make sure msgpack-c always register an array type of - * 32 bits (identified by 0xdf, for number of entries >= 65536). Yes, for every - * array using this API it will use 2 more bytes, not a big ideal. So whoever - * uses this API, use it only if you don't know the exact number of entries to add. - * - * MANDATORY: make sure to always initialize, register every entry and finalize, - * otherwise you will get a corrupted or incomplete msgpack buffer. - * - * Usage example - * ============= - * - * struct flb_mp_map_header mh; - * - * flb_mp_map_header_init(&mh, mp_pck); - * - * -- First key/value entry -- - * flb_mp_map_header_append(&mh); - * msgpack_pack_str(mp_pck, 4); - * msgpack_pack_str_body(mp_pck, "cool", 4); - * msgpack_pack_true(mp_pck); - * - * -- Second key/value entry -- - * flb_mp_map_header_append(&mh); - * msgpack_pack_str(mp_pck, 4); - * msgpack_pack_str_body(mp_pck, "slow", 4); - * msgpack_pack_false(mp_pck); - * - * -- Finalize Map -- - * flb_mp_map_header_end(&mh); - */ - -static inline void mp_header_type_init(struct flb_mp_map_header *mh, - msgpack_packer *mp_pck, - int type) -{ - msgpack_sbuffer *mp_sbuf; - - mp_sbuf = (msgpack_sbuffer *) mp_pck->data; - - /* map sbuffer */ - mh->data = mp_pck->data; - - /* Reset entries */ - mh->entries = 0; - - /* Store the next byte available */ - mh->offset = mp_sbuf->size; -} - -int flb_mp_map_header_init(struct flb_mp_map_header *mh, msgpack_packer *mp_pck) -{ - /* Initialize context for a map */ - mp_header_type_init(mh, mp_pck, FLB_MP_MAP); - - /* - * Pack a map with size = 65536, so we force the underlaying msgpack-c - * to use a 32 bit buffer size (0xdf), reference: - * - * - https://github.com/msgpack/msgpack/blob/master/spec.md#map-format-family - */ - return msgpack_pack_map(mp_pck, 65536); -} - -int flb_mp_array_header_init(struct flb_mp_map_header *mh, msgpack_packer *mp_pck) -{ - /* Initialize context for a map */ - mp_header_type_init(mh, mp_pck, FLB_MP_ARRAY); - - /* - * Pack a map with size = 65536, so we force the underlaying msgpack-c - * to use a 32 bit buffer size (0xdf), reference: - * - * - https://github.com/msgpack/msgpack/blob/master/spec.md#map-format-family - */ - return msgpack_pack_array(mp_pck, 65536); -} - - -int flb_mp_map_header_append(struct flb_mp_map_header *mh) -{ - mh->entries++; - return mh->entries; -} - -int flb_mp_array_header_append(struct flb_mp_map_header *mh) -{ - mh->entries++; - return mh->entries; -} - -void flb_mp_map_header_end(struct flb_mp_map_header *mh) -{ - char *ptr; - msgpack_sbuffer *mp_sbuf; - - mp_sbuf = mh->data; - ptr = (char *) mp_sbuf->data + mh->offset; - flb_mp_set_map_header_size(ptr, mh->entries); -} - -void flb_mp_array_header_end(struct flb_mp_map_header *mh) -{ - char *ptr; - msgpack_sbuffer *mp_sbuf; - - mp_sbuf = mh->data; - ptr = (char *) mp_sbuf->data + mh->offset; - flb_mp_set_array_header_size(ptr, mh->entries); -} - -static int insert_by_subkey_count(struct flb_record_accessor *ra, struct flb_mp_accessor *mpa) -{ - int subkey_count; - int count; - struct mk_list *h; - struct flb_record_accessor *val_ra; - - /* - * sort flb_record_accessor by number of subkey - * - * e.g. - * $kubernetes - * $kubernetes[2]['a'] - * $kubernetes[2]['annotations']['fluentbit.io/tag'] - */ - subkey_count = flb_ra_subkey_count(ra); - mk_list_foreach(h, &mpa->ra_list) { - val_ra = mk_list_entry(h, struct flb_record_accessor, _head); - count = flb_ra_subkey_count(val_ra); - if (count >= subkey_count) { - mk_list_add_before(&ra->_head, &val_ra->_head, &mpa->ra_list); - return 0; - } - } - - /* add to tail of list */ - mk_list_add(&ra->_head, &mpa->ra_list); - return 0; -} - - -/* - * Create an 'mp accessor' context: this context allows to create a list of - * record accessor patterns based on a 'slist' context, where every slist string - * buffer represents a key accessor. - */ -struct flb_mp_accessor *flb_mp_accessor_create(struct mk_list *slist_patterns) -{ - size_t size; - struct mk_list *head; - struct flb_slist_entry *entry; - struct flb_record_accessor *ra; - struct flb_mp_accessor *mpa; - - /* Allocate context */ - mpa = flb_calloc(1, sizeof(struct flb_mp_accessor)); - if (!mpa) { - flb_errno(); - return NULL; - } - mk_list_init(&mpa->ra_list); - - mk_list_foreach(head, slist_patterns) { - entry = mk_list_entry(head, struct flb_slist_entry, _head); - - /* Create the record accessor context */ - ra = flb_ra_create(entry->str, FLB_TRUE); - if (!ra) { - flb_error("[mp accessor] could not create entry for pattern '%s'", - entry->str); - flb_mp_accessor_destroy(mpa); - return NULL; - } - insert_by_subkey_count(ra, mpa); - } - - if (mk_list_size(&mpa->ra_list) == 0) { - return mpa; - } - - size = sizeof(struct flb_mp_accessor_match) * mk_list_size(&mpa->ra_list); - mpa->matches_size = size; - mpa->matches = flb_calloc(1, size); - if (!mpa->matches) { - flb_errno(); - flb_mp_accessor_destroy(mpa); - return NULL; - } - - return mpa; -} - -static inline int accessor_key_find_match(struct flb_mp_accessor *mpa, - msgpack_object *key) -{ - int i; - int count; - struct flb_mp_accessor_match *match; - - count = mk_list_size(&mpa->ra_list); - for (i = 0; i < count; i++) { - match = &mpa->matches[i]; - if (match->matched == FLB_FALSE) { - continue; - } - - if (match->start_key == key) { - return i; - } - } - - return -1; -} - -static inline int accessor_sub_pack(struct flb_mp_accessor_match *match, - msgpack_packer *mp_pck, - msgpack_object *key, - msgpack_object *val) -{ - int i; - int ret; - msgpack_object *k; - msgpack_object *v; - struct flb_mp_map_header mh; - - if (match->key == key || match->key == val) { - return FLB_FALSE; - } - - if (key) { - msgpack_pack_object(mp_pck, *key); - } - - if (val->type == MSGPACK_OBJECT_MAP) { - flb_mp_map_header_init(&mh, mp_pck); - for (i = 0; i < val->via.map.size; i++) { - k = &val->via.map.ptr[i].key; - v = &val->via.map.ptr[i].val; - - ret = accessor_sub_pack(match, mp_pck, k, v); - if (ret == FLB_TRUE) { - flb_mp_map_header_append(&mh); - } - } - flb_mp_map_header_end(&mh); - } - else if (val->type == MSGPACK_OBJECT_ARRAY) { - flb_mp_array_header_init(&mh, mp_pck); - for (i = 0; i < val->via.array.size; i++) { - v = &val->via.array.ptr[i]; - ret = accessor_sub_pack(match, mp_pck, NULL, v); - if (ret == FLB_TRUE) { - flb_mp_array_header_append(&mh); - } - } - flb_mp_array_header_end(&mh); - } - else { - msgpack_pack_object(mp_pck, *val); - } - - return FLB_TRUE; -} - -/* - * Remove keys or nested keys from a map. It compose the final result in a - * new buffer. On error, it returns -1, if the map was modified it returns FLB_TRUE, - * if no modification was required it returns FLB_FALSE. - */ -int flb_mp_accessor_keys_remove(struct flb_mp_accessor *mpa, - msgpack_object *map, - void **out_buf, size_t *out_size) -{ - int i; - int ret; - int rule_id = 0; - int matches = 0; - msgpack_object *key; - msgpack_object *val; - msgpack_object *s_key; - msgpack_object *o_key; - msgpack_object *o_val; - struct mk_list *head; - struct flb_record_accessor *ra; - struct flb_mp_accessor_match *match; - struct flb_mp_map_header mh; - msgpack_sbuffer mp_sbuf; - msgpack_packer mp_pck; - - if (map->via.map.size == 0) { - return FLB_FALSE; - } - - /* Reset matches cache */ - memset(mpa->matches, '\0', mpa->matches_size); - - mk_list_foreach(head, &mpa->ra_list) { - ra = mk_list_entry(head, struct flb_record_accessor, _head); - - /* Apply the record accessor rule against the map */ - ret = flb_ra_get_kv_pair(ra, *map, &s_key, &o_key, &o_val); - if (ret == 0) { - /* There is a match, register in the matches table */ - match = &mpa->matches[rule_id]; - match->matched = FLB_TRUE; - match->start_key = s_key; /* Initial key path that matched */ - match->key = o_key; /* Final key that matched */ - match->val = o_val; /* Final value */ - match->ra = ra; /* Record accessor context */ - matches++; - } - rule_id++; - } - - /* If no matches, no modifications were made */ - if (matches == 0) { - return FLB_FALSE; - } - - /* Some rules matched, compose a new outgoing buffer */ - msgpack_sbuffer_init(&mp_sbuf); - msgpack_packer_init(&mp_pck, &mp_sbuf, msgpack_sbuffer_write); - - /* Initialize map */ - flb_mp_map_header_init(&mh, &mp_pck); - - for (i = 0; i < map->via.map.size; i++) { - key = &map->via.map.ptr[i].key; - val = &map->via.map.ptr[i].val; - - /* - * For every entry on the path, check if we should do a step-by-step - * repackaging or just pack the whole object. - * - * Just check: does this 'key' exists on any path of the record - * accessor patterns ? - * - * Find if the active key in the map, matches an accessor rule, if - * if match we get the match id as return value, otherwise -1. - */ - ret = accessor_key_find_match(mpa, key); - if (ret == -1) { - /* No matches, it's ok to pack the kv pair */ - flb_mp_map_header_append(&mh); - msgpack_pack_object(&mp_pck, *key); - msgpack_pack_object(&mp_pck, *val); - } - else { - /* The key has a match. Now we do a step-by-step packaging */ - match = &mpa->matches[ret]; - ret = accessor_sub_pack(match, &mp_pck, key, val); - if (ret == FLB_TRUE) { - flb_mp_map_header_append(&mh); - } - } - } - flb_mp_map_header_end(&mh); - - *out_buf = mp_sbuf.data; - *out_size = mp_sbuf.size; - - return FLB_TRUE; -} - -void flb_mp_accessor_destroy(struct flb_mp_accessor *mpa) -{ - struct mk_list *tmp; - struct mk_list *head; - struct flb_record_accessor *ra; - - if (!mpa) { - return; - } - - mk_list_foreach_safe(head, tmp, &mpa->ra_list) { - ra = mk_list_entry(head, struct flb_record_accessor, _head); - mk_list_del(&ra->_head); - flb_ra_destroy(ra); - } - - if (mpa->matches) { - flb_free(mpa->matches); - } - flb_free(mpa); -} diff --git a/fluent-bit/src/flb_network.c b/fluent-bit/src/flb_network.c deleted file mode 100644 index 9609e5a03..000000000 --- a/fluent-bit/src/flb_network.c +++ /dev/null @@ -1,2168 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#define _GNU_SOURCE - -#include <stdio.h> -#include <stdlib.h> -#include <string.h> -#include <sys/types.h> -#include <fcntl.h> -#include <errno.h> -#include <ctype.h> - -#ifdef FLB_SYSTEM_WINDOWS -#define poll WSAPoll -#else -#include <sys/poll.h> -#endif - -#include <fluent-bit/flb_info.h> -#include <fluent-bit/flb_compat.h> -#include <fluent-bit/flb_info.h> -#include <fluent-bit/flb_socket.h> -#include <fluent-bit/flb_mem.h> -#include <fluent-bit/flb_str.h> -#include <fluent-bit/flb_sds.h> -#include <fluent-bit/flb_network.h> -#include <fluent-bit/flb_utils.h> -#include <fluent-bit/flb_macros.h> -#include <fluent-bit/flb_upstream.h> -#include <fluent-bit/flb_scheduler.h> - -#include <monkey/mk_core.h> -#include <ares.h> - -#ifndef SOL_TCP -#define SOL_TCP IPPROTO_TCP -#endif - -static pthread_once_t local_thread_net_dns_ctx_init = PTHREAD_ONCE_INIT; -FLB_TLS_DEFINE(struct flb_net_dns, flb_net_dns_ctx); - -/* - * Initialize thread-local-storage, every worker thread has it owns - * dns context with relevant info populated inside the thread. - */ - -static void flb_net_dns_ctx_init_private() -{ - FLB_TLS_INIT(flb_net_dns_ctx); -} - -void flb_net_dns_ctx_init() -{ - pthread_once(&local_thread_net_dns_ctx_init, flb_net_dns_ctx_init_private); -} - -struct flb_net_dns *flb_net_dns_ctx_get() -{ - return FLB_TLS_GET(flb_net_dns_ctx); -} - -void flb_net_dns_ctx_set(struct flb_net_dns *dns_ctx) -{ - FLB_TLS_SET(flb_net_dns_ctx, dns_ctx); -} - -void flb_net_lib_init() -{ - int result; - - result = ares_library_init_mem(ARES_LIB_INIT_ALL, flb_malloc, flb_free, flb_realloc); - - if(0 != result) { - flb_error("[network] c-ares memory settings initialization error : %s", - ares_strerror(result)); - } -} - -void flb_net_ctx_init(struct flb_net_dns *dns_ctx) -{ - mk_list_init(&dns_ctx->lookups); - mk_list_init(&dns_ctx->lookups_drop); -} - -void flb_net_setup_init(struct flb_net_setup *net) -{ - net->dns_mode = NULL; - net->dns_resolver = NULL; - net->dns_prefer_ipv4 = FLB_FALSE; - net->keepalive = FLB_TRUE; - net->keepalive_idle_timeout = 30; - net->keepalive_max_recycle = 0; - net->accept_timeout = 10; - net->connect_timeout = 10; - net->io_timeout = 0; /* Infinite time */ - net->source_address = NULL; -} - -int flb_net_host_set(const char *plugin_name, struct flb_net_host *host, const char *address) -{ - int len; - int olen; - const char *s, *e, *u; - - memset(host, '\0', sizeof(struct flb_net_host)); - - olen = strlen(address); - if (olen == strlen(plugin_name)) { - return 0; - } - - len = strlen(plugin_name) + 3; - if (olen < len) { - return -1; - } - - s = address + len; - if (*s == '[') { - /* IPv6 address (RFC 3986) */ - e = strchr(++s, ']'); - if (!e) { - return -1; - } - host->name = flb_sds_create_len(s, e - s); - host->ipv6 = FLB_TRUE; - s = e + 1; - } - else { - e = s; - while (!(*e == '\0' || *e == ':' || *e == '/')) { - ++e; - } - if (e == s) { - return -1; - } - host->name = flb_sds_create_len(s, e - s); - s = e; - } - - if (*s == ':') { - host->port = atoi(++s); - } - - u = strchr(s, '/'); - if (u) { - host->uri = flb_uri_create(u); - } - host->address = flb_sds_create(address); - - if (host->name) { - host->listen = flb_sds_create(host->name); - } - - return 0; -} - -int flb_net_socket_reset(flb_sockfd_t fd) -{ - int status = 1; - - if (setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &status, sizeof(int)) == -1) { - flb_errno(); - return -1; - } - - return 0; -} - -int flb_net_socket_tcp_nodelay(flb_sockfd_t fd) -{ - int on = 1; - int ret; - - ret = setsockopt(fd, SOL_TCP, TCP_NODELAY, &on, sizeof(on)); - if (ret == -1) { - flb_errno(); - return -1; - } - - return 0; -} - -int flb_net_socket_nonblocking(flb_sockfd_t fd) -{ -#ifdef _WIN32 - unsigned long on = 1; - if (ioctlsocket(fd, FIONBIO, &on) != 0) { -#else - if (fcntl(fd, F_SETFL, fcntl(fd, F_GETFL, 0) | O_NONBLOCK) == -1) { -#endif - flb_errno(); - return -1; - } - - return 0; -} - -int flb_net_socket_rcv_buffer(flb_sockfd_t fd, int rcvbuf) -{ - if (setsockopt(fd, SOL_SOCKET, SO_RCVBUF, &rcvbuf, sizeof(rcvbuf)) != 0) { - flb_errno(); - return -1; - } - - return 0; -} - -int flb_net_socket_blocking(flb_sockfd_t fd) -{ -#ifdef _WIN32 - unsigned long off = 0; - if (ioctlsocket(fd, FIONBIO, &off) != 0) { -#else - if (fcntl(fd, F_SETFL, fcntl(fd, F_GETFL, 0) & ~O_NONBLOCK) == -1) { -#endif - flb_errno(); - return -1; - } - - return 0; -} - -int flb_net_socket_set_rcvtimeout(flb_sockfd_t fd, int timeout_in_seconds) -{ -#ifdef FLB_SYSTEM_WINDOWS - /* WINDOWS */ - DWORD timeout = timeout_in_seconds * 1000; - if (setsockopt(fd, SOL_SOCKET, SO_RCVTIMEO, (const char*)&timeout, sizeof timeout) - == -1) { -#else - /* LINUX and MAC OS X */ - struct timeval tv; - tv.tv_sec = timeout_in_seconds; - tv.tv_usec = 0; - if (setsockopt(fd, SOL_SOCKET, SO_RCVTIMEO, (const char*)&tv, sizeof tv) == -1) { -#endif - flb_errno(); - return -1; - } - - return 0; -} - -/* - * Enable the TCP_FASTOPEN feature for server side implemented in Linux Kernel >= 3.7, - * for more details read here: - * - * TCP Fast Open: expediting web services: http://lwn.net/Articles/508865/ - */ -int flb_net_socket_tcp_fastopen(flb_sockfd_t fd) -{ - int qlen = 5; - return setsockopt(fd, SOL_TCP, TCP_FASTOPEN, &qlen, sizeof(qlen)); -} - -flb_sockfd_t flb_net_socket_create(int family, int nonblock) -{ - flb_sockfd_t fd; - - /* create the socket and set the nonblocking flag status */ - fd = socket(family, SOCK_STREAM, 0); - if (fd == -1) { - flb_errno(); - return -1; - } - - if (nonblock) { - flb_net_socket_nonblocking(fd); - } - - return fd; -} - -flb_sockfd_t flb_net_socket_create_udp(int family, int nonblock) -{ - flb_sockfd_t fd; - - /* create the socket and set the nonblocking flag status */ - fd = socket(family, SOCK_DGRAM, 0); - if (fd == -1) { - flb_errno(); - return -1; - } - - if (nonblock) { - flb_net_socket_nonblocking(fd); - } - - return fd; -} - -/* - * Perform TCP connection for a blocking socket. This interface set's the socket - * to non-blocking mode temporary in order to add a timeout to the connection, - * the blocking mode is restored at the end. - */ -static int net_connect_sync(int fd, const struct sockaddr *addr, socklen_t addrlen, - char *host, int port, int connect_timeout) -{ - int ret; - int err; - int socket_errno; - struct pollfd pfd_read; - - /* Set socket to non-blocking mode */ - flb_net_socket_nonblocking(fd); - - /* connect(2) */ - ret = connect(fd, addr, addrlen); - if (ret == -1) { - /* - * An asynchronous connect can return -1, but what is important is the - * socket status, getting a EINPROGRESS is expected, but any other case - * means a failure. - */ -#ifdef FLB_SYSTEM_WINDOWS - socket_errno = flb_socket_error(fd); - err = 0; -#else - socket_errno = errno; - err = flb_socket_error(fd); -#endif - - if (!FLB_EINPROGRESS(socket_errno) || err != 0) { - goto exit_error; - } - - /* The connection is still in progress, implement a socket timeout */ - flb_trace("[net] connection #%i in process to %s:%i", - fd, host, port); - - /* - * Prepare a timeout using poll(2): we could use our own - * event loop mechanism for this, but it will require an - * extra file descriptor, the poll(2) call is straightforward - * for this use case. - */ - - pfd_read.fd = fd; - pfd_read.events = POLLOUT; - ret = poll(&pfd_read, 1, connect_timeout * 1000); - if (ret == 0) { - /* Timeout */ - flb_error("[net] connection #%i timeout after %i seconds to: " - "%s:%i", - fd, connect_timeout, host, port); - goto exit_error; - } - else if (ret < 0) { - /* Generic error */ - flb_errno(); - flb_error("[net] connection #%i failed to: %s:%i", - fd, host, port); - goto exit_error; - } - } - - /* - * No exception, the connection succeeded, return the normal - * non-blocking mode to the socket. - */ - flb_net_socket_blocking(fd); - return 0; - - exit_error: - flb_net_socket_blocking(fd); - return -1; -} - - -/* - * Asynchronous socket connection: this interface might be called from a co-routine, - * so in order to perform a real async connection and get notified back, it needs - * access to the event loop context and the connection context 'upstream connection. - */ -static int net_connect_async(int fd, - const struct sockaddr *addr, socklen_t addrlen, - char *host, int port, int connect_timeout, - void *async_ctx, struct flb_connection *u_conn) -{ - int ret; - int err; - int error = 0; - int socket_errno; - uint32_t mask; - char so_error_buf[256]; - char *str; - struct flb_upstream *u; - - u = u_conn->upstream; - - /* connect(2) */ - ret = connect(fd, addr, addrlen); - if (ret == 0) { - return 0; - } - - /* - * An asynchronous connect can return -1, but what is important is the - * socket status, getting a EINPROGRESS is expected, but any other case - * means a failure. - */ -#ifdef FLB_SYSTEM_WINDOWS - socket_errno = flb_socket_error(fd); - err = 0; -#else - socket_errno = errno; - err = flb_socket_error(fd); -#endif - /* The logic behind this check is that when establishing a connection - * errno should be EINPROGRESS with no additional information in order - * for it to be a healthy attempt. However, when errno is EINPROGRESS - * and an error occurs it could be saved in the so_error socket field - * which has to be accessed through getsockopt(... SO_ERROR ...) so - * in order to preserve that behavior while also properly detecting - * other errno values as error conditions the comparison was changed. - * - * Windows note : flb_socket_error returns either the value returned - * by WSAGetLastError or the value returned by getsockopt(... SO_ERROR ...) - * if WSAGetLastError returns WSAEWOULDBLOCK as per libevents code. - * - * General note : according to the connect syscall man page (not libc) - * there could be a timing issue with checking SO_ERROR here because - * the suggested use involves checking it after a select or poll call - * returns the socket as writable which is not the case here. - */ - - if (!FLB_EINPROGRESS(socket_errno) || err != 0) { - return -1; - } - - /* The connection is still in progress, implement a socket timeout */ - flb_trace("[net] connection #%i in process to %s:%i", - fd, host, port); - - /* Register the connection socket into the main event loop */ - MK_EVENT_ZERO(&u_conn->event); - - ret = mk_event_add(u_conn->evl, - fd, - FLB_ENGINE_EV_THREAD, - MK_EVENT_WRITE, - &u_conn->event); - - u_conn->event.priority = FLB_ENGINE_PRIORITY_CONNECT; - - if (ret == -1) { - /* - * If we failed here there no much that we can do, just - * let the caller know that we failed. - */ - return -1; - } - - u_conn->coroutine = async_ctx; - - /* - * Return the control to the parent caller, we need to wait for - * the event loop to get back to us. - */ - flb_coro_yield(async_ctx, FLB_FALSE); - - /* We want this field to hold NULL at all times unless we are explicitly - * waiting to be resumed. - */ - u_conn->coroutine = NULL; - - /* Save the mask before the event handler do a reset */ - mask = u_conn->event.mask; - - /* - * If the socket has been invalidated (e.g: timeout or shutdown), just - * print a debug message and return. - */ - if (u_conn->fd == -1) { - flb_debug("[net] TCP connection not longer available: %s:%i", - u->tcp_host, u->tcp_port); - return -1; - } - - /* We got a notification, remove the event registered */ - ret = mk_event_del(u_conn->evl, &u_conn->event); - if (ret == -1) { - flb_error("[io] connect event handler error"); - return -1; - } - - if (u_conn->net_error == ETIMEDOUT) { - flb_debug("[net] TCP connection timed out: %s:%i", - u->tcp_host, u->tcp_port); - return -1; - } - - /* Check the connection status */ - if (mask & MK_EVENT_WRITE) { - error = flb_socket_error(u_conn->fd); - - /* Check the exception */ - if (error != 0) { - /* - * The upstream connection might want to override the - * exception (mostly used for local timeouts: ETIMEDOUT. - */ - if (u_conn->net_error > 0) { - error = u_conn->net_error; - } - - /* Connection is broken, not much to do here */ - str = strerror_r(error, so_error_buf, sizeof(so_error_buf)); - flb_error("[net] TCP connection failed: %s:%i (%s)", - u->tcp_host, u->tcp_port, str); - return -1; - } - } - else { - flb_error("[net] TCP connection, unexpected error: %s:%i", - u->tcp_host, u->tcp_port); - return -1; - } - - return 0; -} - -static void flb_net_dns_lookup_context_destroy(struct flb_dns_lookup_context *lookup_context) -{ - mk_list_del(&lookup_context->_head); - ares_destroy(lookup_context->ares_channel); - flb_free(lookup_context); -} - -static void flb_net_dns_lookup_context_drop(struct flb_dns_lookup_context *lookup_context) -{ - if (!lookup_context->dropped) { - lookup_context->dropped = FLB_TRUE; - - mk_list_del(&lookup_context->_head); - mk_list_add(&lookup_context->_head, &lookup_context->dns_ctx->lookups_drop); - - if (lookup_context->udp_timer != NULL && - lookup_context->udp_timer->active) { - flb_sched_timer_invalidate(lookup_context->udp_timer); - - lookup_context->udp_timer = NULL; - } - } -} - -void flb_net_dns_lookup_context_cleanup(struct flb_net_dns *dns_ctx) -{ - struct flb_dns_lookup_context *lookup_context; - struct flb_coro *coroutine; - struct mk_list *head; - struct mk_list *tmp; - - mk_list_foreach_safe(head, tmp, &dns_ctx->lookups_drop) { - lookup_context = mk_list_entry(head, struct flb_dns_lookup_context, _head); - - coroutine = lookup_context->coroutine; - - flb_net_dns_lookup_context_destroy(lookup_context); - - if (coroutine != NULL) { - flb_coro_resume(coroutine); - } - } -} - -static void flb_net_free_translated_addrinfo(struct addrinfo *input) -{ - struct addrinfo *current_record; - struct addrinfo *next_record; - - if (input != NULL) { - next_record = NULL; - - for (current_record = input ; - current_record != NULL ; - current_record = next_record) { - - if (current_record->ai_addr != NULL) { - flb_free(current_record->ai_addr); - } - - next_record = current_record->ai_next; - - flb_free(current_record); - } - } -} - -static void flb_net_append_addrinfo_entry(struct addrinfo **head, - struct addrinfo **tail, - struct addrinfo *entry) -{ - if (*head == NULL) { - *head = entry; - } - else { - (*tail)->ai_next = entry; - } - - *tail = entry; -} - -static struct addrinfo *flb_net_sort_addrinfo_list(struct addrinfo *input, - int preferred_family) -{ - struct addrinfo *preferred_results_head; - struct addrinfo *remainder_results_head; - struct addrinfo *preferred_results_tail; - struct addrinfo *remainder_results_tail; - struct addrinfo *current_record; - struct addrinfo *next_record; - - remainder_results_head = NULL; - preferred_results_head = NULL; - remainder_results_tail = NULL; - preferred_results_tail = NULL; - current_record = NULL; - next_record = NULL; - - for (current_record = input ; - current_record != NULL ; - current_record = next_record) { - next_record = current_record->ai_next; - current_record->ai_next = NULL; - - if (preferred_family == current_record->ai_family) { - flb_net_append_addrinfo_entry(&preferred_results_head, - &preferred_results_tail, - current_record); - } - else - { - flb_net_append_addrinfo_entry(&remainder_results_head, - &remainder_results_tail, - current_record); - } - } - - if (preferred_results_tail != NULL) { - preferred_results_tail->ai_next = remainder_results_head; - } - - if (preferred_results_head == NULL) { - return remainder_results_head; - } - - return preferred_results_head; -} - -static struct addrinfo *flb_net_translate_ares_addrinfo(struct ares_addrinfo *input) -{ - struct addrinfo *previous_output_record; - struct addrinfo *current_output_record; - struct ares_addrinfo_node *current_ares_record; - int failure_detected; - struct addrinfo *output; - - output = NULL; - failure_detected = 0; - current_output_record = NULL; - previous_output_record = NULL; - - if (input != NULL) { - for (current_ares_record = input->nodes ; - current_ares_record != NULL ; - current_ares_record = current_ares_record->ai_next) { - - current_output_record = flb_calloc(1, sizeof(struct addrinfo)); - - if (current_output_record == NULL) { - flb_errno(); - failure_detected = 1; - break; - } - - if (output == NULL) { - output = current_output_record; - } - - current_output_record->ai_flags = current_ares_record->ai_flags; - current_output_record->ai_family = current_ares_record->ai_family; - current_output_record->ai_socktype = current_ares_record->ai_socktype; - current_output_record->ai_protocol = current_ares_record->ai_protocol; - current_output_record->ai_addrlen = current_ares_record->ai_addrlen; - - current_output_record->ai_addr = flb_malloc(current_output_record->ai_addrlen); - - if (current_output_record->ai_addr == NULL) { - flb_errno(); - failure_detected = 1; - break; - } - - memcpy(current_output_record->ai_addr, - current_ares_record->ai_addr, - current_output_record->ai_addrlen); - - if (previous_output_record != NULL) { - previous_output_record->ai_next = current_output_record; - } - - previous_output_record = current_output_record; - } - } - - if (failure_detected) { - if (output != NULL) { - flb_net_free_translated_addrinfo(output); - - output = NULL; - } - } - - return output; -} - - -static void flb_net_getaddrinfo_callback(void *arg, int status, int timeouts, - struct ares_addrinfo *res) -{ - struct flb_dns_lookup_context *lookup_context; - - lookup_context = (struct flb_dns_lookup_context *) arg; - - if (lookup_context->finished || - lookup_context->dropped) { - return; - } - - if (ARES_SUCCESS == status) { - *(lookup_context->result) = flb_net_translate_ares_addrinfo(res); - - if (*(lookup_context->result) == NULL) { - /* Translation fails only when calloc fails. */ - - *(lookup_context->result_code) = ARES_ENOMEM; - } - else { - *(lookup_context->result_code) = ARES_SUCCESS; - } - - ares_freeaddrinfo(res); - } - else { - *(lookup_context->result_code) = status; - } - - lookup_context->finished = 1; -} - -static int flb_net_getaddrinfo_event_handler(void *arg) -{ - struct flb_dns_lookup_context *lookup_context; - - lookup_context = FLB_DNS_LOOKUP_CONTEXT_FOR_EVENT(arg); - - if (lookup_context->finished || - lookup_context->dropped) { - return 0; - } - - ares_process_fd(lookup_context->ares_channel, - lookup_context->response_event.fd, - lookup_context->response_event.fd); - - if (lookup_context->finished) { - flb_net_dns_lookup_context_drop(lookup_context); - } - - return 0; -} - -static void flb_net_getaddrinfo_timeout_handler(struct flb_config *config, void *data) -{ - struct flb_dns_lookup_context *lookup_context; - - (void) config; - - lookup_context = (struct flb_dns_lookup_context *) data; - - if (lookup_context->finished || - lookup_context->dropped) { - return; - } - - *(lookup_context->udp_timeout_detected) = FLB_TRUE; - lookup_context->finished = FLB_TRUE; - lookup_context->udp_timer = NULL; - - /* We deliverately set udp_timer because we don't want flb_net_dns_lookup_context_drop - * to call flb_sched_timer_invalidate on the timer which was already disabled and - * is about to be destroyed after this this callback returns. - */ - - ares_cancel(lookup_context->ares_channel); - - *(lookup_context->result_code) = ARES_ETIMEOUT; - - flb_net_dns_lookup_context_drop(lookup_context); -} - -static ares_socket_t flb_dns_ares_socket(int af, int type, int protocol, void *userdata) -{ - struct flb_dns_lookup_context *lookup_context; - int event_mask; - ares_socket_t sockfd; - int result; - - lookup_context = (struct flb_dns_lookup_context *) userdata; - - if (lookup_context->ares_socket_created) { - /* This context already had a connection established and the code is not ready - * to handle multiple connections so we abort the process. - */ - errno = EACCES; - - return -1; - } - - sockfd = socket(af, type, protocol); - - if (sockfd == -1) { - return -1; - } - - /* According to configure_socket in ares_process.c:970 if we provide our own socket - * functions we need to set the socket up ourselves but the only specific thing we - * need is for the socket to be set to non blocking mode so that's all we do here. - */ - - result = flb_net_socket_nonblocking(sockfd); - - if (result) { - flb_socket_close(sockfd); - - return -1; - } - - lookup_context->ares_socket_type = type; - lookup_context->ares_socket_created = FLB_TRUE; - - lookup_context->response_event.mask = MK_EVENT_EMPTY; - lookup_context->response_event.status = MK_EVENT_NONE; - lookup_context->response_event.data = &lookup_context->response_event; - lookup_context->response_event.handler = flb_net_getaddrinfo_event_handler; - lookup_context->response_event.fd = sockfd; - - event_mask = MK_EVENT_READ; - - if (SOCK_STREAM == type) { - event_mask |= MK_EVENT_WRITE; - } - - result = mk_event_add(lookup_context->event_loop, sockfd, FLB_ENGINE_EV_CUSTOM, - event_mask, &lookup_context->response_event); - lookup_context->response_event.priority = FLB_ENGINE_PRIORITY_DNS; - if (result) { - flb_socket_close(sockfd); - - return -1; - } - - lookup_context->response_event.type = FLB_ENGINE_EV_CUSTOM; - lookup_context->ares_socket_registered = FLB_TRUE; - - return sockfd; -} - -static int flb_dns_ares_close(ares_socket_t sockfd, void *userdata) -{ - struct flb_dns_lookup_context *lookup_context; - int result; - - lookup_context = (struct flb_dns_lookup_context *) userdata; - - if (lookup_context->ares_socket_registered) { - lookup_context->ares_socket_registered = FLB_FALSE; - - mk_event_del(lookup_context->event_loop, &lookup_context->response_event); - } - - result = flb_socket_close(sockfd); - - return result; -} - -static int flb_dns_ares_connect(ares_socket_t sockfd, const struct sockaddr *addr, - ares_socklen_t addrlen, void *userdata) -{ - return connect(sockfd, addr, addrlen); -} - -static ares_ssize_t flb_dns_ares_recvfrom(ares_socket_t sockfd, void *data, - size_t data_len, int flags, - struct sockaddr *from, ares_socklen_t *from_len, - void *userdata) -{ - return recvfrom(sockfd, data, data_len, flags, from, from_len); -} - -static ares_ssize_t flb_dns_ares_send(ares_socket_t sockfd, const struct iovec *vec, - int len, void *userdata) -{ - return writev(sockfd, vec, len); -} - -static struct flb_dns_lookup_context *flb_net_dns_lookup_context_create( - struct flb_net_dns *dns_ctx, - struct mk_event_loop *evl, - struct flb_coro *coroutine, - char dns_mode, - int *result) -{ - struct flb_dns_lookup_context *lookup_context; - int local_result; - int optmask; - struct ares_options opts = {0}; - - local_result = 0; - optmask = 0; - - if (result == NULL) { - result = &local_result; - } - - /* The initialization order here is important since it makes it easier to handle - * failures - */ - lookup_context = flb_calloc(1, sizeof(struct flb_dns_lookup_context)); - - if (!lookup_context) { - flb_errno(); - - *result = ARES_ENOMEM; - - return NULL; - } - - /* c-ares options: Set the transport layer to the desired protocol and - * the number of retries to 2 - */ - - optmask = ARES_OPT_FLAGS; - opts.tries = 2; - - if (dns_mode == FLB_DNS_USE_TCP) { - opts.flags = ARES_FLAG_USEVC; - } - - *result = ares_init_options((ares_channel *) &lookup_context->ares_channel, - &opts, optmask); - - if (*result != ARES_SUCCESS) { - flb_free(lookup_context); - - return NULL; - } - - lookup_context->ares_socket_functions.asocket = flb_dns_ares_socket; - lookup_context->ares_socket_functions.aclose = flb_dns_ares_close; - lookup_context->ares_socket_functions.aconnect = flb_dns_ares_connect; - lookup_context->ares_socket_functions.arecvfrom = flb_dns_ares_recvfrom; - lookup_context->ares_socket_functions.asendv = flb_dns_ares_send; - lookup_context->ares_socket_created = 0; - lookup_context->event_loop = evl; - lookup_context->udp_timer = NULL; - lookup_context->coroutine = coroutine; - lookup_context->finished = 0; - lookup_context->dropped = 0; - lookup_context->dns_ctx = dns_ctx; - - ares_set_socket_functions(lookup_context->ares_channel, - &lookup_context->ares_socket_functions, - lookup_context); - - *result = ARES_SUCCESS; - - mk_list_add(&lookup_context->_head, &dns_ctx->lookups); - - return lookup_context; -} - -int flb_net_getaddrinfo(const char *node, const char *service, struct addrinfo *hints, - struct addrinfo **res, char *dns_mode_textual, int timeout) -{ - int udp_timeout_detected; - struct flb_dns_lookup_context *lookup_context; - int errno_backup; - int result_code; - struct addrinfo *result_data; - struct ares_addrinfo_hints ares_hints; - struct mk_event_loop *event_loop; - struct flb_coro *coroutine; - char dns_mode; - struct flb_net_dns *dns_ctx; - int result; - struct flb_sched *sched; - - errno_backup = errno; - - dns_mode = FLB_DNS_USE_UDP; - - if (dns_mode_textual != NULL) { - dns_mode = toupper(dns_mode_textual[0]); - } - - event_loop = flb_engine_evl_get(); - assert(event_loop != NULL); - - coroutine = flb_coro_get(); - assert(coroutine != NULL); - - dns_ctx = flb_net_dns_ctx_get(); - assert(dns_ctx != NULL); - - lookup_context = flb_net_dns_lookup_context_create(dns_ctx, event_loop, coroutine, - dns_mode, &result); - - if (result != ARES_SUCCESS) { - errno = errno_backup; - return result; - } - - lookup_context->udp_timeout_detected = &udp_timeout_detected; - lookup_context->result_code = &result_code; - lookup_context->result = &result_data; - - /* We think that either the callback or the timeout handler should be executed always - * but just in case that there is a corner case we initialize result_code with an - * error code so in case none of those is invoked (which shouldn't happen) the code - * is not ARES_SUCCESS and thus cause a NULL pointer to be returned. - */ - result_code = ARES_ESERVFAIL; - result_data = NULL; - udp_timeout_detected = 0; - - /* The timeout we get is expressed in seconds so we need to convert it to - * milliseconds - */ - timeout *= 1000; - - /* We need to ensure that our timer won't overlap with the upstream timeout handler. - */ - if (timeout > 3000) { - timeout -= 1000; - } - else { - timeout -= (timeout / 3); - } - - ares_hints.ai_flags = hints->ai_flags; - ares_hints.ai_family = hints->ai_family; - ares_hints.ai_socktype = hints->ai_socktype; - ares_hints.ai_protocol = hints->ai_protocol; - - ares_getaddrinfo(lookup_context->ares_channel, node, service, &ares_hints, - flb_net_getaddrinfo_callback, lookup_context); - - if (!lookup_context->finished) { - if (lookup_context->ares_socket_created) { - if (lookup_context->ares_socket_type == SOCK_DGRAM) { - /* If the socket type created by c-ares is UDP then we need to create our - * own timeout mechanism before yielding and cancel it if things go as - * expected. - */ - - sched = flb_sched_ctx_get(); - assert(sched != NULL); - - result = flb_sched_timer_cb_create(sched, FLB_SCHED_TIMER_CB_ONESHOT, - timeout, - flb_net_getaddrinfo_timeout_handler, - lookup_context, - &lookup_context->udp_timer); - if (result == -1) { - /* Timer creation failed, it happen because of file descriptor or memory - * exhaustion (ulimits usually) - */ - - result_code = ARES_ENOMEM; - - ares_cancel(lookup_context->ares_channel); - - lookup_context->coroutine = NULL; - - flb_net_dns_lookup_context_drop(lookup_context); - } - else { - flb_coro_yield(coroutine, FLB_FALSE); - } - } - else { - flb_coro_yield(coroutine, FLB_FALSE); - } - } - else { - /* Do we want to do anything special for this condition? */ - } - } - else { - lookup_context->coroutine = NULL; - - flb_net_dns_lookup_context_drop(lookup_context); - } - - if (!result_code) { - *res = result_data; - } - - result = result_code; - errno = errno_backup; - - return result; -} - -int flb_net_bind_address(int fd, char *source_addr) -{ - int ret; - struct addrinfo hint; - struct addrinfo *res = NULL; - struct sockaddr_storage addr; - - memset(&hint, '\0', sizeof hint); - - hint.ai_family = PF_UNSPEC; - hint.ai_flags = AI_NUMERICHOST | AI_NUMERICSERV | AI_PASSIVE; - - ret = getaddrinfo(source_addr, NULL, &hint, &res); - if (ret == -1) { - flb_errno(); - flb_error("[net] cannot read source_address=%s", source_addr); - return -1; - } - - /* Bind the address */ - memcpy(&addr, res->ai_addr, res->ai_addrlen); - freeaddrinfo(res); - ret = bind(fd, (struct sockaddr *) &addr, sizeof(addr)); - if (ret == -1) { - flb_errno(); - flb_error("[net] could not bind source_address=%s", source_addr); - return -1; - } - - return 0; -} - -static void set_ip_family(const char *host, struct addrinfo *hints) -{ - - int ret; - struct in6_addr serveraddr; - - /* check if the given 'host' is a network address, adjust ai_flags */ - ret = inet_pton(AF_INET, host, &serveraddr); - if (ret == 1) { /* valid IPv4 text address ? */ - hints->ai_family = AF_INET; - hints->ai_flags |= AI_NUMERICHOST; - } - else { - ret = inet_pton(AF_INET6, host, &serveraddr); - if (ret == 1) { /* valid IPv6 text address ? */ - hints->ai_family = AF_INET6; - hints->ai_flags |= AI_NUMERICHOST; - } - } -} - -/* Connect to a TCP socket server and returns the file descriptor */ -flb_sockfd_t flb_net_tcp_connect(const char *host, unsigned long port, - char *source_addr, int connect_timeout, - int is_async, - void *async_ctx, - struct flb_connection *u_conn) -{ - int ret; - int use_async_dns; - char resolver_initial; - flb_sockfd_t fd = -1; - char _port[6]; - char address[41]; - struct addrinfo hints; - struct addrinfo *sorted_res, *res, *rp; - - if (is_async == FLB_TRUE && !u_conn) { - flb_error("[net] invalid async mode with not set upstream connection"); - return -1; - } - - memset(&hints, 0, sizeof(hints)); - hints.ai_family = AF_UNSPEC; - hints.ai_socktype = SOCK_STREAM; - - /* Set hints */ - set_ip_family(host, &hints); - - /* fomart the TCP port */ - snprintf(_port, sizeof(_port), "%lu", port); - - use_async_dns = is_async; - - if (u_conn->net->dns_resolver != NULL) { - resolver_initial = toupper(u_conn->net->dns_resolver[0]); - - if (resolver_initial == FLB_DNS_LEGACY) { - use_async_dns = FLB_FALSE; - } - } - - /* retrieve DNS info */ - if (use_async_dns) { - ret = flb_net_getaddrinfo(host, _port, &hints, &res, - u_conn->net->dns_mode, - connect_timeout); - } - else { - ret = getaddrinfo(host, _port, &hints, &res); - } - - if (ret) { - if (use_async_dns) { - flb_warn("[net] getaddrinfo(host='%s', err=%d): %s", host, ret, ares_strerror(ret)); - } - else { - flb_warn("[net] getaddrinfo(host='%s', err=%d): %s", host, ret, gai_strerror(ret)); - } - - return -1; - } - - if (u_conn->net_error > 0) { - if (u_conn->net_error == ETIMEDOUT) { - flb_warn("[net] timeout detected between DNS lookup and connection attempt"); - } - - if (use_async_dns) { - flb_net_free_translated_addrinfo(res); - } - else { - freeaddrinfo(res); - } - - return -1; - } - - sorted_res = res; - - if (u_conn->net->dns_prefer_ipv4) { - sorted_res = flb_net_sort_addrinfo_list(res, AF_INET); - - if (sorted_res == NULL) { - flb_debug("[net] error sorting getaddrinfo results"); - - if (use_async_dns) { - flb_net_free_translated_addrinfo(res); - } - else { - freeaddrinfo(res); - } - - return -1; - } - } - - /* - * Try to connect: on this iteration we try to connect to the first - * available address. - */ - for (rp = sorted_res; rp != NULL; rp = rp->ai_next) { - if (u_conn->net_error > 0) { - if (u_conn->net_error == ETIMEDOUT) { - flb_warn("[net] timeout detected between connection attempts"); - } - } - - /* create socket */ - fd = socket(rp->ai_family, rp->ai_socktype, rp->ai_protocol); - if (fd == -1) { - flb_error("[net] coult not create client socket, retrying"); - continue; - } - - /* asynchronous socket ? */ - if (is_async == FLB_TRUE) { - flb_net_socket_nonblocking(fd); - } - - /* Bind a specific network interface ? */ - if (source_addr != NULL) { - ret = flb_net_bind_address(fd, source_addr); - - if (ret == -1) { - flb_warn("[net] falling back to random interface"); - } - else { - flb_trace("[net] client connect bind address: %s", source_addr); - } - } - - /* Disable Nagle's algorithm */ - flb_net_socket_tcp_nodelay(fd); - - /* Set receive timeout */ - flb_net_socket_set_rcvtimeout(fd, u_conn->net->io_timeout); - - if (u_conn) { - u_conn->fd = fd; - u_conn->event.fd = fd; - } - - flb_connection_set_remote_host(u_conn, rp->ai_addr); - - /* Perform TCP connection */ - if (is_async == FLB_TRUE) { - ret = net_connect_async(fd, rp->ai_addr, rp->ai_addrlen, - (char *) host, port, connect_timeout, - async_ctx, u_conn); - - } - else { - ret = net_connect_sync(fd, rp->ai_addr, rp->ai_addrlen, - (char *) host, port, connect_timeout); - } - - if (u_conn->net_error == ETIMEDOUT) { - /* flb_upstream_conn_timeouts called prepare_destroy_conn which - * closed the file descriptor and removed it from the event so - * we can safely ignore it. - */ - - fd = -1; - - break; - } - - if (ret == -1) { - address[0] = '\0'; - - ret = flb_net_address_to_str(rp->ai_family, rp->ai_addr, - address, sizeof(address)); - - /* If the connection failed, just abort and report the problem */ - flb_debug("[net] socket #%i could not connect to %s:%s", - fd, address, _port); - - if (u_conn) { - u_conn->fd = -1; - u_conn->event.fd = -1; - } - - flb_socket_close(fd); - fd = -1; - - continue; - } - - break; - } - - if (fd == -1) { - flb_debug("[net] could not connect to %s:%s", - host, _port); - } - - if (use_async_dns) { - flb_net_free_translated_addrinfo(res); - } - else { - freeaddrinfo(res); - } - - if (rp == NULL) { - return -1; - } - - return fd; -} - -/* "Connect" to a UDP socket server and returns the file descriptor */ -flb_sockfd_t flb_net_udp_connect(const char *host, unsigned long port, - char *source_addr) -{ - int ret; - flb_sockfd_t fd = -1; - char _port[6]; - struct addrinfo hints; - struct addrinfo *res, *rp; - - memset(&hints, 0, sizeof(hints)); - hints.ai_family = AF_UNSPEC; - hints.ai_socktype = SOCK_DGRAM; - - /* Set hints */ - set_ip_family(host, &hints); - - /* Format UDP port */ - snprintf(_port, sizeof(_port), "%lu", port); - - /* retrieve DNS info */ - ret = getaddrinfo(host, _port, &hints, &res); - if (ret != 0) { - flb_warn("net]: getaddrinfo(host='%s'): %s", - host, gai_strerror(ret)); - return -1; - } - - for (rp = res; rp != NULL; rp = rp->ai_next) { - /* create socket */ - fd = socket(res->ai_family, res->ai_socktype, res->ai_protocol); - if (fd == -1) { - flb_error("[net] coult not create client socket, retrying"); - continue; - } - - /* Bind a specific network interface ? */ - if (source_addr != NULL) { - ret = flb_net_bind_address(fd, source_addr); - if (ret == -1) { - flb_warn("[net] falling back to random interface"); - } - else { - flb_trace("[net] client connect bind address: %s", source_addr); - } - } - - /* - * Why do we connect(2) an UDP socket ?, is this useful ?: Yes. Despite - * an UDP socket it's not in a connection state, connecting through the - * API it helps the Kernel to configure the destination address and - * is totally valid, so then you don't need to use sendto(2). - * - * For our use case this is quite helpful, since the caller keeps using - * the same Fluent Bit I/O API to deliver a message. - */ - if (connect(fd, rp->ai_addr, rp->ai_addrlen) == -1) { - flb_error("[net] UDP socket %i could connect to %s:%s", - fd, host, _port); - flb_socket_close(fd); - fd = -1; - break; - } - break; - } - - freeaddrinfo(res); - - if (rp == NULL) { - return -1; - } - - return fd; -} - -/* Connect to a TCP socket server and returns the file descriptor */ -int flb_net_tcp_fd_connect(flb_sockfd_t fd, const char *host, unsigned long port) -{ - int ret; - struct addrinfo hints; - struct addrinfo *res; - char _port[6]; - - memset(&hints, 0, sizeof hints); - hints.ai_family = AF_UNSPEC; - hints.ai_socktype = SOCK_STREAM; - - snprintf(_port, sizeof(_port), "%lu", port); - ret = getaddrinfo(host, _port, &hints, &res); - if (ret != 0) { - flb_warn("net_tcp_fd_connect: getaddrinfo(host='%s'): %s", - host, gai_strerror(ret)); - return -1; - } - - ret = connect(fd, res->ai_addr, res->ai_addrlen); - freeaddrinfo(res); - - return ret; -} - -flb_sockfd_t flb_net_server(const char *port, const char *listen_addr) -{ - flb_sockfd_t fd = -1; - int ret; - struct addrinfo hints; - struct addrinfo *res, *rp; - - memset(&hints, 0, sizeof hints); - hints.ai_family = AF_UNSPEC; - hints.ai_socktype = SOCK_STREAM; - hints.ai_flags = AI_PASSIVE; - - ret = getaddrinfo(listen_addr, port, &hints, &res); - if (ret != 0) { - flb_warn("net_server: getaddrinfo(listen='%s:%s'): %s", - listen_addr, port, gai_strerror(ret)); - return -1; - } - - for (rp = res; rp != NULL; rp = rp->ai_next) { - fd = flb_net_socket_create(rp->ai_family, 1); - if (fd == -1) { - flb_error("Error creating server socket, retrying"); - continue; - } - - flb_net_socket_tcp_nodelay(fd); - flb_net_socket_reset(fd); - - ret = flb_net_bind(fd, rp->ai_addr, rp->ai_addrlen, 128); - if(ret == -1) { - flb_warn("Cannot listen on %s port %s", listen_addr, port); - flb_socket_close(fd); - continue; - } - break; - } - freeaddrinfo(res); - - if (rp == NULL) { - return -1; - } - - return fd; -} - -flb_sockfd_t flb_net_server_udp(const char *port, const char *listen_addr) -{ - flb_sockfd_t fd = -1; - int ret; - struct addrinfo hints; - struct addrinfo *res, *rp; - - memset(&hints, 0, sizeof hints); - hints.ai_family = AF_UNSPEC; - hints.ai_socktype = SOCK_DGRAM; - hints.ai_flags = AI_PASSIVE; - - ret = getaddrinfo(listen_addr, port, &hints, &res); - if (ret != 0) { - flb_warn("net_server_udp: getaddrinfo(listen='%s:%s'): %s", - listen_addr, port, gai_strerror(ret)); - return -1; - } - - for (rp = res; rp != NULL; rp = rp->ai_next) { - fd = flb_net_socket_create_udp(rp->ai_family, 0); - if (fd == -1) { - flb_error("Error creating server socket, retrying"); - continue; - } - - ret = flb_net_bind_udp(fd, rp->ai_addr, rp->ai_addrlen); - if(ret == -1) { - flb_warn("Cannot listen on %s port %s", listen_addr, port); - flb_socket_close(fd); - continue; - } - break; - } - freeaddrinfo(res); - - if (rp == NULL) { - return -1; - } - - return fd; -} - -#ifdef FLB_HAVE_UNIX_SOCKET -flb_sockfd_t flb_net_server_unix(const char *listen_path, - int stream_mode, - int backlog) -{ - size_t address_length; - size_t path_length; - struct sockaddr_un address; - int ret; - flb_sockfd_t fd; - - if (stream_mode) { - fd = flb_net_socket_create(AF_UNIX, FLB_TRUE); - } - else { - fd = flb_net_socket_create_udp(AF_UNIX, FLB_TRUE); - } - - if (fd != -1) { - memset(&address, 0, sizeof(struct sockaddr_un)); - - path_length = strlen(listen_path); - - address_length = offsetof(struct sockaddr_un, sun_path) + - path_length + - 1; - - address.sun_family = AF_UNIX; - - strncpy(address.sun_path, listen_path, sizeof(address.sun_path)); - - if (stream_mode) { - ret = flb_net_bind(fd, - (const struct sockaddr *) &address, - address_length, - backlog); - } - else { - ret = flb_net_bind_udp(fd, - (const struct sockaddr *) &address, - address_length); - } - - if(ret == -1) { - flb_warn("Cannot bind to or listen on %s", listen_path); - - flb_socket_close(fd); - } - } - else { - flb_error("Error creating server socket"); - } - - return fd; -} -#else -flb_sockfd_t flb_net_server_unix(const char *listen_path, - int stream_mode, - int backlog) -{ - flb_error("Unix sockets are not available in this platform"); - - return -1; -} -#endif - -int flb_net_bind(flb_sockfd_t fd, const struct sockaddr *addr, - socklen_t addrlen, int backlog) -{ - int ret; - - ret = bind(fd, addr, addrlen); - if( ret == -1 ) { - flb_error("Error binding socket"); - return ret; - } - - ret = listen(fd, backlog); - if(ret == -1 ) { - flb_error("Error setting up the listener"); - return -1; - } - - return ret; -} - -int flb_net_bind_udp(flb_sockfd_t fd, const struct sockaddr *addr, - socklen_t addrlen) -{ - int ret; - - ret = bind(fd, addr, addrlen); - if( ret == -1 ) { - flb_error("Error binding socket"); - return ret; - } - - return ret; -} - -flb_sockfd_t flb_net_accept(flb_sockfd_t server_fd) -{ - flb_sockfd_t remote_fd; - struct sockaddr sock_addr; - socklen_t socket_size = sizeof(struct sockaddr); - - // return accept(server_fd, &sock_addr, &socket_size); - -#ifdef FLB_HAVE_ACCEPT4 - remote_fd = accept4(server_fd, &sock_addr, &socket_size, - SOCK_NONBLOCK | SOCK_CLOEXEC); -#else - remote_fd = accept(server_fd, &sock_addr, &socket_size); - flb_net_socket_nonblocking(remote_fd); -#endif - - if (remote_fd == -1) { - perror("accept4"); - } - - return remote_fd; -} - -int flb_net_address_to_str(int family, const struct sockaddr *addr, - char *output_buffer, size_t output_buffer_size) -{ - struct sockaddr *proper_addr; - const char *result; - - if (family == AF_INET) { - proper_addr = (struct sockaddr *) &((struct sockaddr_in *) addr)->sin_addr; - } - else if (family == AF_INET6) { - proper_addr = (struct sockaddr *) &((struct sockaddr_in6 *) addr)->sin6_addr; - } - else { - strncpy(output_buffer, - "CONVERSION ERROR 1", - output_buffer_size); - - return -1; - } - - result = inet_ntop(family, proper_addr, output_buffer, output_buffer_size); - - if (result == NULL) { - strncpy(output_buffer, - "CONVERSION ERROR 2", - output_buffer_size); - - return -2; - } - - return 0; -} - -#ifdef FLB_COMPILE_UNUSED_FUNCTIONS -static int net_socket_get_local_address(flb_sockfd_t fd, - struct sockaddr_storage *address) -{ - socklen_t buffer_size; - int result; - - buffer_size = sizeof(struct sockaddr_storage); - - result = getsockname(fd, (struct sockaddr *) &address, &buffer_size); - - if (result == -1) { - return -1; - } - - return 0; -} -#endif - -static int net_socket_get_peer_address(flb_sockfd_t fd, - struct sockaddr_storage *address) -{ - socklen_t buffer_size; - int result; - - buffer_size = sizeof(struct sockaddr_storage); - - result = getpeername(fd, (struct sockaddr *) address, &buffer_size); - - if (result == -1) { - return -1; - } - - return 0; -} - -static unsigned short int net_address_port(struct sockaddr_storage *address) -{ - unsigned short int port; - - if (address->ss_family == AF_INET) { - port = ((struct sockaddr_in *) address)->sin_port; - } - else if (address->ss_family == AF_INET6) { - port = ((struct sockaddr_in6 *) address)->sin6_port; - } - else { - port = 0; - } - - return ntohs(port); -} - -#ifdef FLB_HAVE_UNIX_SOCKET -static int net_address_unix_socket_peer_pid_raw(flb_sockfd_t fd, - struct sockaddr_storage *address, - char *output_buffer, - int output_buffer_size, - size_t *output_data_size) -{ -#if !defined(FLB_SYSTEM_MACOS) && !defined(FLB_SYSTEM_FREEBSD) - unsigned int peer_credentials_size; - struct ucred peer_credentials; -#endif - size_t required_buffer_size; - int result = 0; - - if (address->ss_family != AF_UNIX) { - return -1; - } - - required_buffer_size = 11; /* maximum 32 bit signed integer */ - required_buffer_size += 1; /* string terminator */ - - if (required_buffer_size > output_buffer_size) { - return -1; - } - -#if !defined(FLB_SYSTEM_MACOS) && !defined(FLB_SYSTEM_FREEBSD) - peer_credentials_size = sizeof(struct ucred); - - result = getsockopt(fd, - SOL_SOCKET, - SO_PEERCRED, - &peer_credentials, - &peer_credentials_size); - - if (result != -1) { - *output_data_size = snprintf(output_buffer, - output_buffer_size, - "%ld", - (long) peer_credentials.pid); - } -#else - *output_data_size = snprintf(output_buffer, - output_buffer_size, - FLB_NETWORK_ADDRESS_UNAVAILABLE); -#endif - - return result; -} - -static int net_address_unix_socket_peer_pid_str(flb_sockfd_t fd, - struct sockaddr_storage *address, - char *output_buffer, - int output_buffer_size, - size_t *output_data_size) -{ - size_t required_buffer_size; - size_t peer_pid_length; - char peer_pid[12]; - int result; - - if (address->ss_family != AF_UNIX) { - return -1; - } - - result = net_address_unix_socket_peer_pid_raw(fd, - address, - peer_pid, - sizeof(peer_pid), - &peer_pid_length); - - if (result != 0) { - return -1; - } - - required_buffer_size = strlen(FLB_NETWORK_UNIX_SOCKET_PEER_ADDRESS_TEMPLATE); - required_buffer_size += peer_pid_length; - required_buffer_size -= 2; /* format string specifiers */ - required_buffer_size += 1; /* string terminator */ - - if (required_buffer_size > output_buffer_size) { - *output_data_size = required_buffer_size; - - return -1; - } - - *output_data_size = snprintf(output_buffer, - output_buffer_size, - FLB_NETWORK_UNIX_SOCKET_PEER_ADDRESS_TEMPLATE, - peer_pid); - - return 0; -} -#endif - -size_t flb_network_address_size(struct sockaddr_storage *address) -{ - if (address->ss_family == AF_INET) { - return sizeof(struct sockaddr_in); - } - else if (address->ss_family == AF_INET6) { - return sizeof(struct sockaddr_in6); - } -#ifdef FLB_HAVE_UNIX_SOCKET - else if (address->ss_family == AF_UNIX) { - return sizeof(struct sockaddr_un); - } -#endif - - return 0; -} - -static int net_address_ip_raw(flb_sockfd_t fd, - struct sockaddr_storage *address, - char *output_buffer, - int output_buffer_size, - size_t *output_data_size) -{ - char peer_pid[12]; - char *address_data; - size_t address_size; - int result; - - errno = 0; - - if (address->ss_family == AF_UNSPEC) { - flb_debug("socket_ip_raw: uninitialized address"); - - return -1; - } - if (address->ss_family == AF_INET) { - address_data = ((char *) &((struct sockaddr_in *) address)->sin_addr); - address_size = sizeof(struct in_addr); - } - else if (address->ss_family == AF_INET6) { - address_data = ((char *) &((struct sockaddr_in6 *) address)->sin6_addr); - address_size = sizeof(struct in6_addr); - } -#ifdef FLB_HAVE_UNIX_SOCKET - else if (address->ss_family == AF_UNIX) { - result = net_address_unix_socket_peer_pid_raw(fd, - address, - peer_pid, - sizeof(peer_pid), - &address_size); - - if (result != 0) { - flb_debug("socket_ip_raw: error getting client process pid"); - - return -1; - } - - address_data = peer_pid; - } -#endif - else { - flb_debug("socket_ip_raw: unsupported address type (%i)", - address->ss_family); - - return -1; - } - - if (output_buffer_size < address_size) { - flb_debug("socket_ip_raw: insufficient buffer size (%i < %zu)", - output_buffer_size, address_size); - - return -1; - } - - memcpy(output_buffer, address_data, address_size); - - if (output_data_size != NULL) { - *output_data_size = address_size; - } - - return 0; -} - -static int net_address_ip_str(flb_sockfd_t fd, - struct sockaddr_storage *address, - char *output_buffer, - int output_buffer_size, - size_t *output_data_size) -{ - void *address_data; - int result; - - errno = 0; - - if (address->ss_family == AF_UNSPEC) { - *output_data_size = snprintf(output_buffer, - output_buffer_size, - FLB_NETWORK_ADDRESS_UNAVAILABLE); - - return 0; - } - else if (address->ss_family == AF_INET) { - address_data = (void *) &((struct sockaddr_in *) address)->sin_addr; - } - else if (address->ss_family == AF_INET6) { - address_data = (void *) &((struct sockaddr_in6 *) address)->sin6_addr; - } -#ifdef FLB_HAVE_UNIX_SOCKET - else if (address->ss_family == AF_UNIX) { - result = net_address_unix_socket_peer_pid_str(fd, - address, - output_buffer, - output_buffer_size, - output_data_size); - - if (result != 0) { - flb_debug("socket_ip_str: error getting client process pid"); - } - - return result; - } -#endif - else { - flb_debug("socket_ip_str: unsupported address type (%i)", - address->ss_family); - - return -1; - } - - if ((inet_ntop(address->ss_family, - address_data, - output_buffer, - output_buffer_size)) == NULL) { - flb_debug("socket_ip_str: Can't get the IP text form (%i)", errno); - - return -1; - } - - *output_data_size = strlen(output_buffer); - - return 0; -} - -int flb_net_socket_peer_address(flb_sockfd_t fd, - struct sockaddr_storage *output_buffer) -{ - return net_socket_get_peer_address(fd, output_buffer); -} - -int flb_net_socket_address_info(flb_sockfd_t fd, - struct sockaddr_storage *address, - unsigned short int *port_output_buffer, - char *str_output_buffer, - int str_output_buffer_size, - size_t *str_output_data_size) -{ - int result; - - result = net_address_ip_str(fd, address, - str_output_buffer, - str_output_buffer_size, - str_output_data_size); - - if (result == 0) { - if (port_output_buffer != NULL) { - *port_output_buffer = net_address_port(address); - } - } - - return result; -} - -int flb_net_socket_ip_peer_str(flb_sockfd_t fd, - char *output_buffer, - int output_buffer_size, - size_t *output_data_size, - int *output_address_family) -{ - struct sockaddr_storage address; - int result; - - result = net_socket_get_peer_address(fd, &address); - - if (result != 0) { - return -1; - } - - if (address.ss_family == AF_UNIX) { - - } - - result = net_address_ip_str(fd, &address, - output_buffer, - output_buffer_size, - output_data_size); - - if (result == 0) { - if (output_address_family != NULL) { - *output_address_family = address.ss_family; - } - } - - return result; -} - -int flb_net_socket_peer_ip_raw(flb_sockfd_t fd, - char *output_buffer, - int output_buffer_size, - size_t *output_data_size, - int *output_address_family) -{ - struct sockaddr_storage address; - int result; - - result = net_socket_get_peer_address(fd, &address); - - if (result != 0) { - return -1; - } - - result = net_address_ip_raw(fd, &address, - output_buffer, - output_buffer_size, - output_data_size); - - if (result == 0) { - if (output_address_family != NULL) { - *output_address_family = address.ss_family; - } - } - - return result; -} - -int flb_net_socket_peer_port(flb_sockfd_t fd, - unsigned short int *output_buffer) -{ - struct sockaddr_storage address; - int result; - - result = net_socket_get_peer_address(fd, &address); - - if (result != 0) { - return -1; - } - - *output_buffer = net_address_port(&address); - - return 0; -} - -int flb_net_socket_peer_info(flb_sockfd_t fd, - unsigned short int *port_output_buffer, - struct sockaddr_storage *raw_output_buffer, - char *str_output_buffer, - int str_output_buffer_size, - size_t *str_output_data_size) -{ - struct sockaddr_storage address; - int result; - - result = net_socket_get_peer_address(fd, &address); - - if (result != 0) { - return -1; - } - - memcpy(raw_output_buffer, - &address, - sizeof(struct sockaddr_storage)); - - return flb_net_socket_address_info(fd, - &address, - port_output_buffer, - str_output_buffer, - str_output_buffer_size, - str_output_data_size); -} diff --git a/fluent-bit/src/flb_oauth2.c b/fluent-bit/src/flb_oauth2.c deleted file mode 100644 index b507adc87..000000000 --- a/fluent-bit/src/flb_oauth2.c +++ /dev/null @@ -1,437 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <fluent-bit/flb_info.h> -#include <fluent-bit/flb_mem.h> -#include <fluent-bit/flb_log.h> -#include <fluent-bit/flb_utils.h> -#include <fluent-bit/flb_oauth2.h> -#include <fluent-bit/flb_upstream.h> -#include <fluent-bit/flb_http_client.h> -#include <fluent-bit/flb_jsmn.h> - -#define free_temporary_buffers() \ - if (prot) { \ - flb_free(prot); \ - } \ - if (host) { \ - flb_free(host); \ - } \ - if (port) { \ - flb_free(port); \ - } \ - if (uri) { \ - flb_free(uri); \ - } - -static inline int key_cmp(const char *str, int len, const char *cmp) { - - if (strlen(cmp) != len) { - return -1; - } - - return strncasecmp(str, cmp, len); -} - -int flb_oauth2_parse_json_response(const char *json_data, size_t json_size, - struct flb_oauth2 *ctx) -{ - int i; - int ret; - int key_len; - int val_len; - int tokens_size = 32; - const char *key; - const char *val; - jsmn_parser parser; - jsmntok_t *t; - jsmntok_t *tokens; - - jsmn_init(&parser); - tokens = flb_calloc(1, sizeof(jsmntok_t) * tokens_size); - if (!tokens) { - flb_errno(); - return -1; - } - - ret = jsmn_parse(&parser, json_data, json_size, tokens, tokens_size); - if (ret <= 0) { - flb_error("[oauth2] cannot parse payload:\n%s", json_data); - flb_free(tokens); - return -1; - } - - t = &tokens[0]; - if (t->type != JSMN_OBJECT) { - flb_error("[oauth2] invalid JSON response:\n%s", json_data); - flb_free(tokens); - return -1; - } - - /* Parse JSON tokens */ - for (i = 1; i < ret; i++) { - t = &tokens[i]; - - if (t->type != JSMN_STRING) { - continue; - } - - if (t->start == -1 || t->end == -1 || (t->start == 0 && t->end == 0)){ - break; - } - - /* Key */ - key = json_data + t->start; - key_len = (t->end - t->start); - - /* Value */ - i++; - t = &tokens[i]; - val = json_data + t->start; - val_len = (t->end - t->start); - - if (key_cmp(key, key_len, "access_token") == 0) { - ctx->access_token = flb_sds_create_len(val, val_len); - } - else if (key_cmp(key, key_len, "token_type") == 0) { - ctx->token_type = flb_sds_create_len(val, val_len); - } - else if (key_cmp(key, key_len, "expires_in") == 0) { - ctx->expires_in = atol(val); - - /* - * Our internal expiration time must be lower that the one set - * by the remote end-point, so we can use valid cached values - * if a token renewal is in place. So we decrease the expire - * interval -10%. - */ - ctx->expires_in -= (ctx->expires_in * 0.10); - } - } - - flb_free(tokens); - if (!ctx->access_token || !ctx->token_type || ctx->expires_in < 60) { - flb_sds_destroy(ctx->access_token); - flb_sds_destroy(ctx->token_type); - ctx->expires_in = 0; - return -1; - } - - return 0; -} - -struct flb_oauth2 *flb_oauth2_create(struct flb_config *config, - const char *auth_url, int expire_sec) -{ - int ret; - char *prot = NULL; - char *host = NULL; - char *port = NULL; - char *uri = NULL; - struct flb_oauth2 *ctx; - - /* allocate context */ - ctx = flb_calloc(1, sizeof(struct flb_oauth2)); - if (!ctx) { - flb_errno(); - return NULL; - } - - /* register token url */ - ctx->auth_url = flb_sds_create(auth_url); - if (!ctx->auth_url) { - flb_errno(); - flb_free(ctx); - return NULL; - } - - /* default payload size to 1kb */ - ctx->payload = flb_sds_create_size(1024); - if (!ctx->payload) { - flb_errno(); - flb_oauth2_destroy(ctx); - return NULL; - } - - ctx->issued = time(NULL); - ctx->expires = ctx->issued + expire_sec; - - /* Parse and split URL */ - ret = flb_utils_url_split(auth_url, &prot, &host, &port, &uri); - if (ret == -1) { - flb_error("[oauth2] invalid URL: %s", auth_url); - goto error; - } - - if (!prot || strcmp(prot, "https") != 0) { - flb_error("[oauth2] invalid endpoint protocol: %s", auth_url); - goto error; - } - - if (!host) { - flb_error("[oauth2] invalid URL host: %s", auth_url); - goto error; - } - - /* Populate context */ - ctx->host = flb_sds_create(host); - if (!ctx->host) { - flb_errno(); - goto error; - } - if (port) { - ctx->port = flb_sds_create(port); - } - else { - ctx->port = flb_sds_create(FLB_OAUTH2_PORT); - } - if (!ctx->port) { - flb_errno(); - goto error; - } - ctx->uri = flb_sds_create(uri); - if (!ctx->uri) { - flb_errno(); - goto error; - } - - /* Create TLS context */ - ctx->tls = flb_tls_create(FLB_TLS_CLIENT_MODE, - FLB_TRUE, /* verify */ - -1, /* debug */ - NULL, /* vhost */ - NULL, /* ca_path */ - NULL, /* ca_file */ - NULL, /* crt_file */ - NULL, /* key_file */ - NULL); /* key_passwd */ - if (!ctx->tls) { - flb_error("[oauth2] error initializing TLS context"); - goto error; - } - - /* Create Upstream context */ - ctx->u = flb_upstream_create_url(config, auth_url, - FLB_IO_TLS, ctx->tls); - if (!ctx->u) { - flb_error("[oauth2] error creating upstream context"); - goto error; - } - - /* Remove Upstream Async flag */ - flb_stream_disable_async_mode(&ctx->u->base); - - free_temporary_buffers(); - return ctx; - - error: - free_temporary_buffers(); - flb_oauth2_destroy(ctx); - - return NULL; -} - -/* Clear the current payload and token */ -void flb_oauth2_payload_clear(struct flb_oauth2 *ctx) -{ - flb_sds_len_set(ctx->payload, 0); - ctx->payload[0] = '\0'; - ctx->expires_in = 0; - if (ctx->access_token){ - flb_sds_destroy(ctx->access_token); - ctx->access_token = NULL; - } - if (ctx->token_type){ - flb_sds_destroy(ctx->token_type); - ctx->token_type = NULL; - } -} - -/* Append a key/value to the request body */ -int flb_oauth2_payload_append(struct flb_oauth2 *ctx, - const char *key_str, int key_len, - const char *val_str, int val_len) -{ - int size; - flb_sds_t tmp; - - if (key_len == -1) { - key_len = strlen(key_str); - } - if (val_len == -1) { - val_len = strlen(val_str); - } - - /* - * Make sure we have enough space in the sds buffer, otherwise - * add more capacity (so further flb_sds_cat calls do not - * realloc(). - */ - size = key_len + val_len + 2; - if (flb_sds_avail(ctx->payload) < size) { - tmp = flb_sds_increase(ctx->payload, size); - if (!tmp) { - flb_errno(); - return -1; - } - - if (tmp != ctx->payload) { - ctx->payload = tmp; - } - } - - if (flb_sds_len(ctx->payload) > 0) { - flb_sds_cat(ctx->payload, "&", 1); - } - - /* Append key and value */ - flb_sds_cat(ctx->payload, key_str, key_len); - flb_sds_cat(ctx->payload, "=", 1); - flb_sds_cat(ctx->payload, val_str, val_len); - - return 0; -} - -void flb_oauth2_destroy(struct flb_oauth2 *ctx) -{ - flb_sds_destroy(ctx->auth_url); - flb_sds_destroy(ctx->payload); - - flb_sds_destroy(ctx->host); - flb_sds_destroy(ctx->port); - flb_sds_destroy(ctx->uri); - - flb_sds_destroy(ctx->access_token); - flb_sds_destroy(ctx->token_type); - - flb_upstream_destroy(ctx->u); - flb_tls_destroy(ctx->tls); - - flb_free(ctx); -} - -char *flb_oauth2_token_get(struct flb_oauth2 *ctx) -{ - int ret; - size_t b_sent; - time_t now; - struct flb_connection *u_conn; - struct flb_http_client *c; - - now = time(NULL); - if (ctx->access_token) { - /* validate unexpired token */ - if (ctx->expires > now && flb_sds_len(ctx->access_token) > 0) { - return ctx->access_token; - } - } - - /* Get Token and store it in the context */ - u_conn = flb_upstream_conn_get(ctx->u); - if (!u_conn) { - flb_stream_enable_flags(&ctx->u->base, FLB_IO_IPV6); - u_conn = flb_upstream_conn_get(ctx->u); - if (!u_conn) { - flb_error("[oauth2] could not get an upstream connection to %s:%i", - ctx->u->tcp_host, ctx->u->tcp_port); - flb_stream_disable_flags(&ctx->u->base, FLB_IO_IPV6); - return NULL; - } - } - - /* Create HTTP client context */ - c = flb_http_client(u_conn, FLB_HTTP_POST, ctx->uri, - ctx->payload, flb_sds_len(ctx->payload), - ctx->host, atoi(ctx->port), - NULL, 0); - if (!c) { - flb_error("[oauth2] error creating HTTP client context"); - flb_upstream_conn_release(u_conn); - return NULL; - } - - /* Append HTTP Header */ - flb_http_add_header(c, - FLB_HTTP_HEADER_CONTENT_TYPE, - sizeof(FLB_HTTP_HEADER_CONTENT_TYPE) -1, - FLB_OAUTH2_HTTP_ENCODING, - sizeof(FLB_OAUTH2_HTTP_ENCODING) - 1); - - /* Issue request */ - ret = flb_http_do(c, &b_sent); - if (ret != 0) { - flb_warn("[oauth2] cannot issue request, http_do=%i", ret); - } - else { - flb_info("[oauth2] HTTP Status=%i", c->resp.status); - if (c->resp.payload_size > 0) { - if (c->resp.status == 200) { - flb_debug("[oauth2] payload:\n%s", c->resp.payload); - } - else { - flb_info("[oauth2] payload:\n%s", c->resp.payload); - } - } - } - - /* Extract token */ - if (c->resp.payload_size > 0 && c->resp.status == 200) { - ret = flb_oauth2_parse_json_response(c->resp.payload, - c->resp.payload_size, ctx); - if (ret == 0) { - flb_info("[oauth2] access token from '%s:%s' retrieved", - ctx->host, ctx->port); - flb_http_client_destroy(c); - flb_upstream_conn_release(u_conn); - ctx->issued = time(NULL); - ctx->expires = ctx->issued + ctx->expires_in; - return ctx->access_token; - } - } - - flb_http_client_destroy(c); - flb_upstream_conn_release(u_conn); - - return NULL; -} - -int flb_oauth2_token_len(struct flb_oauth2 *ctx) -{ - if (!ctx->access_token) { - return -1; - } - - return flb_sds_len(ctx->access_token); -} - -int flb_oauth2_token_expired(struct flb_oauth2 *ctx) -{ - time_t now; - - if (!ctx->access_token) { - return FLB_TRUE; - } - - now = time(NULL); - if (ctx->expires <= now) { - return FLB_TRUE; - } - - return FLB_FALSE; -} diff --git a/fluent-bit/src/flb_output.c b/fluent-bit/src/flb_output.c deleted file mode 100644 index b1548f60d..000000000 --- a/fluent-bit/src/flb_output.c +++ /dev/null @@ -1,1445 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <stdio.h> -#include <stdlib.h> -#include <string.h> - -#include <fluent-bit/flb_info.h> -#include <fluent-bit/flb_mem.h> -#include <fluent-bit/flb_str.h> -#include <fluent-bit/flb_env.h> -#include <fluent-bit/flb_coro.h> -#include <fluent-bit/flb_output.h> -#include <fluent-bit/flb_kv.h> -#include <fluent-bit/flb_io.h> -#include <fluent-bit/flb_uri.h> -#include <fluent-bit/flb_config.h> -#include <fluent-bit/flb_macros.h> -#include <fluent-bit/flb_utils.h> -#include <fluent-bit/flb_plugin.h> -#include <fluent-bit/flb_plugin_proxy.h> -#include <fluent-bit/flb_http_client_debug.h> -#include <fluent-bit/flb_output_thread.h> -#include <fluent-bit/flb_mp.h> -#include <fluent-bit/flb_pack.h> - -FLB_TLS_DEFINE(struct flb_out_flush_params, out_flush_params); - -void flb_output_prepare() -{ - FLB_TLS_INIT(out_flush_params); -} - -/* Validate the the output address protocol */ -static int check_protocol(const char *prot, const char *output) -{ - int len; - char *p; - - p = strstr(output, "://"); - if (p && p != output) { - len = p - output; - } - else { - len = strlen(output); - } - - if (strlen(prot) != len) { - return 0; - } - - /* Output plugin match */ - if (strncasecmp(prot, output, len) == 0) { - return 1; - } - - return 0; -} - - -/* Invoke pre-run call for the output plugin */ -void flb_output_pre_run(struct flb_config *config) -{ - struct mk_list *head; - struct flb_output_instance *ins; - struct flb_output_plugin *p; - - mk_list_foreach(head, &config->outputs) { - ins = mk_list_entry(head, struct flb_output_instance, _head); - p = ins->p; - if (p->cb_pre_run) { - p->cb_pre_run(ins->context, config); - } - } -} - -static void flb_output_free_properties(struct flb_output_instance *ins) -{ - - flb_kv_release(&ins->properties); - flb_kv_release(&ins->net_properties); - -#ifdef FLB_HAVE_TLS - if (ins->tls_vhost) { - flb_sds_destroy(ins->tls_vhost); - } - if (ins->tls_ca_path) { - flb_sds_destroy(ins->tls_ca_path); - } - if (ins->tls_ca_file) { - flb_sds_destroy(ins->tls_ca_file); - } - if (ins->tls_crt_file) { - flb_sds_destroy(ins->tls_crt_file); - } - if (ins->tls_key_file) { - flb_sds_destroy(ins->tls_key_file); - } - if (ins->tls_key_passwd) { - flb_sds_destroy(ins->tls_key_passwd); - } -#endif -} - -void flb_output_flush_prepare_destroy(struct flb_output_flush *out_flush) -{ - struct flb_output_instance *ins = out_flush->o_ins; - struct flb_out_thread_instance *th_ins; - - /* Move output coroutine context from active list to the destroy one */ - if (flb_output_is_threaded(ins) == FLB_TRUE) { - th_ins = flb_output_thread_instance_get(); - pthread_mutex_lock(&th_ins->flush_mutex); - mk_list_del(&out_flush->_head); - mk_list_add(&out_flush->_head, &th_ins->flush_list_destroy); - pthread_mutex_unlock(&th_ins->flush_mutex); - } - else { - mk_list_del(&out_flush->_head); - mk_list_add(&out_flush->_head, &ins->flush_list_destroy); - } -} - -int flb_output_flush_id_get(struct flb_output_instance *ins) -{ - int id; - int max = (2 << 13) - 1; /* max for 14 bits */ - struct flb_out_thread_instance *th_ins; - - if (flb_output_is_threaded(ins) == FLB_TRUE) { - th_ins = flb_output_thread_instance_get(); - id = th_ins->flush_id; - th_ins->flush_id++; - - /* reset once it reach the maximum allowed */ - if (th_ins->flush_id > max) { - th_ins->flush_id = 0; - } - } - else { - id = ins->flush_id; - ins->flush_id++; - - /* reset once it reach the maximum allowed */ - if (ins->flush_id > max) { - ins->flush_id = 0; - } - } - - return id; -} - -void flb_output_coro_add(struct flb_output_instance *ins, struct flb_coro *coro) -{ - struct flb_output_flush *out_flush; - - out_flush = (struct flb_output_flush *) FLB_CORO_DATA(coro); - mk_list_add(&out_flush->_head, &ins->flush_list); -} - -/* - * Queue a task to be flushed at a later time - * Deletes retry context if enqueue fails - */ -static int flb_output_task_queue_enqueue(struct flb_task_queue *queue, - struct flb_task_retry *retry, - struct flb_task *task, - struct flb_output_instance *out_ins, - struct flb_config *config) -{ - struct flb_task_enqueued *queued_task; - - queued_task = flb_malloc(sizeof(struct flb_task_enqueued)); - if (!queued_task) { - flb_errno(); - if (retry) { - flb_task_retry_destroy(retry); - } - return -1; - } - queued_task->retry = retry; - queued_task->out_instance = out_ins; - queued_task->task = task; - queued_task->config = config; - - mk_list_add(&queued_task->_head, &queue->pending); - return 0; -} - -/* - * Pop task from pending queue and flush it - * Will delete retry context if flush fails - */ -static int flb_output_task_queue_flush_one(struct flb_task_queue *queue) -{ - struct flb_task_enqueued *queued_task; - int ret; - int is_empty; - - is_empty = mk_list_is_empty(&queue->pending) == 0; - if (is_empty) { - flb_error("Attempting to flush task from an empty in_progress queue"); - return -1; - } - - queued_task = mk_list_entry_first(&queue->pending, struct flb_task_enqueued, _head); - mk_list_del(&queued_task->_head); - mk_list_add(&queued_task->_head, &queue->in_progress); - - /* - * Remove temporary user now that task is out of singleplex queue. - * Flush will add back the user representing queued_task->out_instance if it succeeds. - */ - flb_task_users_dec(queued_task->task, FLB_FALSE); - ret = flb_output_task_flush(queued_task->task, - queued_task->out_instance, - queued_task->config); - - /* Destroy retry context if needed */ - if (ret == -1) { - if (queued_task->retry) { - flb_task_retry_destroy(queued_task->retry); - } - /* Flush the next task */ - flb_output_task_singleplex_flush_next(queue); - return -1; - } - - return ret; -} - -/* - * Will either run or queue running a single task - * Deletes retry context if enqueue fails - */ -int flb_output_task_singleplex_enqueue(struct flb_task_queue *queue, - struct flb_task_retry *retry, - struct flb_task *task, - struct flb_output_instance *out_ins, - struct flb_config *config) -{ - int ret; - int is_empty; - - /* - * Add temporary user to preserve task while in singleplex queue. - * Temporary user will be removed when task is removed from queue. - * - * Note: if we fail to increment now, then the task may be prematurely - * deleted if the task's users go to 0 while we are waiting in the - * queue. - */ - flb_task_users_inc(task); - - /* Enqueue task */ - ret = flb_output_task_queue_enqueue(queue, retry, task, out_ins, config); - if (ret == -1) { - return -1; - } - - /* Launch task if nothing is running */ - is_empty = mk_list_is_empty(&out_ins->singleplex_queue->in_progress) == 0; - if (is_empty) { - return flb_output_task_queue_flush_one(out_ins->singleplex_queue); - } - - return 0; -} - -/* - * Clear in progress task and flush a single queued task if exists - * Deletes retry context on next flush if flush fails - */ -int flb_output_task_singleplex_flush_next(struct flb_task_queue *queue) -{ - int is_empty; - struct flb_task_enqueued *ended_task; - - /* Remove in progress task */ - is_empty = mk_list_is_empty(&queue->in_progress) == 0; - if (!is_empty) { - ended_task = mk_list_entry_first(&queue->in_progress, - struct flb_task_enqueued, _head); - mk_list_del(&ended_task->_head); - flb_free(ended_task); - } - - /* Flush if there is a pending task queued */ - is_empty = mk_list_is_empty(&queue->pending) == 0; - if (!is_empty) { - return flb_output_task_queue_flush_one(queue); - } - return 0; -} - -/* - * Flush a task through the output plugin, either using a worker thread + coroutine - * or a simple co-routine in the current thread. - */ -int flb_output_task_flush(struct flb_task *task, - struct flb_output_instance *out_ins, - struct flb_config *config) -{ - int ret; - struct flb_output_flush *out_flush; - - if (flb_output_is_threaded(out_ins) == FLB_TRUE) { - flb_task_users_inc(task); - - /* Dispatch the task to the thread pool */ - ret = flb_output_thread_pool_flush(task, out_ins, config); - if (ret == -1) { - flb_task_users_dec(task, FLB_FALSE); - - /* If we are in synchronous mode, flush one waiting task */ - if (out_ins->flags & FLB_OUTPUT_SYNCHRONOUS) { - flb_output_task_singleplex_flush_next(out_ins->singleplex_queue); - } - } - } - else { - /* Queue co-routine handling */ - out_flush = flb_output_flush_create(task, - task->i_ins, - out_ins, - config); - if (!out_flush) { - return -1; - } - - flb_task_users_inc(task); - ret = flb_pipe_w(config->ch_self_events[1], &out_flush, - sizeof(struct flb_output_flush*)); - if (ret == -1) { - flb_errno(); - flb_output_flush_destroy(out_flush); - flb_task_users_dec(task, FLB_FALSE); - - /* If we are in synchronous mode, flush one waiting task */ - if (out_ins->flags & FLB_OUTPUT_SYNCHRONOUS) { - flb_output_task_singleplex_flush_next(out_ins->singleplex_queue); - } - - return -1; - } - } - - return 0; -} - -int flb_output_instance_destroy(struct flb_output_instance *ins) -{ - if (ins->alias) { - flb_sds_destroy(ins->alias); - } - - /* Remove URI context */ - if (ins->host.uri) { - flb_uri_destroy(ins->host.uri); - } - - flb_sds_destroy(ins->host.name); - flb_sds_destroy(ins->host.address); - flb_sds_destroy(ins->host.listen); - flb_sds_destroy(ins->match); - -#ifdef FLB_HAVE_REGEX - if (ins->match_regex) { - flb_regex_destroy(ins->match_regex); - } -#endif - -#ifdef FLB_HAVE_TLS - if (ins->use_tls == FLB_TRUE) { - if (ins->tls) { - flb_tls_destroy(ins->tls); - } - } - - if (ins->tls_config_map) { - flb_config_map_destroy(ins->tls_config_map); - } -#endif - - /* Remove metrics */ -#ifdef FLB_HAVE_METRICS - if (ins->cmt) { - cmt_destroy(ins->cmt); - } - - if (ins->metrics) { - flb_metrics_destroy(ins->metrics); - } -#endif - - /* destroy callback context */ - if (ins->callback) { - flb_callback_destroy(ins->callback); - } - - /* destroy config map */ - if (ins->config_map) { - flb_config_map_destroy(ins->config_map); - } - - if (ins->net_config_map) { - flb_config_map_destroy(ins->net_config_map); - } - - if (ins->ch_events[0] > 0) { - mk_event_closesocket(ins->ch_events[0]); - } - - if (ins->ch_events[1] > 0) { - mk_event_closesocket(ins->ch_events[1]); - } - - /* release properties */ - flb_output_free_properties(ins); - - /* free singleplex queue */ - if (ins->flags & FLB_OUTPUT_SYNCHRONOUS) { - flb_task_queue_destroy(ins->singleplex_queue); - } - - mk_list_del(&ins->_head); - - /* processor */ - if (ins->processor) { - flb_processor_destroy(ins->processor); - } - - flb_free(ins); - - return 0; -} - -/* Invoke exit call for the output plugin */ -void flb_output_exit(struct flb_config *config) -{ - struct mk_list *tmp; - struct mk_list *head; - struct flb_output_instance *ins; - struct flb_output_plugin *p; - void *params; - - mk_list_foreach_safe(head, tmp, &config->outputs) { - ins = mk_list_entry(head, struct flb_output_instance, _head); - p = ins->p; - - /* Stop any worker thread */ - if (flb_output_is_threaded(ins) == FLB_TRUE) { - flb_output_thread_pool_destroy(ins); - } - - /* Check a exit callback */ - if (p->cb_exit) { - p->cb_exit(ins->context, config); - } - flb_output_instance_destroy(ins); - } - - params = FLB_TLS_GET(out_flush_params); - if (params) { - flb_free(params); - } -} - -static inline int instance_id(struct flb_config *config) -{ - struct flb_output_instance *ins; - - if (mk_list_size(&config->outputs) == 0) { - return 0; - } - - ins = mk_list_entry_last(&config->outputs, struct flb_output_instance, - _head); - return (ins->id + 1); -} - -struct flb_output_instance *flb_output_get_instance(struct flb_config *config, - int out_id) -{ - struct mk_list *head; - struct flb_output_instance *ins; - - mk_list_foreach(head, &config->outputs) { - ins = mk_list_entry(head, struct flb_output_instance, _head); - if (ins->id == out_id) { - break; - } - ins = NULL; - } - - if (!ins) { - return NULL; - } - - return ins; -} - -/* - * Invoked everytime a flush callback has finished (returned). This function - * is called from the event loop. - */ -int flb_output_flush_finished(struct flb_config *config, int out_id) -{ - struct mk_list *tmp; - struct mk_list *head; - struct mk_list *list; - struct flb_output_instance *ins; - struct flb_output_flush *out_flush; - struct flb_out_thread_instance *th_ins; - - ins = flb_output_get_instance(config, out_id); - if (!ins) { - return -1; - } - - if (flb_output_is_threaded(ins) == FLB_TRUE) { - th_ins = flb_output_thread_instance_get(); - list = &th_ins->flush_list_destroy; - } - else { - list = &ins->flush_list_destroy; - } - - /* Look for output coroutines that needs to be destroyed */ - mk_list_foreach_safe(head, tmp, list) { - out_flush = mk_list_entry(head, struct flb_output_flush, _head); - flb_output_flush_destroy(out_flush); - } - - return 0; -} - - -/* - * It validate an output type given the string, it return the - * proper type and if valid, populate the global config. - */ -struct flb_output_instance *flb_output_new(struct flb_config *config, - const char *output, void *data, - int public_only) -{ - int ret = -1; - int flags = 0; - struct mk_list *head; - struct flb_output_plugin *plugin; - struct flb_output_instance *instance = NULL; - - if (!output) { - return NULL; - } - - mk_list_foreach(head, &config->out_plugins) { - plugin = mk_list_entry(head, struct flb_output_plugin, _head); - if (!check_protocol(plugin->name, output)) { - plugin = NULL; - continue; - } - - if (public_only && plugin->flags & FLB_OUTPUT_PRIVATE) { - return NULL; - } - break; - } - - if (!plugin) { - return NULL; - } - - /* Create and load instance */ - instance = flb_calloc(1, sizeof(struct flb_output_instance)); - if (!instance) { - flb_errno(); - return NULL; - } - - /* Initialize event type, if not set, default to FLB_OUTPUT_LOGS */ - if (plugin->event_type == 0) { - instance->event_type = FLB_OUTPUT_LOGS; - } - else { - instance->event_type = plugin->event_type; - } - instance->config = config; - instance->log_level = -1; - instance->log_suppress_interval = -1; - instance->test_mode = FLB_FALSE; - instance->is_threaded = FLB_FALSE; - instance->tp_workers = plugin->workers; - - /* Retrieve an instance id for the output instance */ - instance->id = instance_id(config); - - /* format name (with instance id) */ - snprintf(instance->name, sizeof(instance->name) - 1, - "%s.%i", plugin->name, instance->id); - instance->p = plugin; - instance->callback = flb_callback_create(instance->name); - if (!instance->callback) { - if (instance->flags & FLB_OUTPUT_SYNCHRONOUS) { - flb_task_queue_destroy(instance->singleplex_queue); - } - flb_free(instance); - return NULL; - } - - if (plugin->type == FLB_OUTPUT_PLUGIN_CORE) { - instance->context = NULL; - } - else { - struct flb_plugin_proxy_context *ctx; - - ctx = flb_calloc(1, sizeof(struct flb_plugin_proxy_context)); - if (!ctx) { - flb_errno(); - if (instance->flags & FLB_OUTPUT_SYNCHRONOUS) { - flb_task_queue_destroy(instance->singleplex_queue); - } - flb_free(instance); - return NULL; - } - - ctx->proxy = plugin->proxy; - - instance->context = ctx; - } - - instance->alias = NULL; - instance->flags = instance->p->flags; - instance->data = data; - instance->match = NULL; -#ifdef FLB_HAVE_REGEX - instance->match_regex = NULL; -#endif - instance->retry_limit = 1; - instance->host.name = NULL; - instance->host.address = NULL; - instance->net_config_map = NULL; - - /* Storage */ - instance->total_limit_size = -1; - - /* Parent plugin flags */ - flags = instance->flags; - if (flags & FLB_IO_TCP) { - instance->use_tls = FLB_FALSE; - } - else if (flags & FLB_IO_TLS) { - instance->use_tls = FLB_TRUE; - } - else if (flags & FLB_IO_OPT_TLS) { - /* TLS must be enabled manually in the config */ - instance->use_tls = FLB_FALSE; - instance->flags |= FLB_IO_TLS; - } - -#ifdef FLB_HAVE_TLS - instance->tls = NULL; - instance->tls_debug = -1; - instance->tls_verify = FLB_TRUE; - instance->tls_vhost = NULL; - instance->tls_ca_path = NULL; - instance->tls_ca_file = NULL; - instance->tls_crt_file = NULL; - instance->tls_key_file = NULL; - instance->tls_key_passwd = NULL; -#endif - - if (plugin->flags & FLB_OUTPUT_NET) { - ret = flb_net_host_set(plugin->name, &instance->host, output); - if (ret != 0) { - if (instance->flags & FLB_OUTPUT_SYNCHRONOUS) { - flb_task_queue_destroy(instance->singleplex_queue); - } - flb_free(instance); - return NULL; - } - } - - /* Create singleplex queue if SYNCHRONOUS mode is used */ - instance->singleplex_queue = NULL; - if (instance->flags & FLB_OUTPUT_SYNCHRONOUS) { - instance->singleplex_queue = flb_task_queue_create(); - if (!instance->singleplex_queue) { - flb_free(instance); - flb_errno(); - return NULL; - } - } - - flb_kv_init(&instance->properties); - flb_kv_init(&instance->net_properties); - mk_list_init(&instance->upstreams); - mk_list_init(&instance->flush_list); - mk_list_init(&instance->flush_list_destroy); - - mk_list_add(&instance->_head, &config->outputs); - - /* processor instance */ - instance->processor = flb_processor_create(config, instance->name, instance, FLB_PLUGIN_OUTPUT); - - /* Tests */ - instance->test_formatter.callback = plugin->test_formatter.callback; - - - return instance; -} - -static inline int prop_key_check(const char *key, const char *kv, int k_len) -{ - int len; - - len = strlen(key); - if (strncasecmp(key, kv, k_len) == 0 && len == k_len) { - return 0; - } - - return -1; -} - -/* Override a configuration property for the given input_instance plugin */ -int flb_output_set_property(struct flb_output_instance *ins, - const char *k, const char *v) -{ - int len; - int ret; - ssize_t limit; - flb_sds_t tmp; - struct flb_kv *kv; - struct flb_config *config = ins->config; - - len = strlen(k); - tmp = flb_env_var_translate(config->env, v); - if (tmp) { - if (strlen(tmp) == 0) { - flb_sds_destroy(tmp); - tmp = NULL; - } - } - - /* Check if the key is a known/shared property */ - if (prop_key_check("match", k, len) == 0) { - flb_utils_set_plugin_string_property("match", &ins->match, tmp); - } -#ifdef FLB_HAVE_REGEX - else if (prop_key_check("match_regex", k, len) == 0 && tmp) { - ins->match_regex = flb_regex_create(tmp); - flb_sds_destroy(tmp); - } -#endif - else if (prop_key_check("alias", k, len) == 0 && tmp) { - flb_utils_set_plugin_string_property("alias", &ins->alias, tmp); - } - else if (prop_key_check("log_level", k, len) == 0 && tmp) { - ret = flb_log_get_level_str(tmp); - flb_sds_destroy(tmp); - if (ret == -1) { - return -1; - } - ins->log_level = ret; - } - else if (prop_key_check("log_suppress_interval", k, len) == 0 && tmp) { - ret = flb_utils_time_to_seconds(tmp); - flb_sds_destroy(tmp); - if (ret == -1) { - return -1; - } - ins->log_suppress_interval = ret; - } - else if (prop_key_check("host", k, len) == 0) { - flb_utils_set_plugin_string_property("host", &ins->host.name, tmp); - } - else if (prop_key_check("port", k, len) == 0) { - if (tmp) { - ins->host.port = atoi(tmp); - flb_sds_destroy(tmp); - } - else { - ins->host.port = 0; - } - } - else if (prop_key_check("ipv6", k, len) == 0 && tmp) { - ins->host.ipv6 = flb_utils_bool(tmp); - flb_sds_destroy(tmp); - } - else if (prop_key_check("retry_limit", k, len) == 0) { - if (tmp) { - if (strcasecmp(tmp, "no_limits") == 0 || - strcasecmp(tmp, "false") == 0 || - strcasecmp(tmp, "off") == 0) { - /* No limits for retries */ - ins->retry_limit = FLB_OUT_RETRY_UNLIMITED; - } - else if (strcasecmp(tmp, "no_retries") == 0) { - ins->retry_limit = FLB_OUT_RETRY_NONE; - } - else { - ins->retry_limit = atoi(tmp); - if (ins->retry_limit <= 0) { - flb_warn("[config] invalid retry_limit. set default."); - /* set default when input is invalid number */ - ins->retry_limit = 1; - } - } - flb_sds_destroy(tmp); - } - else { - ins->retry_limit = 1; - } - } - else if (strncasecmp("net.", k, 4) == 0 && tmp) { - kv = flb_kv_item_create(&ins->net_properties, (char *) k, NULL); - if (!kv) { - if (tmp) { - flb_sds_destroy(tmp); - } - return -1; - } - kv->val = tmp; - } -#ifdef FLB_HAVE_HTTP_CLIENT_DEBUG - else if (strncasecmp("_debug.http.", k, 12) == 0 && tmp) { - ret = flb_http_client_debug_property_is_valid((char *) k, tmp); - if (ret == FLB_TRUE) { - kv = flb_kv_item_create(&ins->properties, (char *) k, NULL); - if (!kv) { - if (tmp) { - flb_sds_destroy(tmp); - } - return -1; - } - kv->val = tmp; - } - else { - flb_error("[config] invalid property '%s' on instance '%s'", - k, flb_output_name(ins)); - flb_sds_destroy(tmp); - } - } -#endif -#ifdef FLB_HAVE_TLS - else if (prop_key_check("tls", k, len) == 0 && tmp) { - ins->use_tls = flb_utils_bool(tmp); - if (ins->use_tls == FLB_TRUE && ((ins->flags & FLB_IO_TLS) == 0)) { - flb_error("[config] %s does not support TLS", ins->name); - flb_sds_destroy(tmp); - return -1; - } - flb_sds_destroy(tmp); - } - else if (prop_key_check("tls.verify", k, len) == 0 && tmp) { - ins->tls_verify = flb_utils_bool(tmp); - flb_sds_destroy(tmp); - } - else if (prop_key_check("tls.debug", k, len) == 0 && tmp) { - ins->tls_debug = atoi(tmp); - flb_sds_destroy(tmp); - } - else if (prop_key_check("tls.vhost", k, len) == 0) { - flb_utils_set_plugin_string_property("tls.vhost", &ins->tls_vhost, tmp); - } - else if (prop_key_check("tls.ca_path", k, len) == 0) { - flb_utils_set_plugin_string_property("tls.ca_path", &ins->tls_ca_path, tmp); - } - else if (prop_key_check("tls.ca_file", k, len) == 0) { - flb_utils_set_plugin_string_property("tls.ca_file", &ins->tls_ca_file, tmp); - } - else if (prop_key_check("tls.crt_file", k, len) == 0) { - flb_utils_set_plugin_string_property("tls.crt_file", &ins->tls_crt_file, tmp); - } - else if (prop_key_check("tls.key_file", k, len) == 0) { - flb_utils_set_plugin_string_property("tls.key_file", &ins->tls_key_file, tmp); - } - else if (prop_key_check("tls.key_passwd", k, len) == 0) { - flb_utils_set_plugin_string_property("tls.key_passwd", &ins->tls_key_passwd, tmp); - } -#endif - else if (prop_key_check("storage.total_limit_size", k, len) == 0 && tmp) { - if (strcasecmp(tmp, "off") == 0 || - flb_utils_bool(tmp) == FLB_FALSE) { - /* no limit for filesystem storage */ - limit = -1; - flb_info("[config] unlimited filesystem buffer for %s plugin", - ins->name); - } - else { - limit = flb_utils_size_to_bytes(tmp); - if (limit == -1) { - flb_sds_destroy(tmp); - return -1; - } - - if (limit == 0) { - limit = -1; - } - } - - flb_sds_destroy(tmp); - ins->total_limit_size = (size_t) limit; - } - else if (prop_key_check("workers", k, len) == 0 && tmp) { - /* Set the number of workers */ - ins->tp_workers = atoi(tmp); - flb_sds_destroy(tmp); - } - else { - /* - * Create the property, we don't pass the value since we will - * map it directly to avoid an extra memory allocation. - */ - kv = flb_kv_item_create(&ins->properties, (char *) k, NULL); - if (!kv) { - if (tmp) { - flb_sds_destroy(tmp); - } - return -1; - } - kv->val = tmp; - } - - return 0; -} - -/* Configure a default hostname and TCP port if they are not set */ -void flb_output_net_default(const char *host, const int port, - struct flb_output_instance *ins) -{ - /* Set default network configuration */ - if (!ins->host.name) { - ins->host.name = flb_sds_create(host); - } - if (ins->host.port == 0) { - ins->host.port = port; - } -} - -/* Add thread pool for output plugin if configured with workers */ -int flb_output_enable_multi_threading(struct flb_output_instance *ins, struct flb_config *config) -{ - /* Multi-threading enabled ? (through 'workers' property) */ - if (ins->tp_workers > 0) { - if(flb_output_thread_pool_create(config, ins) != 0) { - flb_output_instance_destroy(ins); - return -1; - } - flb_output_thread_pool_start(ins); - } - - return 0; -} - -/* Return an instance name or alias */ -const char *flb_output_name(struct flb_output_instance *ins) -{ - if (ins->alias) { - return ins->alias; - } - - return ins->name; -} - -const char *flb_output_get_property(const char *key, struct flb_output_instance *ins) -{ - return flb_config_prop_get(key, &ins->properties); -} - -#ifdef FLB_HAVE_METRICS -void *flb_output_get_cmt_instance(struct flb_output_instance *ins) -{ - return (void *)ins->cmt; -} -#endif - -int flb_output_net_property_check(struct flb_output_instance *ins, - struct flb_config *config) -{ - int ret = 0; - - /* Get Upstream net_setup configmap */ - ins->net_config_map = flb_upstream_get_config_map(config); - if (!ins->net_config_map) { - flb_output_instance_destroy(ins); - return -1; - } - - /* - * Validate 'net.*' properties: if the plugin use the Upstream interface, - * it might receive some networking settings. - */ - if (mk_list_size(&ins->net_properties) > 0) { - ret = flb_config_map_properties_check(ins->p->name, - &ins->net_properties, - ins->net_config_map); - if (ret == -1) { - if (config->program_name) { - flb_helper("try the command: %s -o %s -h\n", - config->program_name, ins->p->name); - } - return -1; - } - } - - return 0; -} - -int flb_output_plugin_property_check(struct flb_output_instance *ins, - struct flb_config *config) -{ - int ret = 0; - struct mk_list *config_map; - struct flb_output_plugin *p = ins->p; - - if (p->config_map) { - /* - * Create a dynamic version of the configmap that will be used by the specific - * instance in question. - */ - config_map = flb_config_map_create(config, p->config_map); - if (!config_map) { - flb_error("[output] error loading config map for '%s' plugin", - p->name); - flb_output_instance_destroy(ins); - return -1; - } - ins->config_map = config_map; - - /* Validate incoming properties against config map */ - ret = flb_config_map_properties_check(ins->p->name, - &ins->properties, ins->config_map); - if (ret == -1) { - if (config->program_name) { - flb_helper("try the command: %s -o %s -h\n", - config->program_name, ins->p->name); - } - return -1; - } - } - - return 0; -} - -/* Trigger the output plugins setup callbacks to prepare them. */ -int flb_output_init_all(struct flb_config *config) -{ - int ret; -#ifdef FLB_HAVE_METRICS - char *name; -#endif - struct mk_list *tmp; - struct mk_list *head; - struct flb_output_instance *ins; - struct flb_output_plugin *p; - uint64_t ts; - - /* Retrieve the plugin reference */ - mk_list_foreach_safe(head, tmp, &config->outputs) { - ins = mk_list_entry(head, struct flb_output_instance, _head); - if (ins->log_level == -1) { - ins->log_level = config->log->level; - } - p = ins->p; - - /* Output Events Channel */ - ret = mk_event_channel_create(config->evl, - &ins->ch_events[0], - &ins->ch_events[1], - ins); - if (ret != 0) { - flb_error("could not create events channels for '%s'", - flb_output_name(ins)); - flb_output_instance_destroy(ins); - return -1; - } - flb_debug("[%s:%s] created event channels: read=%i write=%i", - ins->p->name, flb_output_name(ins), - ins->ch_events[0], ins->ch_events[1]); - - /* - * Note: mk_event_channel_create() sets a type = MK_EVENT_NOTIFICATION by - * default, we need to overwrite this value so we can do a clean check - * into the Engine when the event is triggered. - */ - ins->event.type = FLB_ENGINE_EV_OUTPUT; - - /* Metrics */ -#ifdef FLB_HAVE_METRICS - /* Get name or alias for the instance */ - name = (char *) flb_output_name(ins); - - /* get timestamp */ - ts = cfl_time_now(); - - /* CMetrics */ - ins->cmt = cmt_create(); - if (!ins->cmt) { - flb_error("[output] could not create cmetrics context"); - return -1; - } - - /* - * Register generic output plugin metrics - */ - - /* fluentbit_output_proc_records_total */ - ins->cmt_proc_records = cmt_counter_create(ins->cmt, "fluentbit", - "output", "proc_records_total", - "Number of processed output records.", - 1, (char *[]) {"name"}); - cmt_counter_set(ins->cmt_proc_records, ts, 0, 1, (char *[]) {name}); - - - /* fluentbit_output_proc_bytes_total */ - ins->cmt_proc_bytes = cmt_counter_create(ins->cmt, "fluentbit", - "output", "proc_bytes_total", - "Number of processed output bytes.", - 1, (char *[]) {"name"}); - cmt_counter_set(ins->cmt_proc_bytes, ts, 0, 1, (char *[]) {name}); - - - /* fluentbit_output_errors_total */ - ins->cmt_errors = cmt_counter_create(ins->cmt, "fluentbit", - "output", "errors_total", - "Number of output errors.", - 1, (char *[]) {"name"}); - cmt_counter_set(ins->cmt_errors, ts, 0, 1, (char *[]) {name}); - - - /* fluentbit_output_retries_total */ - ins->cmt_retries = cmt_counter_create(ins->cmt, "fluentbit", - "output", "retries_total", - "Number of output retries.", - 1, (char *[]) {"name"}); - cmt_counter_set(ins->cmt_retries, ts, 0, 1, (char *[]) {name}); - - /* fluentbit_output_retries_failed_total */ - ins->cmt_retries_failed = cmt_counter_create(ins->cmt, "fluentbit", - "output", "retries_failed_total", - "Number of abandoned batches because " - "the maximum number of re-tries was " - "reached.", - 1, (char *[]) {"name"}); - cmt_counter_set(ins->cmt_retries_failed, ts, 0, 1, (char *[]) {name}); - - - /* fluentbit_output_dropped_records_total */ - ins->cmt_dropped_records = cmt_counter_create(ins->cmt, "fluentbit", - "output", "dropped_records_total", - "Number of dropped records.", - 1, (char *[]) {"name"}); - cmt_counter_set(ins->cmt_dropped_records, ts, 0, 1, (char *[]) {name}); - - /* fluentbit_output_retried_records_total */ - ins->cmt_retried_records = cmt_counter_create(ins->cmt, "fluentbit", - "output", "retried_records_total", - "Number of retried records.", - 1, (char *[]) {"name"}); - cmt_counter_set(ins->cmt_retried_records, ts, 0, 1, (char *[]) {name}); - - /* output_upstream_total_connections */ - ins->cmt_upstream_total_connections = cmt_gauge_create(ins->cmt, - "fluentbit", - "output", - "upstream_total_connections", - "Total Connection count.", - 1, (char *[]) {"name"}); - cmt_gauge_set(ins->cmt_upstream_total_connections, - ts, - 0, - 1, (char *[]) {name}); - - /* output_upstream_total_connections */ - ins->cmt_upstream_busy_connections = cmt_gauge_create(ins->cmt, - "fluentbit", - "output", - "upstream_busy_connections", - "Busy Connection count.", - 1, (char *[]) {"name"}); - cmt_gauge_set(ins->cmt_upstream_busy_connections, - ts, - 0, - 1, (char *[]) {name}); - - /* old API */ - ins->metrics = flb_metrics_create(name); - if (ins->metrics) { - flb_metrics_add(FLB_METRIC_OUT_OK_RECORDS, - "proc_records", ins->metrics); - flb_metrics_add(FLB_METRIC_OUT_OK_BYTES, - "proc_bytes", ins->metrics); - flb_metrics_add(FLB_METRIC_OUT_ERROR, - "errors", ins->metrics); - flb_metrics_add(FLB_METRIC_OUT_RETRY, - "retries", ins->metrics); - flb_metrics_add(FLB_METRIC_OUT_RETRY_FAILED, - "retries_failed", ins->metrics); - flb_metrics_add(FLB_METRIC_OUT_DROPPED_RECORDS, - "dropped_records", ins->metrics); - flb_metrics_add(FLB_METRIC_OUT_RETRIED_RECORDS, - "retried_records", ins->metrics); - } -#endif - -#ifdef FLB_HAVE_PROXY_GO - /* Proxy plugins have their own initialization */ - if (p->type == FLB_OUTPUT_PLUGIN_PROXY) { - ret = flb_plugin_proxy_output_init(p->proxy, ins, config); - if (ret == -1) { - flb_output_instance_destroy(ins); - return -1; - } - - /* Multi-threading enabled if configured */ - ret = flb_output_enable_multi_threading(ins, config); - if (ret == -1) { - flb_error("[output] could not start thread pool for '%s' plugin", - p->name); - return -1; - } - - continue; - } -#endif - -#ifdef FLB_HAVE_TLS - if (ins->use_tls == FLB_TRUE) { - ins->tls = flb_tls_create(FLB_TLS_CLIENT_MODE, - ins->tls_verify, - ins->tls_debug, - ins->tls_vhost, - ins->tls_ca_path, - ins->tls_ca_file, - ins->tls_crt_file, - ins->tls_key_file, - ins->tls_key_passwd); - if (!ins->tls) { - flb_error("[output %s] error initializing TLS context", - ins->name); - flb_output_instance_destroy(ins); - return -1; - } - } -#endif - /* - * Before to call the initialization callback, make sure that the received - * configuration parameters are valid if the plugin is registering a config map. - */ - if (flb_output_plugin_property_check(ins, config) == -1) { - flb_output_instance_destroy(ins); - return -1; - } - -#ifdef FLB_HAVE_TLS - struct flb_config_map *m; - - /* TLS config map (just for 'help' formatting purposes) */ - ins->tls_config_map = flb_tls_get_config_map(config); - if (!ins->tls_config_map) { - flb_output_instance_destroy(ins); - return -1; - } - - /* Override first configmap value based on it plugin flag */ - m = mk_list_entry_first(ins->tls_config_map, struct flb_config_map, _head); - if (p->flags & FLB_IO_TLS) { - m->value.val.boolean = FLB_TRUE; - } - else { - m->value.val.boolean = FLB_FALSE; - } -#endif - - /* Init network defaults */ - flb_net_setup_init(&ins->net_setup); - - if (flb_output_net_property_check(ins, config) == -1) { - flb_output_instance_destroy(ins); - return -1; - } - - /* Initialize plugin through it 'init callback' */ - ret = p->cb_init(ins, config, ins->data); - if (ret == -1) { - flb_error("[output] failed to initialize '%s' plugin", - p->name); - flb_output_instance_destroy(ins); - return -1; - } - - /* Multi-threading enabled if configured */ - ret = flb_output_enable_multi_threading(ins, config); - if (ret == -1) { - flb_error("[output] could not start thread pool for '%s' plugin", - flb_output_name(ins)); - return -1; - } - - /* initialize processors */ - ret = flb_processor_init(ins->processor); - if (ret == -1) { - return -1; - } - } - - return 0; -} - -/* Assign an Configuration context to an Output */ -void flb_output_set_context(struct flb_output_instance *ins, void *context) -{ - ins->context = context; -} - -/* Check that at least one Output is enabled */ -int flb_output_check(struct flb_config *config) -{ - if (mk_list_is_empty(&config->outputs) == 0) { - return -1; - } - return 0; -} - -/* Check output plugin's log level. - * Not for core plugins but for Golang plugins. - * Golang plugins do not have thread-local flb_worker_ctx information. */ -int flb_output_log_check(struct flb_output_instance *ins, int l) -{ - if (ins->log_level < l) { - return FLB_FALSE; - } - - return FLB_TRUE; -} - -/* - * Output plugins might have enabled certain features that have not been passed - * directly to the upstream context. In order to avoid let plugins validate specific - * variables from the instance context like tls, tls.x, keepalive, etc, we populate - * them directly through this function. - */ -int flb_output_upstream_set(struct flb_upstream *u, struct flb_output_instance *ins) -{ - int flags = 0; - - if (!u) { - return -1; - } - - /* TLS */ -#ifdef FLB_HAVE_TLS - if (ins->use_tls == FLB_TRUE) { - flags |= FLB_IO_TLS; - } - else { - flags |= FLB_IO_TCP; - } -#else - flags |= FLB_IO_TCP; -#endif - - /* IPv6 */ - if (ins->host.ipv6 == FLB_TRUE) { - flags |= FLB_IO_IPV6; - } - /* keepalive */ - if (ins->net_setup.keepalive == FLB_TRUE) { - flags |= FLB_IO_TCP_KA; - } - - /* Set flags */ - flb_stream_enable_flags(&u->base, flags); - - flb_upstream_set_total_connections_label(u, - flb_output_name(ins)); - - flb_upstream_set_total_connections_gauge(u, - ins->cmt_upstream_total_connections); - - flb_upstream_set_busy_connections_label(u, - flb_output_name(ins)); - - flb_upstream_set_busy_connections_gauge(u, - ins->cmt_upstream_busy_connections); - - /* - * If the output plugin flush callbacks will run in multiple threads, enable - * the thread safe mode for the Upstream context. - */ - if (ins->tp_workers > 0) { - flb_stream_enable_thread_safety(&u->base); - - mk_list_add(&u->base._head, &ins->upstreams); - } - - /* Set networking options 'net.*' received through instance properties */ - memcpy(&u->base.net, &ins->net_setup, sizeof(struct flb_net_setup)); - - return 0; -} - -int flb_output_upstream_ha_set(void *ha, struct flb_output_instance *ins) -{ - struct mk_list *head; - struct flb_upstream_node *node; - struct flb_upstream_ha *upstream_ha = ha; - - mk_list_foreach(head, &upstream_ha->nodes) { - node = mk_list_entry(head, struct flb_upstream_node, _head); - flb_output_upstream_set(node->u, ins); - } - - return 0; -} - -/* - * Helper function to set HTTP callbacks using the output instance 'callback' - * context. - */ -int flb_output_set_http_debug_callbacks(struct flb_output_instance *ins) -{ -#ifdef FLB_HAVE_HTTP_CLIENT_DEBUG - return flb_http_client_debug_setup(ins->callback, &ins->properties); -#else - return 0; -#endif -} diff --git a/fluent-bit/src/flb_output_thread.c b/fluent-bit/src/flb_output_thread.c deleted file mode 100644 index 52d1f5795..000000000 --- a/fluent-bit/src/flb_output_thread.c +++ /dev/null @@ -1,568 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <fluent-bit/flb_info.h> -#include <fluent-bit/flb_log.h> -#include <fluent-bit/flb_event_loop.h> -#include <fluent-bit/flb_network.h> -#include <fluent-bit/flb_scheduler.h> -#include <fluent-bit/flb_output_plugin.h> -#include <fluent-bit/flb_output_thread.h> -#include <fluent-bit/flb_thread_pool.h> - -static pthread_once_t local_thread_instance_init = PTHREAD_ONCE_INIT; -FLB_TLS_DEFINE(struct flb_out_thread_instance, local_thread_instance); - -void flb_output_thread_instance_init() -{ - FLB_TLS_INIT(local_thread_instance); -} - -struct flb_out_thread_instance *flb_output_thread_instance_get() -{ - struct flb_out_thread_instance *th_ins; - - th_ins = FLB_TLS_GET(local_thread_instance); - return th_ins; -} - -void flb_output_thread_instance_set(struct flb_out_thread_instance *th_ins) -{ - FLB_TLS_SET(local_thread_instance, th_ins); -} - -/* Cleanup function that runs every 1.5 second */ -static void cb_thread_sched_timer(struct flb_config *ctx, void *data) -{ - (void) ctx; - struct flb_output_instance *ins; - - /* Upstream connections timeouts handling */ - ins = (struct flb_output_instance *) data; - flb_upstream_conn_timeouts(&ins->upstreams); -} - -static inline int handle_output_event(struct flb_config *config, - int ch_parent, flb_pipefd_t fd) -{ - int ret; - int bytes; - int out_id; - uint32_t type; - uint32_t key; - uint64_t val; - - bytes = flb_pipe_r(fd, &val, sizeof(val)); - if (bytes == -1) { - flb_errno(); - return -1; - } - - /* Get type and key */ - type = FLB_BITS_U64_HIGH(val); - key = FLB_BITS_U64_LOW(val); - - if (type != FLB_ENGINE_TASK) { - flb_error("[engine] invalid event type %i for output handler", - type); - return -1; - } - - ret = FLB_TASK_RET(key); - out_id = FLB_TASK_OUT(key); - - /* Destroy the output co-routine context */ - flb_output_flush_finished(config, out_id); - - /* - * Notify the parent event loop the return status, just forward the same - * 64 bits value. - */ - ret = flb_pipe_w(ch_parent, &val, sizeof(val)); - if (ret == -1) { - flb_errno(); - return -1; - } - - return 0; -} - -/* - * For every upstream registered, creates a local mapping for the thread. This is - * done to provide local queues of connections so we can use our event loop and I/O - * totally independently without the need of any syncrhonization across threads - */ -static int upstream_thread_create(struct flb_out_thread_instance *th_ins, - struct flb_output_instance *ins) -{ - struct mk_list *head; - struct flb_upstream *u; - struct flb_upstream *th_u; - - mk_list_foreach(head, &ins->upstreams) { - u = mk_list_entry(head, struct flb_upstream, base._head); - - th_u = flb_calloc(1, sizeof(struct flb_upstream)); - if (!th_u) { - flb_errno(); - return -1; - } - th_u->parent_upstream = u; - flb_upstream_queue_init(&th_u->queue); - mk_list_add(&th_u->base._head, &th_ins->upstreams); - } - - return 0; -} - -int count_upstream_busy_connections(struct flb_out_thread_instance *th_ins) -{ - int c = 0; - struct mk_list *head; - struct flb_upstream *u; - - mk_list_foreach(head, &th_ins->upstreams) { - u = mk_list_entry(head, struct flb_upstream, base._head); - c += mk_list_size(&u->queue.busy_queue); - } - - return c; -} - -static void upstream_thread_destroy(struct flb_out_thread_instance *th_ins) -{ - struct mk_list *tmp; - struct mk_list *head; - struct flb_upstream *th_u; - - mk_list_foreach_safe(head, tmp, &th_ins->upstreams) { - th_u = mk_list_entry(head, struct flb_upstream, base._head); - flb_upstream_destroy(th_u); - } -} - -/* - * This is the worker function that creates an event loop and synchronize - * messages from the engine like 'flush' requests. Note that the running - * plugin flush callback has not notion about it threaded context. - * - * Each worker spawn a co-routine per flush request. - */ -static void output_thread(void *data) -{ - int n; - int ret; - int running = FLB_TRUE; - int stopping = FLB_FALSE; - int thread_id; - char tmp[64]; - struct mk_event event_local; - struct mk_event *event; - struct flb_sched *sched; - struct flb_task *task; - struct flb_connection *u_conn; - struct flb_output_instance *ins; - struct flb_output_flush *out_flush; - struct flb_out_thread_instance *th_ins = data; - struct flb_out_flush_params *params; - struct flb_net_dns dns_ctx; - - /* Register thread instance */ - flb_output_thread_instance_set(th_ins); - - ins = th_ins->ins; - thread_id = th_ins->th->id; - - flb_coro_thread_init(); - - flb_net_ctx_init(&dns_ctx); - flb_net_dns_ctx_set(&dns_ctx); - - /* - * Expose the event loop to the I/O interfaces: since we are in a separate - * thread, the upstream connection interfaces need access to the event - * loop for event notifications. Invoking the flb_engine_evl_set() function - * it sets the event loop reference in a TLS (thread local storage) variable - * of the scope of this thread. - */ - flb_engine_evl_set(th_ins->evl); - - /* Set the upstream queue */ - flb_upstream_list_set(&th_ins->upstreams); - - /* Create a scheduler context */ - sched = flb_sched_create(ins->config, th_ins->evl); - if (!sched) { - flb_plg_error(ins, "could not create thread scheduler"); - return; - } - flb_sched_ctx_set(sched); - - /* - * Sched a permanent callback triggered every 1.5 second to let other - * components of this thread run tasks at that interval. - */ - ret = flb_sched_timer_cb_create(sched, - FLB_SCHED_TIMER_CB_PERM, - 1500, cb_thread_sched_timer, ins, NULL); - if (ret == -1) { - flb_plg_error(ins, "could not schedule permanent callback"); - return; - } - - snprintf(tmp, sizeof(tmp) - 1, "flb-out-%s-w%i", ins->name, thread_id); - mk_utils_worker_rename(tmp); - - memset(&event_local, 0, sizeof(struct mk_event)); - - /* Channel used by flush callbacks to notify it return status */ - ret = mk_event_channel_create(th_ins->evl, - &th_ins->ch_thread_events[0], - &th_ins->ch_thread_events[1], - &event_local); - if (ret == -1) { - flb_plg_error(th_ins->ins, "could not create thread channel"); - flb_engine_evl_set(NULL); - return; - } - event_local.type = FLB_ENGINE_EV_OUTPUT; - - flb_plg_info(th_ins->ins, "worker #%i started", thread_id); - - /* Thread event loop */ - while (running) { - mk_event_wait(th_ins->evl); - flb_event_priority_live_foreach(event, th_ins->evl_bktq, th_ins->evl, - FLB_ENGINE_LOOP_MAX_ITER) { - /* - * FIXME - * ----- - * - handle return status by plugin flush callback. - */ - if (event->type == FLB_ENGINE_EV_CORE) { - - } - else if (event->type & FLB_ENGINE_EV_SCHED) { - /* - * Note that this scheduler event handler has more features - * designed to be used from the parent thread, on this specific - * use case we just care about simple timers created on this - * thread or threaded by some output plugin. - */ - flb_sched_event_handler(sched->config, event); - } - else if (event->type == FLB_ENGINE_EV_THREAD_OUTPUT) { - /* Read the task reference */ - n = flb_pipe_r(event->fd, &task, sizeof(struct flb_task *)); - if (n <= 0) { - flb_errno(); - continue; - } - - /* - * If the address receives 0xdeadbeef, means the thread must - * be terminated. - */ - if (task == (struct flb_task *) 0xdeadbeef) { - stopping = FLB_TRUE; - flb_plg_info(th_ins->ins, "thread worker #%i stopping...", - thread_id); - continue; - } - - /* Start the co-routine with the flush callback */ - out_flush = flb_output_flush_create(task, - task->i_ins, - th_ins->ins, - th_ins->config); - if (!out_flush) { - continue; - } - flb_coro_resume(out_flush->coro); - } - else if (event->type == FLB_ENGINE_EV_CUSTOM) { - event->handler(event); - } - else if (event->type == FLB_ENGINE_EV_THREAD) { - /* - * Check if we have some co-routine associated to this event, - * if so, resume the co-routine - */ - u_conn = (struct flb_connection *) event; - - if (u_conn->coroutine) { - flb_trace("[engine] resuming coroutine=%p", u_conn->coroutine); - flb_coro_resume(u_conn->coroutine); - } - } - else if (event->type == FLB_ENGINE_EV_OUTPUT) { - /* - * The flush callback has finished working and delivered it - * return status. At this intermediary step we cleanup the - * co-routine resources created before and then forward - * the return message to the parent event loop so the Task - * can be updated. - */ - handle_output_event(th_ins->config, ins->ch_events[1], event->fd); - } - else { - flb_plg_warn(ins, "unhandled event type => %i\n", event->type); - } - } - - flb_net_dns_lookup_context_cleanup(&dns_ctx); - - /* Destroy upstream connections from the 'pending destroy list' */ - flb_upstream_conn_pending_destroy_list(&th_ins->upstreams); - flb_sched_timer_cleanup(sched); - - /* Check if we should stop the event loop */ - if (stopping == FLB_TRUE && mk_list_size(&th_ins->flush_list) == 0) { - /* - * If there are no busy network connections (and no coroutines) its - * safe to stop it. - */ - if (count_upstream_busy_connections(th_ins) == 0) { - running = FLB_FALSE; - } - } - } - - /* - * Final cleanup, destroy all resources associated with: - * - * - local upstream maps - * - available connections, likely these are unused keepalive connections - * - any 'new' connection in the pending destroy list - * - event loop context - * - scheduler context - * - parameters helper for coroutines - */ - upstream_thread_destroy(th_ins); - flb_upstream_conn_active_destroy_list(&th_ins->upstreams); - flb_upstream_conn_pending_destroy_list(&th_ins->upstreams); - - flb_sched_destroy(sched); - params = FLB_TLS_GET(out_flush_params); - if (params) { - flb_free(params); - } - mk_event_loop_destroy(th_ins->evl); - flb_bucket_queue_destroy(th_ins->evl_bktq); - - flb_plg_info(ins, "thread worker #%i stopped", thread_id); -} - -int flb_output_thread_pool_flush(struct flb_task *task, - struct flb_output_instance *out_ins, - struct flb_config *config) -{ - int n; - struct flb_tp_thread *th; - struct flb_out_thread_instance *th_ins; - - /* Choose the worker that will handle the Task (round-robin) */ - th = flb_tp_thread_get_rr(out_ins->tp); - if (!th) { - return -1; - } - - th_ins = th->params.data; - - flb_plg_debug(out_ins, "task_id=%i assigned to thread #%i", - task->id, th->id); - - n = flb_pipe_w(th_ins->ch_parent_events[1], &task, sizeof(struct flb_task*)); - - if (n == -1) { - flb_errno(); - return -1; - } - - return 0; -} - -int flb_output_thread_pool_create(struct flb_config *config, - struct flb_output_instance *ins) -{ - int i; - int ret; - struct flb_tp *tp; - struct flb_tp_thread *th; - struct mk_event_loop *evl; - struct flb_bucket_queue *evl_bktq; - struct flb_out_thread_instance *th_ins; - - /* Create the thread pool context */ - tp = flb_tp_create(config); - if (!tp) { - return -1; - } - ins->tp = tp; - ins->is_threaded = FLB_TRUE; - - /* - * Initialize thread-local-storage, every worker thread has it owns - * context with relevant info populated inside the thread. - */ - pthread_once(&local_thread_instance_init, flb_output_thread_instance_init); - - /* Create workers */ - for (i = 0; i < ins->tp_workers; i++) { - th_ins = flb_malloc(sizeof(struct flb_out_thread_instance)); - if (!th_ins) { - flb_errno(); - continue; - } - memset(th_ins, 0, sizeof(struct flb_out_thread_instance)); - - th_ins->config = config; - th_ins->ins = ins; - th_ins->flush_id = 0; - mk_list_init(&th_ins->flush_list); - mk_list_init(&th_ins->flush_list_destroy); - pthread_mutex_init(&th_ins->flush_mutex, NULL); - mk_list_init(&th_ins->upstreams); - - upstream_thread_create(th_ins, ins); - - /* Create the event loop for this thread */ - evl = mk_event_loop_create(64); - if (!evl) { - flb_plg_error(ins, "could not create thread event loop"); - flb_free(th_ins); - continue; - } - evl_bktq = flb_bucket_queue_create(FLB_ENGINE_PRIORITY_COUNT); - if (!evl_bktq) { - flb_plg_error(ins, "could not create thread event loop bucket queue"); - flb_free(evl); - flb_free(th_ins); - continue; - } - th_ins->evl = evl; - th_ins->evl_bktq = evl_bktq; - - /* - * Event loop setup between parent engine and this thread - * - * - FLB engine uses 'ch_parent_events[1]' to dispatch tasks to this thread - * - Thread receive message on ch_parent_events[0] - * - * The mk_event_channel_create() will attach the pipe read end ch_parent_events[0] - * to the local event loop 'evl'. - */ - ret = mk_event_channel_create(th_ins->evl, - &th_ins->ch_parent_events[0], - &th_ins->ch_parent_events[1], - th_ins); - if (ret == -1) { - flb_plg_error(th_ins->ins, "could not create thread channel"); - mk_event_loop_destroy(th_ins->evl); - flb_bucket_queue_destroy(th_ins->evl_bktq); - flb_free(th_ins); - continue; - } - /* Signal type to indicate a "flush" request */ - th_ins->event.type = FLB_ENGINE_EV_THREAD_OUTPUT; - th_ins->event.priority = FLB_ENGINE_PRIORITY_THREAD; - - /* Spawn the thread */ - th = flb_tp_thread_create(tp, output_thread, th_ins, config); - if (!th) { - flb_plg_error(ins, "could not register worker thread #%i", i); - continue; - } - th_ins->th = th; - } - - return 0; -} - -int flb_output_thread_pool_coros_size(struct flb_output_instance *ins) -{ - int n; - int size = 0; - struct mk_list *head; - struct flb_tp *tp = ins->tp; - struct flb_tp_thread *th; - struct flb_out_thread_instance *th_ins; - - /* Signal each worker thread that needs to stop doing work */ - mk_list_foreach(head, &tp->list_threads) { - th = mk_list_entry(head, struct flb_tp_thread, _head); - if (th->status != FLB_THREAD_POOL_RUNNING) { - continue; - } - - th_ins = th->params.data; - - pthread_mutex_lock(&th_ins->flush_mutex); - n = mk_list_size(&th_ins->flush_list); - pthread_mutex_unlock(&th_ins->flush_mutex); - - size += n; - } - - return size; -} - -void flb_output_thread_pool_destroy(struct flb_output_instance *ins) -{ - int n; - struct flb_task *stop = (struct flb_task *) 0xdeadbeef; - struct flb_tp *tp = ins->tp; - struct mk_list *head; - struct flb_out_thread_instance *th_ins; - struct flb_tp_thread *th; - - if (!tp) { - return; - } - - /* Signal each worker thread that needs to stop doing work */ - mk_list_foreach(head, &tp->list_threads) { - th = mk_list_entry(head, struct flb_tp_thread, _head); - if (th->status != FLB_THREAD_POOL_RUNNING) { - continue; - } - - th_ins = th->params.data; - n = flb_pipe_w(th_ins->ch_parent_events[1], &stop, sizeof(stop)); - if (n < 0) { - flb_errno(); - flb_plg_error(th_ins->ins, "could not signal worker thread"); - flb_free(th_ins); - continue; - } - pthread_join(th->tid, NULL); - flb_free(th_ins); - } - - flb_tp_destroy(ins->tp); - ins->tp = NULL; -} - -int flb_output_thread_pool_start(struct flb_output_instance *ins) -{ - struct flb_tp *tp = ins->tp; - - flb_tp_thread_start_all(tp); - return 0; -} diff --git a/fluent-bit/src/flb_pack.c b/fluent-bit/src/flb_pack.c deleted file mode 100644 index adcaa22c9..000000000 --- a/fluent-bit/src/flb_pack.c +++ /dev/null @@ -1,1270 +0,0 @@ -/*-*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <stdlib.h> -#include <string.h> - -#include <fluent-bit/flb_info.h> -#include <fluent-bit/flb_mem.h> -#include <fluent-bit/flb_sds.h> -#include <fluent-bit/flb_error.h> -#include <fluent-bit/flb_utils.h> -#include <fluent-bit/flb_sds.h> -#include <fluent-bit/flb_time.h> -#include <fluent-bit/flb_pack.h> -#include <fluent-bit/flb_unescape.h> - -/* cmetrics */ -#include <cmetrics/cmetrics.h> -#include <cmetrics/cmt_decode_msgpack.h> -#include <cmetrics/cmt_encode_text.h> - -#include <msgpack.h> -#include <math.h> -#include <jsmn/jsmn.h> - -#define try_to_write_str flb_utils_write_str - -static int convert_nan_to_null = FLB_FALSE; - -static int flb_pack_set_null_as_nan(int b) { - if (b == FLB_TRUE || b == FLB_FALSE) { - convert_nan_to_null = b; - } - return convert_nan_to_null; -} - -int flb_json_tokenise(const char *js, size_t len, - struct flb_pack_state *state) -{ - int ret; - int new_tokens = 256; - size_t old_size; - size_t new_size; - void *tmp; - - ret = jsmn_parse(&state->parser, js, len, - state->tokens, state->tokens_size); - while (ret == JSMN_ERROR_NOMEM) { - /* Get current size of the array in bytes */ - old_size = state->tokens_size * sizeof(jsmntok_t); - - /* New size: add capacity for new 256 entries */ - new_size = old_size + (sizeof(jsmntok_t) * new_tokens); - - tmp = flb_realloc(state->tokens, new_size); - if (!tmp) { - flb_errno(); - return -1; - } - state->tokens = tmp; - state->tokens_size += new_tokens; - - ret = jsmn_parse(&state->parser, js, len, - state->tokens, state->tokens_size); - } - - if (ret == JSMN_ERROR_INVAL) { - return FLB_ERR_JSON_INVAL; - } - - if (ret == JSMN_ERROR_PART) { - /* This is a partial JSON message, just stop */ - flb_trace("[json tokenise] incomplete"); - return FLB_ERR_JSON_PART; - } - - state->tokens_count += ret; - return 0; -} - -static inline int is_float(const char *buf, int len) -{ - const char *end = buf + len; - const char *p = buf; - - while (p <= end) { - if (*p == 'e' && p < end && *(p + 1) == '-') { - return 1; - } - else if (*p == '.') { - return 1; - } - p++; - } - - return 0; -} - -/* Sanitize incoming JSON string */ -static inline int pack_string_token(struct flb_pack_state *state, - const char *str, int len, - msgpack_packer *pck) -{ - int s; - int out_len; - char *tmp; - char *out_buf; - - if (state->buf_size < len + 1) { - s = len + 1; - tmp = flb_realloc(state->buf_data, s); - if (!tmp) { - flb_errno(); - return -1; - } - else { - state->buf_data = tmp; - state->buf_size = s; - } - } - out_buf = state->buf_data; - - /* Always decode any UTF-8 or special characters */ - out_len = flb_unescape_string_utf8(str, len, out_buf); - - /* Pack decoded text */ - msgpack_pack_str(pck, out_len); - msgpack_pack_str_body(pck, out_buf, out_len); - - return out_len; -} - -/* Receive a tokenized JSON message and convert it to MsgPack */ -static char *tokens_to_msgpack(struct flb_pack_state *state, - const char *js, - int *out_size, int *last_byte, - int *out_records) -{ - int i; - int flen; - int arr_size; - int records = 0; - const char *p; - char *buf; - const jsmntok_t *t; - msgpack_packer pck; - msgpack_sbuffer sbuf; - jsmntok_t *tokens; - - tokens = state->tokens; - arr_size = state->tokens_count; - - if (arr_size == 0) { - return NULL; - } - - /* initialize buffers */ - msgpack_sbuffer_init(&sbuf); - msgpack_packer_init(&pck, &sbuf, msgpack_sbuffer_write); - - for (i = 0; i < arr_size ; i++) { - t = &tokens[i]; - - if (t->start == -1 || t->end == -1 || (t->start == 0 && t->end == 0)) { - break; - } - - if (t->parent == -1) { - *last_byte = t->end; - records++; - } - - flen = (t->end - t->start); - switch (t->type) { - case JSMN_OBJECT: - msgpack_pack_map(&pck, t->size); - break; - case JSMN_ARRAY: - msgpack_pack_array(&pck, t->size); - break; - case JSMN_STRING: - pack_string_token(state, js + t->start, flen, &pck); - break; - case JSMN_PRIMITIVE: - p = js + t->start; - if (*p == 'f') { - msgpack_pack_false(&pck); - } - else if (*p == 't') { - msgpack_pack_true(&pck); - } - else if (*p == 'n') { - msgpack_pack_nil(&pck); - } - else { - if (is_float(p, flen)) { - msgpack_pack_double(&pck, atof(p)); - } - else { - msgpack_pack_int64(&pck, atoll(p)); - } - } - break; - case JSMN_UNDEFINED: - msgpack_sbuffer_destroy(&sbuf); - return NULL; - } - } - - *out_size = sbuf.size; - *out_records = records; - buf = sbuf.data; - - return buf; -} - -/* - * It parse a JSON string and convert it to MessagePack format, this packer is - * useful when a complete JSON message exists, otherwise it will fail until - * the message is complete. - * - * This routine do not keep a state in the parser, do not use it for big - * JSON messages. - */ -static int pack_json_to_msgpack(const char *js, size_t len, char **buffer, - size_t *size, int *root_type, int *records, - size_t *consumed) -{ - int ret = -1; - int n_records; - int out; - int last; - char *buf = NULL; - struct flb_pack_state state; - - ret = flb_pack_state_init(&state); - if (ret != 0) { - return -1; - } - ret = flb_json_tokenise(js, len, &state); - if (ret != 0) { - ret = -1; - goto flb_pack_json_end; - } - - if (state.tokens_count == 0) { - ret = -1; - goto flb_pack_json_end; - } - - buf = tokens_to_msgpack(&state, js, &out, &last, &n_records); - if (!buf) { - ret = -1; - goto flb_pack_json_end; - } - - *root_type = state.tokens[0].type; - *size = out; - *buffer = buf; - *records = n_records; - - if (consumed != NULL) { - *consumed = last; - } - - ret = 0; - - flb_pack_json_end: - flb_pack_state_reset(&state); - return ret; -} - -/* Pack unlimited serialized JSON messages into msgpack */ -int flb_pack_json(const char *js, size_t len, char **buffer, size_t *size, - int *root_type, size_t *consumed) -{ - int records; - - return pack_json_to_msgpack(js, len, buffer, size, root_type, &records, consumed); -} - -/* - * Pack unlimited serialized JSON messages into msgpack, finally it writes on - * 'out_records' the number of messages. - */ -int flb_pack_json_recs(const char *js, size_t len, char **buffer, size_t *size, - int *root_type, int *out_records, size_t *consumed) -{ - return pack_json_to_msgpack(js, len, buffer, size, root_type, out_records, consumed); -} - -/* Initialize a JSON packer state */ -int flb_pack_state_init(struct flb_pack_state *s) -{ - int tokens = 256; - size_t size = 256; - - jsmn_init(&s->parser); - - size = sizeof(jsmntok_t) * tokens; - s->tokens = flb_malloc(size); - if (!s->tokens) { - flb_errno(); - return -1; - } - s->tokens_size = tokens; - s->tokens_count = 0; - s->last_byte = 0; - s->multiple = FLB_FALSE; - - s->buf_data = flb_malloc(size); - if (!s->buf_data) { - flb_errno(); - flb_free(s->tokens); - s->tokens = NULL; - return -1; - } - s->buf_size = size; - s->buf_len = 0; - - return 0; -} - -void flb_pack_state_reset(struct flb_pack_state *s) -{ - flb_free(s->tokens); - s->tokens = NULL; - s->tokens_size = 0; - s->tokens_count = 0; - s->last_byte = 0; - s->buf_size = 0; - flb_free(s->buf_data); - s->buf_data = NULL; -} - - -/* - * It parse a JSON string and convert it to MessagePack format. The main - * difference of this function and the previous flb_pack_json() is that it - * keeps a parser and tokens state, allowing to process big messages and - * resume the parsing process instead of start from zero. - */ -int flb_pack_json_state(const char *js, size_t len, - char **buffer, int *size, - struct flb_pack_state *state) -{ - int ret; - int out; - int delim = 0; - int last = 0; - int records; - char *buf; - jsmntok_t *t; - - ret = flb_json_tokenise(js, len, state); - state->multiple = FLB_TRUE; - if (ret == FLB_ERR_JSON_PART && state->multiple == FLB_TRUE) { - /* - * If the caller enabled 'multiple' flag, it means that the incoming - * JSON message may have multiple messages concatenated and likely - * the last one is only incomplete. - * - * The following routine aims to determinate how many JSON messages - * are OK in the array of tokens, if any, process them and adjust - * the JSMN context/buffers. - */ - - /* - * jsmn_parse updates jsmn_parser members. (state->parser) - * A member 'toknext' points next incomplete object token. - * We use toknext - 1 as an index of last member of complete JSON. - */ - int i; - int found = 0; - - if (state->parser.toknext == 0) { - return ret; - } - - for (i = (int)state->parser.toknext - 1; i >= 1; i--) { - t = &state->tokens[i]; - - if (t->parent == -1 && (t->end != 0)) { - found++; - delim = i; - break; - } - } - - if (found == 0) { - return ret; /* FLB_ERR_JSON_PART */ - } - state->tokens_count += delim; - } - else if (ret != 0) { - return ret; - } - - if (state->tokens_count == 0 || state->tokens == NULL) { - state->last_byte = last; - return FLB_ERR_JSON_INVAL; - } - - buf = tokens_to_msgpack(state, js, &out, &last, &records); - if (!buf) { - return -1; - } - - *size = out; - *buffer = buf; - state->last_byte = last; - - return 0; -} - -int flb_metadata_pop_from_msgpack(msgpack_object **metadata, msgpack_unpacked *upk, - msgpack_object **map) -{ - if (metadata == NULL || upk == NULL) { - return -1; - } - - if (upk->data.type != MSGPACK_OBJECT_ARRAY) { - return -1; - } - - *metadata = &upk->data.via.array.ptr[0].via.array.ptr[1]; - *map = &upk->data.via.array.ptr[1]; - - return 0; -} - -static int pack_print_fluent_record(size_t cnt, msgpack_unpacked result) -{ - msgpack_object *metadata; - msgpack_object root; - msgpack_object *obj; - struct flb_time tms; - msgpack_object o; - - root = result.data; - if (root.type != MSGPACK_OBJECT_ARRAY) { - return -1; - } - - o = root.via.array.ptr[0]; - if (o.type != MSGPACK_OBJECT_ARRAY) { - return -1; - } - - /* decode expected timestamp only (integer, float or ext) */ - o = o.via.array.ptr[0]; - if (o.type != MSGPACK_OBJECT_POSITIVE_INTEGER && - o.type != MSGPACK_OBJECT_FLOAT && - o.type != MSGPACK_OBJECT_EXT) { - return -1; - } - - /* This is a Fluent Bit record, just do the proper unpacking/printing */ - flb_time_pop_from_msgpack(&tms, &result, &obj); - flb_metadata_pop_from_msgpack(&metadata, &result, &obj); - - fprintf(stdout, "[%zd] [%"PRIu32".%09lu, ", cnt, - (uint32_t) tms.tm.tv_sec, tms.tm.tv_nsec); - - msgpack_object_print(stdout, *metadata); - - fprintf(stdout, ", "); - - msgpack_object_print(stdout, *obj); - - fprintf(stdout, "]\n"); - - return 0; -} - -void flb_pack_print(const char *data, size_t bytes) -{ - int ret; - msgpack_unpacked result; - size_t off = 0, cnt = 0; - - msgpack_unpacked_init(&result); - while (msgpack_unpack_next(&result, data, bytes, &off) == MSGPACK_UNPACK_SUCCESS) { - /* Check if we are processing an internal Fluent Bit record */ - ret = pack_print_fluent_record(cnt, result); - if (ret == 0) { - continue; - } - - printf("[%zd] ", cnt++); - msgpack_object_print(stdout, result.data); - printf("\n"); - } - msgpack_unpacked_destroy(&result); -} - -void flb_pack_print_metrics(const char *data, size_t bytes) -{ - int ret; - size_t off = 0; - cfl_sds_t text; - struct cmt *cmt = NULL; - - /* get cmetrics context */ - ret = cmt_decode_msgpack_create(&cmt, (char *) data, bytes, &off); - if (ret != 0) { - flb_error("could not process metrics payload"); - return; - } - - /* convert to text representation */ - text = cmt_encode_text_create(cmt); - - /* destroy cmt context */ - cmt_destroy(cmt); - - printf("%s", text); - fflush(stdout); - - cmt_encode_text_destroy(text); -} - -static inline int try_to_write(char *buf, int *off, size_t left, - const char *str, size_t str_len) -{ - if (str_len <= 0){ - str_len = strlen(str); - } - if (left <= *off+str_len) { - return FLB_FALSE; - } - memcpy(buf+*off, str, str_len); - *off += str_len; - return FLB_TRUE; -} - - -/* - * Check if a key exists in the map using the 'offset' as an index to define - * which element needs to start looking from - */ -static inline int key_exists_in_map(msgpack_object key, msgpack_object map, int offset) -{ - int i; - msgpack_object p; - - if (key.type != MSGPACK_OBJECT_STR) { - return FLB_FALSE; - } - - for (i = offset; i < map.via.map.size; i++) { - p = map.via.map.ptr[i].key; - if (p.type != MSGPACK_OBJECT_STR) { - continue; - } - - if (key.via.str.size != p.via.str.size) { - continue; - } - - if (memcmp(key.via.str.ptr, p.via.str.ptr, p.via.str.size) == 0) { - return FLB_TRUE; - } - } - - return FLB_FALSE; -} - -static int msgpack2json(char *buf, int *off, size_t left, - const msgpack_object *o) -{ - int i; - int dup; - int ret = FLB_FALSE; - int loop; - int packed; - - switch(o->type) { - case MSGPACK_OBJECT_NIL: - ret = try_to_write(buf, off, left, "null", 4); - break; - - case MSGPACK_OBJECT_BOOLEAN: - ret = try_to_write(buf, off, left, - (o->via.boolean ? "true":"false"),0); - - break; - - case MSGPACK_OBJECT_POSITIVE_INTEGER: - { - char temp[32] = {0}; - i = snprintf(temp, sizeof(temp)-1, "%"PRIu64, o->via.u64); - ret = try_to_write(buf, off, left, temp, i); - } - break; - - case MSGPACK_OBJECT_NEGATIVE_INTEGER: - { - char temp[32] = {0}; - i = snprintf(temp, sizeof(temp)-1, "%"PRId64, o->via.i64); - ret = try_to_write(buf, off, left, temp, i); - } - break; - case MSGPACK_OBJECT_FLOAT32: - case MSGPACK_OBJECT_FLOAT64: - { - char temp[512] = {0}; - if (o->via.f64 == (double)(long long int)o->via.f64) { - i = snprintf(temp, sizeof(temp)-1, "%.1f", o->via.f64); - } - else if (convert_nan_to_null && isnan(o->via.f64) ) { - i = snprintf(temp, sizeof(temp)-1, "null"); - } - else { - i = snprintf(temp, sizeof(temp)-1, "%.16g", o->via.f64); - } - ret = try_to_write(buf, off, left, temp, i); - } - break; - - case MSGPACK_OBJECT_STR: - if (try_to_write(buf, off, left, "\"", 1) && - (o->via.str.size > 0 ? - try_to_write_str(buf, off, left, o->via.str.ptr, o->via.str.size) - : 1/* nothing to do */) && - try_to_write(buf, off, left, "\"", 1)) { - ret = FLB_TRUE; - } - break; - - case MSGPACK_OBJECT_BIN: - if (try_to_write(buf, off, left, "\"", 1) && - (o->via.bin.size > 0 ? - try_to_write_str(buf, off, left, o->via.bin.ptr, o->via.bin.size) - : 1 /* nothing to do */) && - try_to_write(buf, off, left, "\"", 1)) { - ret = FLB_TRUE; - } - break; - - case MSGPACK_OBJECT_EXT: - if (!try_to_write(buf, off, left, "\"", 1)) { - goto msg2json_end; - } - /* ext body. fortmat is similar to printf(1) */ - { - char temp[32] = {0}; - int len; - loop = o->via.ext.size; - for(i=0; i<loop; i++) { - len = snprintf(temp, sizeof(temp)-1, "\\x%02x", (char)o->via.ext.ptr[i]); - if (!try_to_write(buf, off, left, temp, len)) { - goto msg2json_end; - } - } - } - if (!try_to_write(buf, off, left, "\"", 1)) { - goto msg2json_end; - } - ret = FLB_TRUE; - break; - - case MSGPACK_OBJECT_ARRAY: - loop = o->via.array.size; - - if (!try_to_write(buf, off, left, "[", 1)) { - goto msg2json_end; - } - if (loop != 0) { - msgpack_object* p = o->via.array.ptr; - if (!msgpack2json(buf, off, left, p)) { - goto msg2json_end; - } - for (i=1; i<loop; i++) { - if (!try_to_write(buf, off, left, ",", 1) || - !msgpack2json(buf, off, left, p+i)) { - goto msg2json_end; - } - } - } - - ret = try_to_write(buf, off, left, "]", 1); - break; - - case MSGPACK_OBJECT_MAP: - loop = o->via.map.size; - if (!try_to_write(buf, off, left, "{", 1)) { - goto msg2json_end; - } - if (loop != 0) { - msgpack_object k; - msgpack_object_kv *p = o->via.map.ptr; - - packed = 0; - dup = FLB_FALSE; - - k = o->via.map.ptr[0].key; - for (i = 0; i < loop; i++) { - k = o->via.map.ptr[i].key; - dup = key_exists_in_map(k, *o, i + 1); - if (dup == FLB_TRUE) { - continue; - } - - if (packed > 0) { - if (!try_to_write(buf, off, left, ",", 1)) { - goto msg2json_end; - } - } - - if ( - !msgpack2json(buf, off, left, &(p+i)->key) || - !try_to_write(buf, off, left, ":", 1) || - !msgpack2json(buf, off, left, &(p+i)->val) ) { - goto msg2json_end; - } - packed++; - } - } - - ret = try_to_write(buf, off, left, "}", 1); - break; - - default: - flb_warn("[%s] unknown msgpack type %i", __FUNCTION__, o->type); - } - - msg2json_end: - return ret; -} - -/** - * convert msgpack to JSON string. - * This API is similar to snprintf. - * - * @param json_str The buffer to fill JSON string. - * @param json_size The size of json_str. - * @param data The msgpack_unpacked data. - * @return success ? a number characters filled : negative value - */ -int flb_msgpack_to_json(char *json_str, size_t json_size, - const msgpack_object *obj) -{ - int ret = -1; - int off = 0; - - if (json_str == NULL || obj == NULL) { - return -1; - } - - ret = msgpack2json(json_str, &off, json_size - 1, obj); - json_str[off] = '\0'; - return ret ? off: ret; -} - -flb_sds_t flb_msgpack_raw_to_json_sds(const void *in_buf, size_t in_size) -{ - int ret; - size_t off = 0; - size_t out_size; - size_t realloc_size; - - msgpack_unpacked result; - msgpack_object *root; - flb_sds_t out_buf; - flb_sds_t tmp_buf; - - /* buffer size strategy */ - out_size = in_size * FLB_MSGPACK_TO_JSON_INIT_BUFFER_SIZE; - realloc_size = in_size * FLB_MSGPACK_TO_JSON_REALLOC_BUFFER_SIZE; - if (realloc_size < 256) { - realloc_size = 256; - } - - out_buf = flb_sds_create_size(out_size); - if (!out_buf) { - flb_errno(); - return NULL; - } - - msgpack_unpacked_init(&result); - ret = msgpack_unpack_next(&result, in_buf, in_size, &off); - if (ret != MSGPACK_UNPACK_SUCCESS) { - flb_sds_destroy(out_buf); - msgpack_unpacked_destroy(&result); - return NULL; - } - - root = &result.data; - while (1) { - ret = flb_msgpack_to_json(out_buf, out_size, root); - if (ret <= 0) { - tmp_buf = flb_sds_increase(out_buf, realloc_size); - if (tmp_buf) { - out_buf = tmp_buf; - out_size += realloc_size; - } - else { - flb_errno(); - flb_sds_destroy(out_buf); - msgpack_unpacked_destroy(&result); - return NULL; - } - } - else { - break; - } - } - - msgpack_unpacked_destroy(&result); - flb_sds_len_set(out_buf, ret); - - return out_buf; -} - -/* - * Given a 'format' string type, return it integer representation. This - * is used by output plugins that uses pack functions to convert - * msgpack records to JSON. - */ -int flb_pack_to_json_format_type(const char *str) -{ - if (strcasecmp(str, "msgpack") == 0) { - return FLB_PACK_JSON_FORMAT_NONE; - } - else if (strcasecmp(str, "json") == 0) { - return FLB_PACK_JSON_FORMAT_JSON; - } - else if (strcasecmp(str, "json_stream") == 0) { - return FLB_PACK_JSON_FORMAT_STREAM; - } - else if (strcasecmp(str, "json_lines") == 0) { - return FLB_PACK_JSON_FORMAT_LINES; - } - - return -1; -} - -/* Given a 'date string type', return it integer representation */ -int flb_pack_to_json_date_type(const char *str) -{ - if (strcasecmp(str, "double") == 0) { - return FLB_PACK_JSON_DATE_DOUBLE; - } - else if (strcasecmp(str, "java_sql_timestamp") == 0) { - return FLB_PACK_JSON_DATE_JAVA_SQL_TIMESTAMP; - } - else if (strcasecmp(str, "iso8601") == 0) { - return FLB_PACK_JSON_DATE_ISO8601; - } - else if (strcasecmp(str, "epoch") == 0) { - return FLB_PACK_JSON_DATE_EPOCH; - } - else if (strcasecmp(str, "epoch_ms") == 0 || - strcasecmp(str, "epoch_millis") == 0 || - strcasecmp(str, "epoch_milliseconds") == 0) { - return FLB_PACK_JSON_DATE_EPOCH_MS; - } - - return -1; -} - - -static int msgpack_pack_formatted_datetime(flb_sds_t out_buf, char time_formatted[], int max_len, - msgpack_packer* tmp_pck, struct flb_time* tms, - const char *date_format, - const char *time_format) -{ - int len; - size_t s; - struct tm tm; - - gmtime_r(&tms->tm.tv_sec, &tm); - - s = strftime(time_formatted, max_len, - date_format, &tm); - if (!s) { - flb_debug("strftime failed in flb_pack_msgpack_to_json_format"); - return 1; - } - - /* Format the time, use microsecond precision not nanoseconds */ - max_len -= s; - len = snprintf(&time_formatted[s], - max_len, - time_format, - (uint64_t) tms->tm.tv_nsec / 1000); - if (len >= max_len) { - flb_debug("snprintf: %d >= %d in flb_pack_msgpack_to_json_format", len, max_len); - return 2; - } - s += len; - msgpack_pack_str(tmp_pck, s); - msgpack_pack_str_body(tmp_pck, time_formatted, s); - return 0; -} - -flb_sds_t flb_pack_msgpack_to_json_format(const char *data, uint64_t bytes, - int json_format, int date_format, - flb_sds_t date_key) -{ - int i; - int ok = MSGPACK_UNPACK_SUCCESS; - int records = 0; - int map_size; - size_t off = 0; - char time_formatted[38]; - flb_sds_t out_tmp; - flb_sds_t out_js; - flb_sds_t out_buf = NULL; - msgpack_unpacked result; - msgpack_object root; - msgpack_object map; - msgpack_sbuffer tmp_sbuf; - msgpack_packer tmp_pck; - msgpack_object *obj; - msgpack_object *k; - msgpack_object *v; - struct flb_time tms; - - /* For json lines and streams mode we need a pre-allocated buffer */ - if (json_format == FLB_PACK_JSON_FORMAT_LINES || - json_format == FLB_PACK_JSON_FORMAT_STREAM) { - out_buf = flb_sds_create_size(bytes + bytes / 4); - if (!out_buf) { - flb_errno(); - return NULL; - } - } - - /* Create temporary msgpack buffer */ - msgpack_sbuffer_init(&tmp_sbuf); - msgpack_packer_init(&tmp_pck, &tmp_sbuf, msgpack_sbuffer_write); - - /* - * If the format is the original msgpack style of one big array, - * registrate the array, otherwise is not necessary. FYI, original format: - * - * [ - * [timestamp, map], - * [timestamp, map], - * [T, M]... - * ] - */ - if (json_format == FLB_PACK_JSON_FORMAT_JSON) { - records = flb_mp_count(data, bytes); - if (records <= 0) { - msgpack_sbuffer_destroy(&tmp_sbuf); - return NULL; - } - msgpack_pack_array(&tmp_pck, records); - } - - msgpack_unpacked_init(&result); - while (msgpack_unpack_next(&result, data, bytes, &off) == ok) { - /* Each array must have two entries: time and record */ - root = result.data; - if (root.type != MSGPACK_OBJECT_ARRAY) { - continue; - } - if (root.via.array.size != 2) { - continue; - } - - /* Unpack time */ - flb_time_pop_from_msgpack(&tms, &result, &obj); - - /* Get the record/map */ - map = root.via.array.ptr[1]; - if (map.type != MSGPACK_OBJECT_MAP) { - continue; - } - map_size = map.via.map.size; - - if (date_key != NULL) { - msgpack_pack_map(&tmp_pck, map_size + 1); - } - else { - msgpack_pack_map(&tmp_pck, map_size); - } - - if (date_key != NULL) { - /* Append date key */ - msgpack_pack_str(&tmp_pck, flb_sds_len(date_key)); - msgpack_pack_str_body(&tmp_pck, date_key, flb_sds_len(date_key)); - - /* Append date value */ - switch (date_format) { - case FLB_PACK_JSON_DATE_DOUBLE: - msgpack_pack_double(&tmp_pck, flb_time_to_double(&tms)); - break; - case FLB_PACK_JSON_DATE_JAVA_SQL_TIMESTAMP: - if (msgpack_pack_formatted_datetime(out_buf, time_formatted, sizeof(time_formatted), &tmp_pck, &tms, - FLB_PACK_JSON_DATE_JAVA_SQL_TIMESTAMP_FMT, ".%06" PRIu64)) { - flb_sds_destroy(out_buf); - msgpack_sbuffer_destroy(&tmp_sbuf); - msgpack_unpacked_destroy(&result); - return NULL; - } - break; - case FLB_PACK_JSON_DATE_ISO8601: - if (msgpack_pack_formatted_datetime(out_buf, time_formatted, sizeof(time_formatted), &tmp_pck, &tms, - FLB_PACK_JSON_DATE_ISO8601_FMT, ".%06" PRIu64 "Z")) { - flb_sds_destroy(out_buf); - msgpack_sbuffer_destroy(&tmp_sbuf); - msgpack_unpacked_destroy(&result); - return NULL; - } - break; - case FLB_PACK_JSON_DATE_EPOCH: - msgpack_pack_uint64(&tmp_pck, (long long unsigned)(tms.tm.tv_sec)); - break; - case FLB_PACK_JSON_DATE_EPOCH_MS: - msgpack_pack_uint64(&tmp_pck, flb_time_to_millisec(&tms)); - break; - } - } - - /* Append remaining keys/values */ - for (i = 0; i < map_size; i++) { - k = &map.via.map.ptr[i].key; - v = &map.via.map.ptr[i].val; - msgpack_pack_object(&tmp_pck, *k); - msgpack_pack_object(&tmp_pck, *v); - } - - /* - * If the format is the original msgpack style, just continue since - * we don't care about separator or JSON convertion at this point. - */ - if (json_format == FLB_PACK_JSON_FORMAT_JSON) { - continue; - } - - /* - * Here we handle two types of records concatenation: - * - * FLB_PACK_JSON_FORMAT_LINES: add breakline (\n) after each record - * - * - * {'ts':abc,'k1':1} - * {'ts':abc,'k1':2} - * {N} - * - * FLB_PACK_JSON_FORMAT_STREAM: no separators, e.g: - * - * {'ts':abc,'k1':1}{'ts':abc,'k1':2}{N} - */ - if (json_format == FLB_PACK_JSON_FORMAT_LINES || - json_format == FLB_PACK_JSON_FORMAT_STREAM) { - - /* Encode current record into JSON in a temporary variable */ - out_js = flb_msgpack_raw_to_json_sds(tmp_sbuf.data, tmp_sbuf.size); - if (!out_js) { - flb_sds_destroy(out_buf); - msgpack_sbuffer_destroy(&tmp_sbuf); - msgpack_unpacked_destroy(&result); - return NULL; - } - - /* - * One map record has been converted, now append it to the - * outgoing out_buf sds variable. - */ - out_tmp = flb_sds_cat(out_buf, out_js, flb_sds_len(out_js)); - if (!out_tmp) { - flb_sds_destroy(out_js); - flb_sds_destroy(out_buf); - msgpack_sbuffer_destroy(&tmp_sbuf); - msgpack_unpacked_destroy(&result); - return NULL; - } - - /* Release temporary json sds buffer */ - flb_sds_destroy(out_js); - - /* If a realloc happened, check the returned address */ - if (out_tmp != out_buf) { - out_buf = out_tmp; - } - - /* Append the breakline only for json lines mode */ - if (json_format == FLB_PACK_JSON_FORMAT_LINES) { - out_tmp = flb_sds_cat(out_buf, "\n", 1); - if (!out_tmp) { - flb_sds_destroy(out_buf); - msgpack_sbuffer_destroy(&tmp_sbuf); - msgpack_unpacked_destroy(&result); - return NULL; - } - if (out_tmp != out_buf) { - out_buf = out_tmp; - } - } - msgpack_sbuffer_clear(&tmp_sbuf); - } - } - - /* Release the unpacker */ - msgpack_unpacked_destroy(&result); - - /* Format to JSON */ - if (json_format == FLB_PACK_JSON_FORMAT_JSON) { - out_buf = flb_msgpack_raw_to_json_sds(tmp_sbuf.data, tmp_sbuf.size); - msgpack_sbuffer_destroy(&tmp_sbuf); - - if (!out_buf) { - return NULL; - } - } - else { - msgpack_sbuffer_destroy(&tmp_sbuf); - } - - if (out_buf && flb_sds_len(out_buf) == 0) { - flb_sds_destroy(out_buf); - return NULL; - } - - return out_buf; -} - -/** - * convert msgpack to JSON string. - * This API is similar to snprintf. - * @param size Estimated length of json str. - * @param data The msgpack_unpacked data. - * @return success ? allocated json str ptr : NULL - */ -char *flb_msgpack_to_json_str(size_t size, const msgpack_object *obj) -{ - int ret; - char *buf = NULL; - char *tmp; - - if (obj == NULL) { - return NULL; - } - - if (size <= 0) { - size = 128; - } - - buf = flb_malloc(size); - if (!buf) { - flb_errno(); - return NULL; - } - - while (1) { - ret = flb_msgpack_to_json(buf, size, obj); - if (ret <= 0) { - /* buffer is small. retry.*/ - size += 128; - tmp = flb_realloc(buf, size); - if (tmp) { - buf = tmp; - } - else { - flb_free(buf); - flb_errno(); - return NULL; - } - } - else { - break; - } - } - - return buf; -} - -int flb_pack_time_now(msgpack_packer *pck) -{ - int ret; - struct flb_time t; - - flb_time_get(&t); - ret = flb_time_append_to_msgpack(&t, pck, 0); - - return ret; -} - -int flb_msgpack_expand_map(char *map_data, size_t map_size, - msgpack_object_kv **kv_arr, int kv_arr_len, - char** out_buf, int* out_size) -{ - msgpack_sbuffer sbuf; - msgpack_packer pck; - msgpack_unpacked result; - size_t off = 0; - char *ret_buf; - int map_num; - int i; - int len; - - if (map_data == NULL){ - return -1; - } - - msgpack_unpacked_init(&result); - if ((i=msgpack_unpack_next(&result, map_data, map_size, &off)) != - MSGPACK_UNPACK_SUCCESS ) { - msgpack_unpacked_destroy(&result); - return -1; - } - if (result.data.type != MSGPACK_OBJECT_MAP) { - msgpack_unpacked_destroy(&result); - return -1; - } - - len = result.data.via.map.size; - map_num = kv_arr_len + len; - - msgpack_sbuffer_init(&sbuf); - msgpack_packer_init(&pck, &sbuf, msgpack_sbuffer_write); - msgpack_pack_map(&pck, map_num); - - for (i=0; i<len; i++) { - msgpack_pack_object(&pck, result.data.via.map.ptr[i].key); - msgpack_pack_object(&pck, result.data.via.map.ptr[i].val); - } - for (i=0; i<kv_arr_len; i++){ - msgpack_pack_object(&pck, kv_arr[i]->key); - msgpack_pack_object(&pck, kv_arr[i]->val); - } - msgpack_unpacked_destroy(&result); - - *out_size = sbuf.size; - ret_buf = flb_malloc(sbuf.size); - *out_buf = ret_buf; - if (*out_buf == NULL) { - flb_errno(); - msgpack_sbuffer_destroy(&sbuf); - return -1; - } - memcpy(*out_buf, sbuf.data, sbuf.size); - msgpack_sbuffer_destroy(&sbuf); - - return 0; -} - -int flb_pack_init(struct flb_config *config) -{ - int ret; - - if (config == NULL) { - return -1; - } - ret = flb_pack_set_null_as_nan(config->convert_nan_to_null); - - return ret; -} diff --git a/fluent-bit/src/flb_pack_gelf.c b/fluent-bit/src/flb_pack_gelf.c deleted file mode 100644 index 0aac9ee81..000000000 --- a/fluent-bit/src/flb_pack_gelf.c +++ /dev/null @@ -1,826 +0,0 @@ -/*-*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <fluent-bit/flb_info.h> -#include <fluent-bit/flb_pack.h> - -static flb_sds_t flb_msgpack_gelf_key(flb_sds_t *s, int in_array, - const char *prefix_key, int prefix_key_len, - int concat, - const char *key, int key_len) -{ - int i; - flb_sds_t tmp; - static char valid_char[256] = { - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, - 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }; - int start_len, end_len; - - if (in_array == FLB_FALSE) { - tmp = flb_sds_cat(*s, ", \"", 3); - if (tmp == NULL) { - return NULL; - } - *s = tmp; - } - - if (prefix_key_len > 0) { - start_len = flb_sds_len(*s); - - tmp = flb_sds_cat(*s, prefix_key, prefix_key_len); - if (tmp == NULL) { - return NULL; - } - *s = tmp; - - end_len = flb_sds_len(*s); - for(i=start_len; i < end_len; i++) { - if (!valid_char[(unsigned char)(*s)[i]]) { - (*s)[i] = '_'; - } - } - } - - if (concat == FLB_TRUE) { - tmp = flb_sds_cat(*s, "_", 1); - if (tmp == NULL) { - return NULL; - } - *s = tmp; - } - - if (key_len > 0) { - start_len = flb_sds_len(*s); - - tmp = flb_sds_cat(*s, key, key_len); - if (tmp == NULL) { - return NULL; - } - *s = tmp; - - end_len = flb_sds_len(*s); - for(i=start_len; i < end_len; i++) { - if (!valid_char[(unsigned char)(*s)[i]]) { - (*s)[i] = '_'; - } - } - } - - if (in_array == FLB_FALSE) { - tmp = flb_sds_cat(*s, "\":", 2); - if (tmp == NULL) { - return NULL; - } - *s = tmp; - } - else { - tmp = flb_sds_cat(*s, "=", 1); - if (tmp == NULL) { - return NULL; - } - *s = tmp; - } - - return *s; -} - -static flb_sds_t flb_msgpack_gelf_value(flb_sds_t *s, int quote, - const char *val, int val_len) -{ - flb_sds_t tmp; - - if (quote == FLB_TRUE) { - tmp = flb_sds_cat(*s, "\"", 1); - if (tmp == NULL) { - return NULL; - } - *s = tmp; - - if (val_len > 0) { - tmp = flb_sds_cat_utf8(s, val, val_len); - if (tmp == NULL) { - return NULL; - } - *s = tmp; - } - - tmp = flb_sds_cat(*s, "\"", 1); - if (tmp == NULL) { - return NULL; - } - *s = tmp; - } - else { - tmp = flb_sds_cat(*s, val, val_len); - if (tmp == NULL) { - return NULL; - } - *s = tmp; - } - - return *s; -} - -static flb_sds_t flb_msgpack_gelf_value_ext(flb_sds_t *s, int quote, - const char *val, int val_len) -{ - static const char int2hex[] = "0123456789abcdef"; - flb_sds_t tmp; - - if (quote == FLB_TRUE) { - tmp = flb_sds_cat(*s, "\"", 1); - if (tmp == NULL) { - return NULL; - } - *s = tmp; - } - /* ext body. fortmat is similar to printf(1) */ - { - int i; - char temp[5]; - for(i=0; i < val_len; i++) { - char c = (char)val[i]; - temp[0] = '\\'; - temp[1] = 'x'; - temp[2] = int2hex[ (unsigned char) ((c & 0xf0) >> 4)]; - temp[3] = int2hex[ (unsigned char) (c & 0x0f)]; - temp[4] = '\0'; - tmp = flb_sds_cat(*s, temp, 4); - if (tmp == NULL) { - return NULL; - } - *s = tmp; - } - } - if (quote == FLB_TRUE) { - tmp = flb_sds_cat(*s, "\"", 1); - if (tmp == NULL) { - return NULL; - } - *s = tmp; - } - - return *s; -} - -static flb_sds_t flb_msgpack_gelf_flatten(flb_sds_t *s, msgpack_object *o, - const char *prefix, int prefix_len, - int in_array) -{ - int i; - int loop; - flb_sds_t tmp; - - switch(o->type) { - case MSGPACK_OBJECT_NIL: - tmp = flb_sds_cat(*s, "null", 4); - if (tmp == NULL) { - return NULL; - } - *s = tmp; - break; - - case MSGPACK_OBJECT_BOOLEAN: - if (o->via.boolean) { - tmp = flb_msgpack_gelf_value(s, !in_array, "true", 4); - } - else { - tmp = flb_msgpack_gelf_value(s, !in_array, "false", 5); - } - if (tmp == NULL) { - return NULL; - } - *s = tmp; - break; - - case MSGPACK_OBJECT_POSITIVE_INTEGER: - tmp = flb_sds_printf(s, "%lu", (unsigned long)o->via.u64); - if (tmp == NULL) { - return NULL; - } - *s = tmp; - break; - - case MSGPACK_OBJECT_NEGATIVE_INTEGER: - tmp = flb_sds_printf(s, "%ld", (signed long)o->via.i64); - if (tmp == NULL) { - return NULL; - } - *s = tmp; - break; - - case MSGPACK_OBJECT_FLOAT32: - case MSGPACK_OBJECT_FLOAT64: - tmp = flb_sds_printf(s, "%f", o->via.f64); - if (tmp == NULL) { - return NULL; - } - *s = tmp; - break; - - case MSGPACK_OBJECT_STR: - tmp = flb_msgpack_gelf_value(s, !in_array, - o->via.str.ptr, - o->via.str.size); - if (tmp == NULL) { - return NULL; - } - *s = tmp; - break; - - case MSGPACK_OBJECT_BIN: - tmp = flb_msgpack_gelf_value(s, !in_array, - o->via.bin.ptr, - o->via.bin.size); - if (tmp == NULL) { - return NULL; - } - *s = tmp; - break; - - case MSGPACK_OBJECT_EXT: - tmp = flb_msgpack_gelf_value_ext(s, !in_array, - o->via.ext.ptr, - o->via.ext.size); - if (tmp == NULL) { - return NULL; - } - *s = tmp; - break; - - case MSGPACK_OBJECT_ARRAY: - loop = o->via.array.size; - - if (!in_array) { - tmp = flb_sds_cat(*s, "\"", 1); - if (tmp == NULL) { - return NULL; - } - *s = tmp; - } - if (loop != 0) { - msgpack_object* p = o->via.array.ptr; - for (i=0; i<loop; i++) { - if (i > 0) { - tmp = flb_sds_cat(*s, ", ", 2); - if (tmp == NULL) { - return NULL; - } - *s = tmp; - } - tmp = flb_msgpack_gelf_flatten(s, p+i, - prefix, prefix_len, - FLB_TRUE); - if (tmp == NULL) { - return NULL; - } - *s = tmp; - } - } - - if (!in_array) { - tmp = flb_sds_cat(*s, "\"", 1); - if (tmp == NULL) { - return NULL; - } - *s = tmp; - } - break; - - case MSGPACK_OBJECT_MAP: - loop = o->via.map.size; - if (loop != 0) { - msgpack_object_kv *p = o->via.map.ptr; - for (i = 0; i < loop; i++) { - msgpack_object *k = &((p+i)->key); - msgpack_object *v = &((p+i)->val); - - if (k->type != MSGPACK_OBJECT_STR) { - continue; - } - - const char *key = k->via.str.ptr; - int key_len = k->via.str.size; - - if (v->type == MSGPACK_OBJECT_MAP) { - char *obj_prefix = NULL; - int obj_prefix_len = 0; - - obj_prefix_len = key_len; - if (prefix_len > 0) { - obj_prefix_len += prefix_len + 1; - } - - obj_prefix = flb_malloc(obj_prefix_len + 1); - if (obj_prefix == NULL) { - return NULL; - } - - if (prefix_len > 0) { - memcpy(obj_prefix, prefix, prefix_len); - obj_prefix[prefix_len] = '_'; - memcpy(obj_prefix + prefix_len + 1, key, key_len); - } - else { - memcpy(obj_prefix, key, key_len); - } - obj_prefix[obj_prefix_len] = '\0'; - - tmp = flb_msgpack_gelf_flatten(s, v, - obj_prefix, obj_prefix_len, - in_array); - if (tmp == NULL) { - flb_free(obj_prefix); - return NULL; - } - *s = tmp; - - flb_free(obj_prefix); - } - else { - if (in_array == FLB_TRUE && i > 0) { - tmp = flb_sds_cat(*s, " ", 1); - if (tmp == NULL) { - return NULL; - } - *s = tmp; - } - if (in_array && prefix_len <= 0) { - tmp = flb_msgpack_gelf_key(s, in_array, - NULL, 0, - FLB_FALSE, - key, key_len); - } - else { - tmp = flb_msgpack_gelf_key(s, in_array, - prefix, prefix_len, - FLB_TRUE, - key, key_len); - } - if (tmp == NULL) { - return NULL; - } - *s = tmp; - - tmp = flb_msgpack_gelf_flatten(s, v, NULL, 0, in_array); - if (tmp == NULL) { - return NULL; - } - *s = tmp; - } - } - } - break; - - default: - flb_warn("[%s] unknown msgpack type %i", __FUNCTION__, o->type); - } - - return *s; -} - -flb_sds_t flb_msgpack_to_gelf(flb_sds_t *s, msgpack_object *o, - struct flb_time *tm, - struct flb_gelf_fields *fields) -{ - int i; - int loop; - flb_sds_t tmp; - - int host_key_found = FLB_FALSE; - int timestamp_key_found = FLB_FALSE; - int level_key_found = FLB_FALSE; - int short_message_key_found = FLB_FALSE; - int full_message_key_found = FLB_FALSE; - - char *host_key = NULL; - char *timestamp_key = NULL; - char *level_key = NULL; - char *short_message_key = NULL; - char *full_message_key = NULL; - - int host_key_len = 0; - int timestamp_key_len = false; - int level_key_len = 0; - int short_message_key_len = 0; - int full_message_key_len = 0; - - if (s == NULL || o == NULL) { - return NULL; - } - - /* Make sure the incoming object is a map */ - if (o->type != MSGPACK_OBJECT_MAP) { - return NULL; - } - - if (fields != NULL && fields->host_key != NULL) { - host_key = fields->host_key; - host_key_len = flb_sds_len(fields->host_key); - } - else { - host_key = "host"; - host_key_len = 4; - } - - if (fields != NULL && fields->timestamp_key != NULL) { - timestamp_key = fields->timestamp_key; - timestamp_key_len = flb_sds_len(fields->timestamp_key); - } - else { - timestamp_key = "timestamp"; - timestamp_key_len = 9; - } - - if (fields != NULL && fields->level_key != NULL) { - level_key = fields->level_key; - level_key_len = flb_sds_len(fields->level_key); - } - else { - level_key = "level"; - level_key_len = 5; - } - - if (fields != NULL && fields->short_message_key != NULL) { - short_message_key = fields->short_message_key; - short_message_key_len = flb_sds_len(fields->short_message_key); - } - else { - short_message_key = "short_message"; - short_message_key_len = 13; - } - - if (fields != NULL && fields->full_message_key != NULL) { - full_message_key = fields->full_message_key; - full_message_key_len = flb_sds_len(fields->full_message_key); - } - else { - full_message_key = "full_message"; - full_message_key_len = 12; - } - - tmp = flb_sds_cat(*s, "{\"version\":\"1.1\"", 16); - if (tmp == NULL) { - return NULL; - } - *s = tmp; - - loop = o->via.map.size; - if (loop != 0) { - msgpack_object_kv *p = o->via.map.ptr; - - for (i = 0; i < loop; i++) { - const char *key = NULL; - int key_len; - const char *val = NULL; - int val_len = 0; - int quote = FLB_FALSE; - int custom_key = FLB_FALSE; - - msgpack_object *k = &p[i].key; - msgpack_object *v = &p[i].val; - msgpack_object vtmp; // used when converting level value from string to int - - if (k->type != MSGPACK_OBJECT_BIN && k->type != MSGPACK_OBJECT_STR) { - continue; - } - - if (k->type == MSGPACK_OBJECT_STR) { - key = k->via.str.ptr; - key_len = k->via.str.size; - } - else { - key = k->via.bin.ptr; - key_len = k->via.bin.size; - } - - if ((key_len == host_key_len) && - !strncmp(key, host_key, host_key_len)) { - if (host_key_found == FLB_TRUE) { - continue; - } - host_key_found = FLB_TRUE; - key = "host"; - key_len = 4; - } - else if ((key_len == short_message_key_len) && - !strncmp(key, short_message_key, short_message_key_len)) { - if (short_message_key_found == FLB_TRUE) { - continue; - } - short_message_key_found = FLB_TRUE; - key = "short_message"; - key_len = 13; - } - else if ((key_len == timestamp_key_len) && - !strncmp(key, timestamp_key, timestamp_key_len)) { - if (timestamp_key_found == FLB_TRUE) { - continue; - } - timestamp_key_found = FLB_TRUE; - key = "timestamp"; - key_len = 9; - } - else if ((key_len == level_key_len) && - !strncmp(key, level_key, level_key_len )) { - if (level_key_found == FLB_TRUE) { - continue; - } - level_key_found = FLB_TRUE; - key = "level"; - key_len = 5; - if (v->type == MSGPACK_OBJECT_POSITIVE_INTEGER) { - if ( v->via.u64 > 7 ) { - flb_warn("[flb_msgpack_to_gelf] level is %" PRIu64 ", " - "but should be in 0..7 or a syslog keyword", v->via.u64); - } - } - else if (v->type == MSGPACK_OBJECT_STR) { - val = v->via.str.ptr; - val_len = v->via.str.size; - if (val_len == 1 && val[0] >= '0' && val[0] <= '7') { - v = &vtmp; - v->type = MSGPACK_OBJECT_POSITIVE_INTEGER; - v->via.u64 = (uint64_t)(val[0] - '0'); - } - else { - int n; - char* allowed_levels[] = { - "emerg", "alert", "crit", "err", - "warning", "notice", "info", "debug", - NULL - }; - for (n = 0; allowed_levels[n] != NULL; ++n) { - if (val_len == strlen(allowed_levels[n]) && - !strncasecmp(val, allowed_levels[n], val_len)) { - v = &vtmp; - v->type = MSGPACK_OBJECT_POSITIVE_INTEGER; - v->via.u64 = (uint64_t)n; - break; - } - } - if (allowed_levels[n] == NULL) { - flb_warn("[flb_msgpack_to_gelf] level is '%.*s', " - "but should be in 0..7 or a syslog keyword", val_len, val); - } - } - } - else { - flb_error("[flb_msgpack_to_gelf] level must be a non-negative integer or a string"); - return NULL; - } - } - else if ((key_len == full_message_key_len) && - !strncmp(key, full_message_key, full_message_key_len)) { - if (full_message_key_found == FLB_TRUE) { - continue; - } - full_message_key_found = FLB_TRUE; - key = "full_message"; - key_len = 12; - } - else if ((key_len == 2) && !strncmp(key, "id", 2)) { - /* _id key not allowed */ - continue; - } - else { - custom_key = FLB_TRUE; - } - - if (v->type == MSGPACK_OBJECT_MAP) { - char *prefix = NULL; - int prefix_len = 0; - - prefix_len = key_len + 1; - prefix = flb_calloc(1, prefix_len + 1); - if (prefix == NULL) { - return NULL; - } - - prefix[0] = '_'; - strncpy(prefix + 1, key, key_len); - prefix[prefix_len] = '\0'; - - tmp = flb_msgpack_gelf_flatten(s, v, - prefix, prefix_len, FLB_FALSE); - if (tmp == NULL) { - flb_free(prefix); - return NULL; - } - *s = tmp; - flb_free(prefix); - - } - else if (v->type == MSGPACK_OBJECT_ARRAY) { - if (custom_key == FLB_TRUE) { - tmp = flb_msgpack_gelf_key(s, FLB_FALSE, "_", 1, FLB_FALSE, - key, key_len); - } - else { - tmp = flb_msgpack_gelf_key(s, FLB_FALSE, NULL, 0, FLB_FALSE, - key, key_len); - } - if (tmp == NULL) { - return NULL; - } - *s = tmp; - - tmp = flb_msgpack_gelf_flatten(s, v, NULL, 0, FLB_FALSE); - if (tmp == NULL) { - return NULL; - } - *s = tmp; - } - else { - char temp[48] = {0}; - if (v->type == MSGPACK_OBJECT_NIL) { - val = "null"; - val_len = 4; - continue; - } - else if (v->type == MSGPACK_OBJECT_BOOLEAN) { - quote = FLB_TRUE; - val = v->via.boolean ? "true" : "false"; - val_len = v->via.boolean ? 4 : 5; - } - else if (v->type == MSGPACK_OBJECT_POSITIVE_INTEGER) { - val = temp; - val_len = snprintf(temp, sizeof(temp) - 1, - "%" PRIu64, v->via.u64); - /* - * Check if the value length is larger than our string. - * this is needed to avoid stack-based overflows. - */ - if (val_len > sizeof(temp)) { - return NULL; - } - } - else if (v->type == MSGPACK_OBJECT_NEGATIVE_INTEGER) { - val = temp; - val_len = snprintf(temp, sizeof(temp) - 1, - "%" PRId64, v->via.i64); - /* - * Check if the value length is larger than our string. - * this is needed to avoid stack-based overflows. - */ - if (val_len > sizeof(temp)) { - return NULL; - } - } - else if (v->type == MSGPACK_OBJECT_FLOAT) { - val = temp; - val_len = snprintf(temp, sizeof(temp) - 1, - "%f", v->via.f64); - /* - * Check if the value length is larger than our string. - * this is needed to avoid stack-based overflows. - */ - if (val_len > sizeof(temp)) { - return NULL; - } - } - else if (v->type == MSGPACK_OBJECT_STR) { - /* String value */ - quote = FLB_TRUE; - val = v->via.str.ptr; - val_len = v->via.str.size; - } - else if (v->type == MSGPACK_OBJECT_BIN) { - /* Bin value */ - quote = FLB_TRUE; - val = v->via.bin.ptr; - val_len = v->via.bin.size; - } - else if (v->type == MSGPACK_OBJECT_EXT) { - quote = FLB_TRUE; - val = v->via.ext.ptr; - val_len = v->via.ext.size; - } - - if (!val || !key) { - continue; - } - - if (custom_key == FLB_TRUE) { - tmp = flb_msgpack_gelf_key(s, FLB_FALSE, "_", 1, FLB_FALSE, - key, key_len); - } - else { - tmp = flb_msgpack_gelf_key(s, FLB_FALSE, NULL, 0, FLB_FALSE, - key, key_len); - } - if (tmp == NULL) { - return NULL; - } - *s = tmp; - - if (v->type == MSGPACK_OBJECT_EXT) { - tmp = flb_msgpack_gelf_value_ext(s, quote, val, val_len); - } - else { - tmp = flb_msgpack_gelf_value(s, quote, val, val_len); - } - if (tmp == NULL) { - return NULL; - } - *s = tmp; - } - } - } - - if (timestamp_key_found == FLB_FALSE && tm != NULL) { - tmp = flb_msgpack_gelf_key(s, FLB_FALSE, NULL, 0, FLB_FALSE, - "timestamp", 9); - if (tmp == NULL) { - return NULL; - } - *s = tmp; - - /* gelf supports milliseconds */ - tmp = flb_sds_printf(s, "%li.%03lu", - (long)tm->tm.tv_sec, tm->tm.tv_nsec / 1000000); - if (tmp == NULL) { - return NULL; - } - *s = tmp; - } - - if (short_message_key_found == FLB_FALSE) { - flb_error("[flb_msgpack_to_gelf] missing short_message key"); - return NULL; - } - - tmp = flb_sds_cat(*s, "}", 1); - if (tmp == NULL) { - return NULL; - } - *s = tmp; - - return *s; -} - -flb_sds_t flb_msgpack_raw_to_gelf(char *buf, size_t buf_size, - struct flb_time *tm, struct flb_gelf_fields *fields) -{ - int ret; - size_t off = 0; - size_t gelf_size; - msgpack_unpacked result; - flb_sds_t s; - flb_sds_t tmp; - - if (!buf || buf_size <= 0) { - return NULL; - } - - msgpack_unpacked_init(&result); - ret = msgpack_unpack_next(&result, buf, buf_size, &off); - if (ret != MSGPACK_UNPACK_SUCCESS) { - msgpack_unpacked_destroy(&result); - return NULL; - } - - gelf_size = (buf_size * 1.3); - s = flb_sds_create_size(gelf_size); - if (s == NULL) { - msgpack_unpacked_destroy(&result); - return NULL; - } - - tmp = flb_msgpack_to_gelf(&s, &result.data, tm, fields); - if (tmp == NULL) { - flb_sds_destroy(s); - msgpack_unpacked_destroy(&result); - return NULL; - } - s = tmp; - - msgpack_unpacked_destroy(&result); - - return s; -} diff --git a/fluent-bit/src/flb_parser.c b/fluent-bit/src/flb_parser.c deleted file mode 100644 index 4ccecc91b..000000000 --- a/fluent-bit/src/flb_parser.c +++ /dev/null @@ -1,1304 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <fluent-bit/flb_info.h> -#include <fluent-bit/flb_log.h> -#include <fluent-bit/flb_mem.h> -#include <fluent-bit/flb_str.h> -#include <fluent-bit/flb_parser.h> -#include <fluent-bit/flb_parser_decoder.h> -#include <fluent-bit/flb_time.h> -#include <fluent-bit/flb_error.h> -#include <fluent-bit/flb_utils.h> -#include <fluent-bit/flb_config.h> -#include <fluent-bit/flb_strptime.h> -#include <fluent-bit/flb_env.h> -#include <fluent-bit/flb_str.h> -#include <fluent-bit/flb_kv.h> -#include <fluent-bit/flb_config_format.h> -#include <fluent-bit/multiline/flb_ml.h> -#include <fluent-bit/multiline/flb_ml_parser.h> -#include <fluent-bit/multiline/flb_ml_rule.h> - -#include <cfl/cfl.h> -#include <cfl/cfl_kvlist.h> - -#include <sys/types.h> -#include <sys/stat.h> -#include <limits.h> -#include <string.h> - -static inline uint32_t digits10(uint64_t v) { - if (v < 10) return 1; - if (v < 100) return 2; - if (v < 1000) return 3; - if (v < 1000000000000UL) { - if (v < 100000000UL) { - if (v < 1000000) { - if (v < 10000) return 4; - return 5 + (v >= 100000); - } - return 7 + (v >= 10000000UL); - } - if (v < 10000000000UL) { - return 9 + (v >= 1000000000UL); - } - return 11 + (v >= 100000000000UL); - } - return 12 + digits10(v / 1000000000000UL); -} - -static unsigned u64_to_str(uint64_t value, char* dst) { - static const char digits[201] = - "0001020304050607080910111213141516171819" - "2021222324252627282930313233343536373839" - "4041424344454647484950515253545556575859" - "6061626364656667686970717273747576777879" - "8081828384858687888990919293949596979899"; - uint32_t const length = digits10(value); - uint32_t next = length - 1; - while (value >= 100) { - int const i = (value % 100) * 2; - value /= 100; - dst[next] = digits[i + 1]; - dst[next - 1] = digits[i]; - next -= 2; - } - - /* Handle last 1-2 digits */ - if (value < 10) { - dst[next] = '0' + (uint32_t) value; - } else { - int i = (uint32_t) value * 2; - dst[next] = digits[i + 1]; - dst[next - 1] = digits[i]; - } - return length; -} - -int flb_parser_regex_do(struct flb_parser *parser, - const char *buf, size_t length, - void **out_buf, size_t *out_size, - struct flb_time *out_time); - -int flb_parser_json_do(struct flb_parser *parser, - const char *buf, size_t length, - void **out_buf, size_t *out_size, - struct flb_time *out_time); - -int flb_parser_ltsv_do(struct flb_parser *parser, - const char *buf, size_t length, - void **out_buf, size_t *out_size, - struct flb_time *out_time); - -int flb_parser_logfmt_do(struct flb_parser *parser, - const char *buf, size_t length, - void **out_buf, size_t *out_size, - struct flb_time *out_time); - -/* - * This function is used to free all aspects of a parser - * which is provided by the caller of flb_create_parser. - * Specifically, this function frees all but parser.types and - * parser.decoders from a parser. - * - * This function is only to be used in parser creation routines. - */ -static void flb_interim_parser_destroy(struct flb_parser *parser) -{ - if (parser->type == FLB_PARSER_REGEX) { - flb_regex_destroy(parser->regex); - flb_free(parser->p_regex); - } - - flb_free(parser->name); - if (parser->time_fmt) { - flb_free(parser->time_fmt); - } - if (parser->time_fmt_year) { - flb_free(parser->time_fmt_year); - } - if (parser->time_fmt_full) { - flb_free(parser->time_fmt_full); - } - if (parser->time_key) { - flb_free(parser->time_key); - } - - mk_list_del(&parser->_head); - flb_free(parser); -} - -struct flb_parser *flb_parser_create(const char *name, const char *format, - const char *p_regex, - int skip_empty, - const char *time_fmt, const char *time_key, - const char *time_offset, - int time_keep, - int time_strict, - int logfmt_no_bare_keys, - struct flb_parser_types *types, - int types_len, - struct mk_list *decoders, - struct flb_config *config) -{ - int ret; - int len; - int diff = 0; - int size; - int is_epoch = FLB_FALSE; - char *tmp; - char *timeptr; - struct mk_list *head; - struct flb_parser *p; - struct flb_regex *regex; - - /* Iterate current parsers and make sure the new one don't exists */ - mk_list_foreach(head, &config->parsers) { - p = mk_list_entry(head, struct flb_parser, _head); - if (p->name && strcmp(p->name, name) == 0) { - flb_error("[parser] parser named '%s' already exists, skip.", - name); - return NULL; - } - } - - /* Allocate context */ - p = flb_calloc(1, sizeof(struct flb_parser)); - if (!p) { - flb_errno(); - return NULL; - } - p->decoders = decoders; - mk_list_add(&p->_head, &config->parsers); - - /* Format lookup */ - if (strcasecmp(format, "regex") == 0) { - p->type = FLB_PARSER_REGEX; - } - else if (strcasecmp(format, "json") == 0) { - p->type = FLB_PARSER_JSON; - } - else if (strcasecmp(format, "ltsv") == 0) { - p->type = FLB_PARSER_LTSV; - } - else if (strcasecmp(format, "logfmt") == 0) { - p->type = FLB_PARSER_LOGFMT; - } - else { - flb_error("[parser:%s] Invalid format %s", name, format); - mk_list_del(&p->_head); - flb_free(p); - return NULL; - } - - if (p->type == FLB_PARSER_REGEX) { - if (!p_regex) { - flb_error("[parser:%s] Invalid regex pattern", name); - mk_list_del(&p->_head); - flb_free(p); - return NULL; - } - - regex = flb_regex_create(p_regex); - if (!regex) { - flb_error("[parser:%s] Invalid regex pattern %s", name, p_regex); - mk_list_del(&p->_head); - flb_free(p); - return NULL; - } - p->regex = regex; - p->skip_empty = skip_empty; - p->p_regex = flb_strdup(p_regex); - } - - p->name = flb_strdup(name); - - if (time_fmt) { - p->time_fmt_full = flb_strdup(time_fmt); - if (!p->time_fmt_full) { - flb_error("[parser:%s] could not duplicate time fmt full", name); - flb_interim_parser_destroy(p); - return NULL; - } - p->time_fmt = flb_strdup(time_fmt); - if (!p->time_fmt) { - flb_error("[parser:%s] could not duplicate time fmt", name); - flb_interim_parser_destroy(p); - return NULL; - } - - /* Check if the format is considering the year */ - if (strstr(p->time_fmt, "%Y") || strstr(p->time_fmt, "%y")) { - p->time_with_year = FLB_TRUE; - } - else if (strstr(p->time_fmt, "%s")) { - is_epoch = FLB_TRUE; - p->time_with_year = FLB_TRUE; - } - else { - size = strlen(p->time_fmt); - p->time_with_year = FLB_FALSE; - p->time_fmt_year = flb_malloc(size + 4); - if (!p->time_fmt_year) { - flb_errno(); - flb_interim_parser_destroy(p); - return NULL; - } - - /* Append the year at the beginning */ - tmp = p->time_fmt_year; - *tmp++ = '%'; - *tmp++ = 'Y'; - *tmp++ = ' '; - - memcpy(tmp, p->time_fmt, size); - tmp += size; - *tmp++ = '\0'; - } - - /* Check if the format contains a timezone (%z) */ - if (strstr(p->time_fmt, "%z") || strstr(p->time_fmt, "%Z") || - strstr(p->time_fmt, "%SZ") || strstr(p->time_fmt, "%S.%LZ")) { -#if defined(FLB_HAVE_GMTOFF) || !defined(FLB_HAVE_SYSTEM_STRPTIME) - p->time_with_tz = FLB_TRUE; -#else - flb_error("[parser] timezone offset not supported"); - flb_error("[parser] you cannot use %%z/%%Z on this platform"); - flb_interim_parser_destroy(p); - return NULL; -#endif - } - - /* - * Check if the format expect fractional seconds - * - * Since strptime(3) does not support fractional seconds, this - * requires a workaround/hack in our parser. This is a known - * issue and addressed in different ways in other languages. - * - * The following links are a good reference: - * - * - http://stackoverflow.com/questions/7114690/how-to-parse-syslog-timestamp - * - http://code.activestate.com/lists/python-list/521885 - */ - if (is_epoch == FLB_TRUE || p->time_with_year == FLB_TRUE) { - timeptr = p->time_fmt; - } - else { - timeptr = p->time_fmt_year; - } - - tmp = strstr(timeptr, "%L"); - if (tmp) { - tmp[0] = '\0'; - tmp[1] = '\0'; - p->time_frac_secs = (tmp + 2); - } - - /* Optional fixed timezone offset */ - if (time_offset) { - diff = 0; - len = strlen(time_offset); - ret = flb_parser_tzone_offset(time_offset, len, &diff); - if (ret == -1) { - flb_interim_parser_destroy(p); - return NULL; - } - p->time_offset = diff; - } - } - - if (time_key) { - p->time_key = flb_strdup(time_key); - } - - p->time_keep = time_keep; - p->time_strict = time_strict; - p->logfmt_no_bare_keys = logfmt_no_bare_keys; - p->types = types; - p->types_len = types_len; - return p; -} - -void flb_parser_destroy(struct flb_parser *parser) -{ - int i = 0; - - if (parser->type == FLB_PARSER_REGEX) { - flb_regex_destroy(parser->regex); - flb_free(parser->p_regex); - } - - flb_free(parser->name); - if (parser->time_fmt) { - flb_free(parser->time_fmt); - flb_free(parser->time_fmt_full); - } - if (parser->time_fmt_year) { - flb_free(parser->time_fmt_year); - } - if (parser->time_key) { - flb_free(parser->time_key); - } - if (parser->types_len != 0) { - for (i=0; i<parser->types_len; i++){ - flb_free(parser->types[i].key); - } - flb_free(parser->types); - } - - if (parser->decoders) { - flb_parser_decoder_list_destroy(parser->decoders); - } - - mk_list_del(&parser->_head); - flb_free(parser); -} - -void flb_parser_exit(struct flb_config *config) -{ - struct mk_list *tmp; - struct mk_list *head; - struct flb_parser *parser; - - /* release 'parsers' */ - mk_list_foreach_safe(head, tmp, &config->parsers) { - parser = mk_list_entry(head, struct flb_parser, _head); - flb_parser_destroy(parser); - } - - /* release 'multiline parsers' */ - flb_ml_exit(config); -} - -static int proc_types_str(const char *types_str, struct flb_parser_types **types) -{ - int i = 0; - int types_num = 0; - char *type_str = NULL; - size_t len; - struct mk_list *split; - struct mk_list *head; - struct flb_split_entry *sentry; - - split = flb_utils_split(types_str, ' ', 256); - types_num = mk_list_size(split); - *types = flb_malloc(sizeof(struct flb_parser_types) * types_num); - - for(i=0; i<types_num; i++){ - (*types)[i].key = NULL; - (*types)[i].type = FLB_PARSER_TYPE_STRING; - } - i = 0; - mk_list_foreach(head ,split) { - sentry = mk_list_entry(head, struct flb_split_entry ,_head); - type_str = strchr(sentry->value ,':'); - - if (type_str == NULL) { - i++; - continue; - } - len = type_str - sentry->value; - (*types)[i].key = flb_strndup(sentry->value, len); - (*types)[i].key_len = len; - - type_str++; - if (!strcasecmp(type_str, "integer")) { - (*types)[i].type = FLB_PARSER_TYPE_INT; - } - else if(!strcasecmp(type_str, "bool")) { - (*types)[i].type = FLB_PARSER_TYPE_BOOL; - } - else if(!strcasecmp(type_str, "float")){ - (*types)[i].type = FLB_PARSER_TYPE_FLOAT; - } - else if(!strcasecmp(type_str, "hex")){ - (*types)[i].type = FLB_PARSER_TYPE_HEX; - } - else { - (*types)[i].type = FLB_PARSER_TYPE_STRING; - } - i++; - } - flb_utils_split_free(split); - - return i; -} - -static flb_sds_t get_parser_key(struct flb_config *config, - struct flb_cf *cf, struct flb_cf_section *s, - char *key) - -{ - flb_sds_t tmp; - flb_sds_t val; - - tmp = flb_cf_section_property_get_string(cf, s, key); - if (!tmp) { - return NULL; - } - - val = flb_env_var_translate(config->env, tmp); - if (!val) { - flb_sds_destroy(tmp); - return NULL; - } - - if (flb_sds_len(val) == 0) { - flb_sds_destroy(val); - flb_sds_destroy(tmp); - return NULL; - } - - flb_sds_destroy(tmp); - return val; -} - -/* Config file: read 'parser' definitions */ -static int parser_conf_file(const char *cfg, struct flb_cf *cf, - struct flb_config *config) -{ - int i = 0; - flb_sds_t name; - flb_sds_t format; - flb_sds_t regex; - flb_sds_t time_fmt; - flb_sds_t time_key; - flb_sds_t time_offset; - flb_sds_t types_str; - flb_sds_t tmp_str; - int skip_empty; - int time_keep; - int time_strict; - int logfmt_no_bare_keys; - int types_len; - struct mk_list *head; - struct mk_list *decoders = NULL; - struct flb_cf_section *s; - struct flb_parser_types *types = NULL; - - /* Read all 'parser' sections */ - mk_list_foreach(head, &cf->parsers) { - name = NULL; - format = NULL; - regex = NULL; - time_fmt = NULL; - time_key = NULL; - time_offset = NULL; - types_str = NULL; - tmp_str = NULL; - - /* retrieve the section context */ - s = mk_list_entry(head, struct flb_cf_section, _head_section); - - /* name */ - name = get_parser_key(config, cf, s, "name"); - if (!name) { - flb_error("[parser] no parser 'name' found in file '%s'", cfg); - goto fconf_early_error; - } - - /* format */ - format = get_parser_key(config, cf, s, "format"); - if (!format) { - flb_error("[parser] no parser 'format' found for '%s' in file '%s'", - name, cfg); - goto fconf_early_error; - } - - /* regex (if 'format' == 'regex') */ - regex = get_parser_key(config, cf, s, "regex"); - if (!regex && strcmp(format, "regex") == 0) { - flb_error("[parser] no parser 'regex' found for '%s' in file '%s", - name, cfg); - goto fconf_early_error; - } - - /* skip_empty_values */ - skip_empty = FLB_TRUE; - tmp_str = get_parser_key(config, cf, s, "skip_empty_values"); - if (tmp_str) { - skip_empty = flb_utils_bool(tmp_str); - flb_sds_destroy(tmp_str); - } - - /* time_format */ - time_fmt = get_parser_key(config, cf, s, "time_format"); - - /* time_key */ - time_key = get_parser_key(config, cf, s, "time_key"); - - /* time_keep */ - time_keep = FLB_FALSE; - tmp_str = get_parser_key(config, cf, s, "time_keep"); - if (tmp_str) { - time_keep = flb_utils_bool(tmp_str); - flb_sds_destroy(tmp_str); - } - - /* time_strict */ - time_strict = FLB_TRUE; - tmp_str = get_parser_key(config, cf, s, "time_strict"); - if (tmp_str) { - time_strict = flb_utils_bool(tmp_str); - flb_sds_destroy(tmp_str); - } - - /* time_offset (UTC offset) */ - time_offset = get_parser_key(config, cf, s, "time_offset"); - - /* logfmt_no_bare_keys */ - logfmt_no_bare_keys = FLB_FALSE; - tmp_str = get_parser_key(config, cf, s, "logfmt_no_bare_keys"); - if (tmp_str) { - logfmt_no_bare_keys = flb_utils_bool(tmp_str); - flb_sds_destroy(tmp_str); - } - - /* types */ - types_str = get_parser_key(config, cf, s, "types"); - if (types_str) { - types_len = proc_types_str(types_str, &types); - } - else { - types_len = 0; - } - - /* Decoders */ - decoders = flb_parser_decoder_list_create(s); - - /* Create the parser context */ - if (!flb_parser_create(name, format, regex, skip_empty, - time_fmt, time_key, time_offset, time_keep, time_strict, - logfmt_no_bare_keys, types, types_len, decoders, config)) { - goto fconf_error; - } - - flb_debug("[parser] new parser registered: %s", name); - - flb_sds_destroy(name); - flb_sds_destroy(format); - - if (regex) { - flb_sds_destroy(regex); - } - if (time_fmt) { - flb_sds_destroy(time_fmt); - } - if (time_key) { - flb_sds_destroy(time_key); - } - if (time_offset) { - flb_sds_destroy(time_offset); - } - if (types_str) { - flb_sds_destroy(types_str); - } - decoders = NULL; - } - - return 0; - - /* Use early exit before call to flb_parser_create */ - fconf_early_error: - if (name) { - flb_sds_destroy(name); - } - if (format) { - flb_sds_destroy(format); - } - if (regex) { - flb_sds_destroy(regex); - } - return -1; - - fconf_error: - flb_sds_destroy(name); - flb_sds_destroy(format); - if (regex) { - flb_sds_destroy(regex); - } - if (time_fmt) { - flb_sds_destroy(time_fmt); - } - if (time_key) { - flb_sds_destroy(time_key); - } - if (time_offset) { - flb_sds_destroy(time_offset); - } - if (types_str) { - flb_sds_destroy(types_str); - } - if (types_len) { - for (i=0; i<types_len; i++){ - if (types[i].key != NULL) { - flb_free(types[i].key); - } - } - flb_free(types); - } - if (decoders) { - flb_parser_decoder_list_destroy(decoders); - } - return -1; -} - -static int multiline_load_regex_rules(struct flb_ml_parser *ml_parser, - struct flb_cf_section *section, - struct flb_config *config) -{ - int ret; - char *to_state = NULL; - struct mk_list list; - struct cfl_list *head; - struct cfl_kvpair *entry; - struct flb_slist_entry *from_state; - struct flb_slist_entry *regex_pattern; - struct flb_slist_entry *tmp; - - cfl_list_foreach(head, §ion->properties->list) { - entry = cfl_list_entry(head, struct cfl_kvpair, _head); - - /* only process 'rule' keys */ - if (strcasecmp(entry->key, "rule") != 0) { - continue; - } - - mk_list_init(&list); - ret = flb_slist_split_tokens(&list, entry->val->data.as_string, 3); - if (ret == -1) { - flb_error("[multiline parser: %s] invalid section on key '%s'", - ml_parser->name, entry->key); - return -1; - } - - /* Get entries from the line */ - from_state = flb_slist_entry_get(&list, 0); - regex_pattern = flb_slist_entry_get(&list, 1); - tmp = flb_slist_entry_get(&list, 2); - if (tmp) { - to_state = tmp->str; - } - else { - to_state = NULL; - } - - if (!from_state) { - flb_error("[multiline parser: %s] 'from_state' is mandatory", - ml_parser->name); - flb_slist_destroy(&list); - return -1; - } - - if (!regex_pattern) { - flb_error("[multiline parser: %s] 'regex_pattern' is mandatory", - ml_parser->name); - flb_slist_destroy(&list); - return -1; - } - - ret = flb_ml_rule_create(ml_parser, - from_state->str, - regex_pattern->str, - to_state, - NULL); - if (ret == -1) { - flb_error("[multiline parser: %s] error creating rule", - ml_parser->name); - flb_slist_destroy(&list); - return -1; - } - - flb_slist_destroy(&list); - } - - /* Map the rules (mandatory for regex rules) */ - ret = flb_ml_parser_init(ml_parser); - if (ret != 0) { - flb_error("[multiline parser: %s] invalid mapping rules, check the states", - ml_parser->name); - return -1; - } - - return 0; -} - - -/* config file: read 'multiline_parser' sections */ -static int multiline_parser_conf_file(const char *cfg, struct flb_cf *cf, - struct flb_config *config) -{ - int ret; - int type; - flb_sds_t name; - flb_sds_t match_string; - int negate; - flb_sds_t key_content; - flb_sds_t key_pattern; - flb_sds_t key_group; - flb_sds_t parser; - flb_sds_t tmp; - int flush_timeout; - struct flb_parser *parser_ctx = NULL; - struct mk_list *head; - struct flb_cf_section *s; - struct flb_ml_parser *ml_parser; - - /* read all 'multiline_parser' sections */ - mk_list_foreach(head, &cf->multiline_parsers) { - ml_parser = NULL; - name = NULL; - type = -1; - match_string = NULL; - negate = FLB_FALSE; - key_content = NULL; - key_pattern = NULL; - key_group = NULL; - parser = NULL; - flush_timeout = -1; - tmp = NULL; - - s = mk_list_entry(head, struct flb_cf_section, _head_section); - - /* name */ - name = get_parser_key(config, cf, s, "name"); - if (!name) { - flb_error("[multiline_parser] no 'name' defined in file '%s'", cfg); - goto fconf_error; - } - - /* type */ - tmp = get_parser_key(config, cf, s, "type"); - if (!tmp) { - flb_error("[multiline_parser] no 'type' defined in file '%s'", cfg); - goto fconf_error; - } - else { - type = flb_ml_type_lookup(tmp); - if (type == -1) { - flb_error("[multiline_parser] invalid type '%s'", tmp); - goto fconf_error; - } - flb_sds_destroy(tmp); - } - - /* match_string */ - match_string = get_parser_key(config, cf, s, "match_string"); - - /* negate */ - tmp = get_parser_key(config, cf, s, "negate"); - if (tmp) { - negate = flb_utils_bool(tmp); - flb_sds_destroy(tmp); - } - - /* key_content */ - key_content = get_parser_key(config, cf, s, "key_content"); - - /* key_pattern */ - key_pattern = get_parser_key(config, cf, s, "key_pattern"); - - /* key_group */ - key_group = get_parser_key(config, cf, s, "key_group"); - - /* parser */ - parser = get_parser_key(config, cf, s, "parser"); - - /* flush_timeout */ - tmp = get_parser_key(config, cf, s, "flush_timeout"); - if (tmp) { - flush_timeout = atoi(tmp); - } - - if (parser) { - parser_ctx = flb_parser_get(parser, config); - } - ml_parser = flb_ml_parser_create(config, name, type, match_string, - negate, flush_timeout, key_content, - key_group, key_pattern, - parser_ctx, parser); - if (!ml_parser) { - goto fconf_error; - } - - /* if type is regex, process rules */ - if (type == FLB_ML_REGEX) { - ret = multiline_load_regex_rules(ml_parser, s, config); - if (ret != 0) { - goto fconf_error; - } - } - - flb_sds_destroy(name); - flb_sds_destroy(match_string); - flb_sds_destroy(key_content); - flb_sds_destroy(key_pattern); - flb_sds_destroy(key_group); - flb_sds_destroy(parser); - flb_sds_destroy(tmp); - } - - return 0; - - fconf_error: - if (ml_parser) { - flb_ml_parser_destroy(ml_parser); - } - flb_sds_destroy(name); - flb_sds_destroy(match_string); - flb_sds_destroy(key_content); - flb_sds_destroy(key_pattern); - flb_sds_destroy(key_group); - flb_sds_destroy(parser); - flb_sds_destroy(tmp); - - return -1; -} - -int flb_parser_conf_file_stat(const char *file, struct flb_config *config) -{ - int ret; - struct stat st; - - ret = stat(file, &st); - if (ret == -1 && errno == ENOENT) { - /* Try to resolve the real path (if exists) */ - if (file[0] == '/') { - flb_utils_error(FLB_ERR_CFG_PARSER_FILE); - return -1; - } - - if (config->conf_path) { - /* Handle as special case here. */ - return -2; - } - - return -1; - } - - return 0; -} - -/* Load parsers from a configuration file */ -int flb_parser_conf_file(const char *file, struct flb_config *config) -{ - int ret; - char tmp[PATH_MAX + 1]; - char *cfg = NULL; - struct flb_cf *cf = NULL; - -#ifndef FLB_HAVE_STATIC_CONF - ret = flb_parser_conf_file_stat(file, config); - if (ret == -1) { - return -1; - } - else if (ret == -2) { - snprintf(tmp, PATH_MAX, "%s%s", config->conf_path, file); - cfg = tmp; - } - else { - cfg = (char *) file; - } - - cf = flb_cf_create_from_file(NULL, cfg); -#else - cf = flb_config_static_open(file); -#endif - - if (!cf) { - return -1; - } - - /* process 'parser' sections */ - ret = parser_conf_file(cfg, cf, config); - if (ret == -1) { - flb_cf_destroy(cf); - return -1; - } - - /* processs 'multiline_parser' sections */ - ret = multiline_parser_conf_file(cfg, cf, config); - if (ret == -1) { - flb_cf_destroy(cf); - return -1; - } - - /* link the 'cf parser' context to the config list */ - mk_list_add(&cf->_head, &config->cf_parsers_list); - return 0; -} - -struct flb_parser *flb_parser_get(const char *name, struct flb_config *config) -{ - struct mk_list *head; - struct flb_parser *parser; - - if (config == NULL || mk_list_size(&config->parsers) <= 0) { - return NULL; - } - - mk_list_foreach(head, &config->parsers) { - parser = mk_list_entry(head, struct flb_parser, _head); - if (parser == NULL || parser->name == NULL) { - continue; - } - if (strcmp(parser->name, name) == 0) { - return parser; - } - } - - return NULL; -} - -int flb_parser_do(struct flb_parser *parser, const char *buf, size_t length, - void **out_buf, size_t *out_size, struct flb_time *out_time) -{ - - if (parser->type == FLB_PARSER_REGEX) { - return flb_parser_regex_do(parser, buf, length, - out_buf, out_size, out_time); - } - else if (parser->type == FLB_PARSER_JSON) { - return flb_parser_json_do(parser, buf, length, - out_buf, out_size, out_time); - } - else if (parser->type == FLB_PARSER_LTSV) { - return flb_parser_ltsv_do(parser, buf, length, - out_buf, out_size, out_time); - } - else if (parser->type == FLB_PARSER_LOGFMT) { - return flb_parser_logfmt_do(parser, buf, length, - out_buf, out_size, out_time); - } - - return -1; -} - -/* Given a timezone string, return it numeric offset */ -int flb_parser_tzone_offset(const char *str, int len, int *tmdiff) -{ - int neg; - long hour; - long min; - const char *end; - const char *p = str; - - /* Check timezones */ - if (*p == 'Z') { - /* This is UTC, no changes required */ - *tmdiff = 0; - return 0; - } - - /* Unexpected timezone string */ - if (*p != '+' && *p != '-') { - *tmdiff = 0; - return -1; - } - - /* Ensure there is enough data */ - if (len < 4) { - *tmdiff = 0; - return -1; - } - - /* Negative value ? */ - neg = (*p++ == '-'); - - /* Locate end */ - end = str + len; - - /* Gather hours and minutes */ - hour = ((p[0] - '0') * 10) + (p[1] - '0'); - if (end - p == 5 && p[2] == ':') { - /* Ensure there is enough data */ - if (len < 5) { - *tmdiff = 0; - return -1; - } - min = ((p[3] - '0') * 10) + (p[4] - '0'); - } - else { - min = ((p[2] - '0') * 10) + (p[3] - '0'); - } - - if (hour < 0 || hour > 59 || min < 0 || min > 59) { - return -1; - } - - *tmdiff = ((hour * 3600) + (min * 60)); - if (neg) { - *tmdiff = -*tmdiff; - } - - return 0; -} - -/* - * Parse the '%L' (subseconds) part into `subsec`. - * - * 2020-10-23 12:00:31.415213 JST - * ---------- - * - * Return the number of characters consumed, or -1 on error. - */ -static int parse_subseconds(char *str, int len, double *subsec) -{ - char buf[16]; - char *end; - int consumed; - int digits = 9; /* 1 ns = 000000001 (9 digits) */ - - if (len < digits) { - digits = len; - } - memcpy(buf, "0.", 2); - memcpy(buf + 2, str, digits); - buf[digits + 2] = '\0'; - - *subsec = strtod(buf, &end); - - consumed = end - buf - 2; - if (consumed <= 0) { - return -1; - } - return consumed; -} - -int flb_parser_time_lookup(const char *time_str, size_t tsize, - time_t now, - struct flb_parser *parser, - struct flb_tm *tm, double *ns) -{ - int ret; - time_t time_now; - char *p = NULL; - char *fmt; - int time_len = tsize; - const char *time_ptr = time_str; - char tmp[64]; - struct tm tmy; - - *ns = 0; - - if (tsize > sizeof(tmp) - 1) { - flb_error("[parser] time string length is too long"); - return -1; - } - - /* - * Some records coming from old Syslog messages do not contain the - * year, so it's required to ingest this information in the value - * to be parsed. - */ - if (parser->time_with_year == FLB_FALSE) { - /* Given time string is too long */ - if (time_len + 6 >= sizeof(tmp)) { - return -1; - } - - /* - * This is not the most elegant way but for now it let - * get the work done. - */ - if (now <= 0) { - time_now = time(NULL); - } - else { - time_now = now; - } - - gmtime_r(&time_now, &tmy); - - /* Make the timestamp default to today */ - tm->tm.tm_mon = tmy.tm_mon; - tm->tm.tm_mday = tmy.tm_mday; - - uint64_t t = tmy.tm_year + 1900; - - fmt = tmp; - u64_to_str(t, fmt); - fmt += 4; - *fmt++ = ' '; - - memcpy(fmt, time_ptr, time_len); - fmt += time_len; - *fmt++ = '\0'; - - time_ptr = tmp; - time_len = strlen(tmp); - p = flb_strptime(time_ptr, parser->time_fmt_year, tm); - } - else { - /* - * We must ensure string passed to flb_strptime is - * null-terminated, which time_ptr is not guaranteed - * to be. So we use tmp to hold our string. - */ - if (time_len >= sizeof(tmp)) { - return -1; - } - memcpy(tmp, time_ptr, time_len); - tmp[time_len] = '\0'; - time_ptr = tmp; - time_len = strlen(tmp); - - p = flb_strptime(time_ptr, parser->time_fmt, tm); - } - - if (p == NULL) { - if (parser->time_strict) { - flb_error("[parser] cannot parse '%.*s'", (int)tsize, time_str); - return -1; - } - flb_debug("[parser] non-exact match '%.*s'", (int)tsize, time_str); - return 0; - } - - if (parser->time_frac_secs) { - ret = parse_subseconds(p, time_len - (p - time_ptr), ns); - if (ret < 0) { - if (parser->time_strict) { - flb_error("[parser] cannot parse %%L for '%.*s'", (int)tsize, time_str); - return -1; - } - flb_debug("[parser] non-exact match on %%L '%.*s'", (int)tsize, time_str); - return 0; - } - p += ret; - - /* Parse the remaining part after %L */ - p = flb_strptime(p, parser->time_frac_secs, tm); - if (p == NULL) { - if (parser->time_strict) { - flb_error("[parser] cannot parse '%.*s' after %%L", (int)tsize, time_str); - return -1; - } - flb_debug("[parser] non-exact match after %%L '%.*s'", (int)tsize, time_str); - return 0; - } - } - - if (parser->time_with_tz == FLB_FALSE) { - flb_tm_gmtoff(tm) = parser->time_offset; - } - - return 0; -} - -int flb_parser_typecast(const char *key, int key_len, - const char *val, int val_len, - msgpack_packer *pck, - struct flb_parser_types *types, - int types_len) -{ - int i; - int error = FLB_FALSE; - char *tmp_str; - int casted = FLB_FALSE; - - for(i=0; i<types_len; i++){ - if (types[i].key != NULL - && key_len == types[i].key_len && - !strncmp(key, types[i].key, key_len)) { - - casted = FLB_TRUE; - - msgpack_pack_str(pck, key_len); - msgpack_pack_str_body(pck, key, key_len); - - switch (types[i].type) { - case FLB_PARSER_TYPE_INT: - { - long long lval; - - /* msgpack char is not null terminated. - So make a temporary copy. - */ - tmp_str = flb_strndup(val, val_len); - lval = atoll(tmp_str); - flb_free(tmp_str); - msgpack_pack_int64(pck, lval); - } - break; - case FLB_PARSER_TYPE_HEX: - { - unsigned long long lval; - tmp_str = flb_strndup(val, val_len); - lval = strtoull(tmp_str, NULL, 16); - flb_free(tmp_str); - msgpack_pack_uint64(pck, lval); - } - break; - - case FLB_PARSER_TYPE_FLOAT: - { - double dval; - tmp_str = flb_strndup(val, val_len); - dval = atof(tmp_str); - flb_free(tmp_str); - msgpack_pack_double(pck, dval); - } - break; - case FLB_PARSER_TYPE_BOOL: - if (val_len >= 4 && !strncasecmp(val, "true", 4)) { - msgpack_pack_true(pck); - } - else if(val_len >= 5 && !strncasecmp(val, "false", 5)){ - msgpack_pack_false(pck); - } - else { - error = FLB_TRUE; - } - break; - case FLB_PARSER_TYPE_STRING: - msgpack_pack_str(pck, val_len); - msgpack_pack_str_body(pck, val, val_len); - break; - default: - error = FLB_TRUE; - } - if (error == FLB_TRUE) { - /* We need to null-terminate key for flb_warn, as it expects - * a null-terminated string, which key is not guaranteed - * to be */ - char *nt_key = flb_malloc(key_len + 1); - if (nt_key != NULL) { - memcpy(nt_key, key, key_len); - nt_key[key_len] = '\0'; - flb_warn("[PARSER] key=%s cast error. save as string.", nt_key); - flb_free(nt_key); - } - msgpack_pack_str(pck, val_len); - msgpack_pack_str_body(pck, val, val_len); - } - break; - } - } - - if (casted == FLB_FALSE) { - msgpack_pack_str(pck, key_len); - msgpack_pack_str_body(pck, key, key_len); - msgpack_pack_str(pck, val_len); - msgpack_pack_str_body(pck, val, val_len); - } - return 0; -} diff --git a/fluent-bit/src/flb_parser_decoder.c b/fluent-bit/src/flb_parser_decoder.c deleted file mode 100644 index ddcb950e0..000000000 --- a/fluent-bit/src/flb_parser_decoder.c +++ /dev/null @@ -1,777 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <fluent-bit/flb_info.h> -#include <fluent-bit/flb_mem.h> -#include <fluent-bit/flb_str.h> -#include <fluent-bit/flb_log.h> -#include <fluent-bit/flb_pack.h> -#include <fluent-bit/flb_parser_decoder.h> -#include <fluent-bit/flb_unescape.h> -#include <fluent-bit/flb_utils.h> -#include <fluent-bit/flb_kv.h> - -#include <cfl/cfl.h> -#include <cfl/cfl_kvlist.h> - -#include <msgpack.h> - -#define TYPE_OUT_STRING 0 /* unstructured text */ -#define TYPE_OUT_OBJECT 1 /* structured msgpack object */ - -/* Decode a stringified JSON message */ -static int decode_json(struct flb_parser_dec *dec, - const char *in_buf, size_t in_size, - char **out_buf, size_t *out_size, int *out_type) -{ - int ret; - int root_type; - int records; - char *buf; - const char *p; - size_t size; - size_t len; - - p = in_buf; - while (*p == ' ') p++; - - len = in_size - (p - in_buf); - - /* It must be a map or array */ - if (p[0] != '{' && p[0] != '[') { - return -1; - } - - ret = flb_pack_json_recs(p, len, &buf, &size, &root_type, &records, NULL); - if (ret != 0) { - return -1; - } - - /* We expect to decode only one JSON element */ - if (records != 1) { - flb_free(buf); - return -1; - } - - /* Only process a packed JSON object */ - if (root_type != FLB_PACK_JSON_OBJECT) { - flb_free(buf); - return -1; - } - - *out_buf = buf; - *out_size = size; - *out_type = TYPE_OUT_OBJECT; - - return 0; -} - -static int decode_escaped(struct flb_parser_dec *dec, - const char *in_buf, size_t in_size, - char **out_buf, size_t *out_size, int *out_type) -{ - int len; - - /* Unescape string */ - len = flb_unescape_string(in_buf, in_size, &dec->buffer); - *out_buf = dec->buffer; - *out_size = len; - *out_type = TYPE_OUT_STRING; - - return 0; -} - -static int decode_escaped_utf8(struct flb_parser_dec *dec, - const char *in_buf, size_t in_size, - char **out_buf, size_t *out_size, int *out_type) -{ - int len; - - len = flb_unescape_string_utf8(in_buf, in_size, dec->buffer); - *out_buf = dec->buffer; - *out_size = len; - *out_type = TYPE_OUT_STRING; - - return 0; -} - -static int decode_mysql_quoted(struct flb_parser_dec *dec, - char *in_buf, size_t in_size, - char **out_buf, size_t *out_size, int *out_type) -{ - int len; - if(in_size < 2) { - dec->buffer[0] = in_buf[0]; - dec->buffer[1] = 0; - *out_buf = dec->buffer; - *out_size = in_size; - *out_type = TYPE_OUT_STRING; - } - else if(in_buf[0] == '\'' && in_buf[in_size-1] == '\'') { - len = flb_mysql_unquote_string(in_buf+1, in_size-2, &dec->buffer); - *out_buf = dec->buffer; - *out_size = len; - *out_type = TYPE_OUT_STRING; - } - else if(in_buf[0] == '\"' && in_buf[in_size-1] == '\"') { - len = flb_mysql_unquote_string(in_buf+1, in_size-2, &dec->buffer); - *out_buf = dec->buffer; - *out_size = len; - *out_type = TYPE_OUT_STRING; - } - else { - memcpy(dec->buffer, in_buf, in_size); - dec->buffer[in_size] = 0; - *out_buf = dec->buffer; - *out_size = in_size; - *out_type = TYPE_OUT_STRING; - } - - return 0; -} - -static int merge_record_and_extra_keys(const char *in_buf, size_t in_size, - const char *extra_buf, size_t extra_size, - char **out_buf, size_t *out_size) -{ - int i; - int ret; - int map_size = 0; - size_t in_off = 0; - size_t extra_off = 0; - msgpack_sbuffer mp_sbuf; - msgpack_packer mp_pck; - msgpack_unpacked in_result; - msgpack_unpacked extra_result; - msgpack_object k; - msgpack_object v; - msgpack_object map; - - msgpack_unpacked_init(&in_result); - msgpack_unpacked_init(&extra_result); - - /* Check if the extra buffer have some serialized data */ - ret = msgpack_unpack_next(&extra_result, extra_buf, extra_size, &extra_off); - if (ret != MSGPACK_UNPACK_SUCCESS) { - msgpack_unpacked_destroy(&in_result); - msgpack_unpacked_destroy(&extra_result); - return -1; - } - msgpack_unpack_next(&in_result, in_buf, in_size, &in_off); - - - msgpack_sbuffer_init(&mp_sbuf); - msgpack_packer_init(&mp_pck, &mp_sbuf, msgpack_sbuffer_write); - - map_size = in_result.data.via.map.size; - map_size += extra_result.data.via.map.size; - - msgpack_pack_map(&mp_pck, map_size); - map = in_result.data; - for (i = 0; i < map.via.map.size; i++) { - k = map.via.map.ptr[i].key; - v = map.via.map.ptr[i].val; - msgpack_pack_object(&mp_pck, k); - msgpack_pack_object(&mp_pck, v); - } - - map = extra_result.data; - for (i = 0; i < map.via.map.size; i++) { - k = map.via.map.ptr[i].key; - v = map.via.map.ptr[i].val; - msgpack_pack_object(&mp_pck, k); - msgpack_pack_object(&mp_pck, v); - } - - msgpack_unpacked_destroy(&in_result); - msgpack_unpacked_destroy(&extra_result); - - *out_buf = mp_sbuf.data; - *out_size = mp_sbuf.size; - - return 0; -} - -/* - * Given a msgpack map, apply the parser-decoder rules defined and generate - * a new msgpack buffer. - */ -int flb_parser_decoder_do(struct mk_list *decoders, - const char *in_buf, size_t in_size, - char **out_buf, size_t *out_size) -{ - int i; - int ret; - int matched; - int is_decoded; - int is_decoded_as; - int in_type; - int out_type; - int dec_type; - int extra_keys = FLB_FALSE; - size_t off = 0; - char *dec_buf; - size_t dec_size; - flb_sds_t tmp_sds = NULL; - flb_sds_t data_sds = NULL; - flb_sds_t in_sds = NULL; - flb_sds_t out_sds = NULL; - struct mk_list *head; - struct mk_list *r_head; - struct flb_parser_dec *dec = NULL; - struct flb_parser_dec_rule *rule; - msgpack_object k; - msgpack_object v; - msgpack_object map; - msgpack_unpacked result; - msgpack_sbuffer mp_sbuf; - msgpack_packer mp_pck; - /* Contexts to handle extra keys to be appended at the end of the log */ - msgpack_sbuffer extra_mp_sbuf; - msgpack_packer extra_mp_pck; - - /* Initialize unpacker */ - msgpack_unpacked_init(&result); - msgpack_unpack_next(&result, in_buf, in_size, &off); - map = result.data; - - if (map.type != MSGPACK_OBJECT_MAP) { - msgpack_unpacked_destroy(&result); - return -1; - } - - /* - * First check if any field in the record matches a decoder rule. It's - * better to check this before hand otherwise we need to jump directly - * to create a "possible new outgoing buffer". - */ - matched = -1; - for (i = 0; i < map.via.map.size; i++) { - k = map.via.map.ptr[i].key; - if (k.type != MSGPACK_OBJECT_STR) { - continue; - } - - /* Try to match this key name with decoder's rule */ - mk_list_foreach(head, decoders) { - dec = mk_list_entry(head, struct flb_parser_dec, _head); - if (flb_sds_cmp(dec->key, k.via.str.ptr, - k.via.str.size) == 0) { - /* we have a match, stop the check */ - matched = i; - break; - } - else { - matched = -1; - } - } - - if (matched >= 0) { - break; - } - } - - /* No matches, no need to continue */ - if (matched == -1) { - msgpack_unpacked_destroy(&result); - return -1; - } - - /* Create new outgoing buffer */ - msgpack_sbuffer_init(&mp_sbuf); - msgpack_packer_init(&mp_pck, &mp_sbuf, msgpack_sbuffer_write); - - /* Register the map (same size) */ - msgpack_pack_map(&mp_pck, map.via.map.size); - - /* Compose new outgoing buffer */ - for (i = 0; i < map.via.map.size; i++) { - k = map.via.map.ptr[i].key; - v = map.via.map.ptr[i].val; - - /* Pack right away previous fields in the map */ - if (i < matched) { - msgpack_pack_object(&mp_pck, k); - msgpack_pack_object(&mp_pck, v); - continue; - } - - /* Process current key names and decoder rules */ - if (k.type != MSGPACK_OBJECT_STR || v.type != MSGPACK_OBJECT_STR) { - msgpack_pack_object(&mp_pck, k); - msgpack_pack_object(&mp_pck, v); - continue; - } - - /* - * Per key, we allow only one successful 'Decode_Field' and one - * successful 'Decode_Field_As' rules. Otherwise it may lead - * to duplicated entries in the final map. - * - * is_decoded => Decode_Field successul ? - * is_decoded_as => Decode_Field_As successful ? - */ - is_decoded = FLB_FALSE; - is_decoded_as = FLB_FALSE; - - /* Lookup for decoders associated to the current 'key' */ - mk_list_foreach(head, decoders) { - dec = mk_list_entry(head, struct flb_parser_dec, _head); - if (flb_sds_cmp(dec->key, k.via.str.ptr, - k.via.str.size) == 0) { - break; - } - dec = NULL; - } - - /* No decoder found, pack content */ - if (!dec) { - msgpack_pack_object(&mp_pck, k); - msgpack_pack_object(&mp_pck, v); - continue; - } - - if (!in_sds) { - in_sds = flb_sds_create_size(v.via.str.size); - if (!in_sds) { - break; - } - out_sds = flb_sds_create_size(v.via.str.size); - if (!out_sds) { - break; - } - data_sds = flb_sds_create_size(v.via.str.size); - } - - /* Copy original content */ - tmp_sds = flb_sds_copy(data_sds, v.via.str.ptr, - v.via.str.size); - if (tmp_sds != data_sds) { - data_sds = tmp_sds; - } - - /* - * We got a match: 'key name' == 'decoder field name', validate - * that we have enough space in our temporary buffer. - */ - if (flb_sds_alloc(dec->buffer) < flb_sds_alloc(data_sds)) { - /* Increase buffer size */ - size_t diff; - diff = (flb_sds_alloc(data_sds) - flb_sds_alloc(dec->buffer)); - tmp_sds = flb_sds_increase(dec->buffer, diff); - if (!tmp_sds) { - flb_errno(); - break; - } - if (tmp_sds != dec->buffer) { - dec->buffer = tmp_sds; - } - } - - /* Process decoder rules */ - ret = -1; - dec_buf = NULL; - - /* - * If some rule type is FLB_PARSER_DEC_DEFAULT, means that it will - * try to register some extra fields as part of the record. For such - * case we prepare a temporary buffer to hold these extra keys. - * - * The content of this buffer is just a serialized number of maps. - */ - if (dec->add_extra_keys == FLB_TRUE) { - /* We need to clean up already allocated extra buffers */ - if (extra_keys == FLB_TRUE) { - msgpack_sbuffer_destroy(&extra_mp_sbuf); - } - extra_keys = FLB_TRUE; - msgpack_sbuffer_init(&extra_mp_sbuf); - msgpack_packer_init(&extra_mp_pck, &extra_mp_sbuf, - msgpack_sbuffer_write); - } - - mk_list_foreach(r_head, &dec->rules) { - rule = mk_list_entry(r_head, struct flb_parser_dec_rule, _head); - - if (rule->type == FLB_PARSER_DEC_DEFAULT && - rule->action == FLB_PARSER_ACT_DO_NEXT && - is_decoded == FLB_TRUE) { - continue; - } - - if (is_decoded_as == FLB_TRUE && in_type != TYPE_OUT_STRING) { - continue; - } - - /* Process using defined decoder backend */ - if (rule->backend == FLB_PARSER_DEC_JSON) { - ret = decode_json(dec, (char *) data_sds, flb_sds_len(data_sds), - &dec_buf, &dec_size, &dec_type); - } - else if (rule->backend == FLB_PARSER_DEC_ESCAPED) { - ret = decode_escaped(dec, - (char *) data_sds, flb_sds_len(data_sds), - &dec_buf, &dec_size, &dec_type); - } - else if (rule->backend == FLB_PARSER_DEC_ESCAPED_UTF8) { - ret = decode_escaped_utf8(dec, - (char *) data_sds, flb_sds_len(data_sds), - &dec_buf, &dec_size, &dec_type); - } - else if (rule->backend == FLB_PARSER_DEC_MYSQL_QUOTED) { - ret = decode_mysql_quoted(dec, - (char *) data_sds, flb_sds_len(data_sds), - &dec_buf, &dec_size, &dec_type); - } - - /* Check decoder status */ - if (ret == -1) { - /* Current decoder failed, should we try the next one ? */ - if (rule->action == FLB_PARSER_ACT_TRY_NEXT || - rule->action == FLB_PARSER_ACT_DO_NEXT) { - continue; - } - - /* Stop: no more rules should be applied */ - break; - } - - /* Internal packing: replace value content in the same key */ - if (rule->type == FLB_PARSER_DEC_AS) { - tmp_sds = flb_sds_copy(in_sds, dec_buf, dec_size); - if (tmp_sds != in_sds) { - in_sds = tmp_sds; - } - tmp_sds = flb_sds_copy(data_sds, dec_buf, dec_size); - if (tmp_sds != data_sds) { - data_sds = tmp_sds; - } - in_type = dec_type; - is_decoded_as = FLB_TRUE; - } - else if (rule->type == FLB_PARSER_DEC_DEFAULT) { - tmp_sds = flb_sds_copy(out_sds, dec_buf, dec_size); - if (tmp_sds != out_sds) { - out_sds = tmp_sds; - } - out_type = dec_type; - is_decoded = FLB_TRUE; - } - - - if (dec_buf != dec->buffer) { - flb_free(dec_buf); - } - dec_buf = NULL; - dec_size = 0; - - /* Apply more rules ? */ - if (rule->action == FLB_PARSER_ACT_DO_NEXT) { - continue; - } - break; - } - - /* Package the key */ - msgpack_pack_object(&mp_pck, k); - - /* We need to place some value for the key in question */ - if (is_decoded_as == FLB_TRUE) { - if (in_type == TYPE_OUT_STRING) { - msgpack_pack_str(&mp_pck, flb_sds_len(in_sds)); - msgpack_pack_str_body(&mp_pck, - in_sds, flb_sds_len(in_sds)); - } - else if (in_type == TYPE_OUT_OBJECT) { - msgpack_sbuffer_write(&mp_sbuf, - in_sds, flb_sds_len(in_sds)); - } - } - else { - /* Pack original value */ - msgpack_pack_object(&mp_pck, v); - } - - /* Package as external keys */ - if (is_decoded == FLB_TRUE) { - if (out_type == TYPE_OUT_STRING) { - flb_error("[parser_decoder] string type is not allowed"); - } - else if (out_type == TYPE_OUT_OBJECT) { - msgpack_sbuffer_write(&extra_mp_sbuf, - out_sds, flb_sds_len(out_sds)); - } - } - } - - if (in_sds) { - flb_sds_destroy(in_sds); - } - if (out_sds) { - flb_sds_destroy(out_sds); - } - if (data_sds) { - flb_sds_destroy(data_sds); - } - - msgpack_unpacked_destroy(&result); - *out_buf = mp_sbuf.data; - *out_size = mp_sbuf.size; - - if (extra_keys == FLB_TRUE) { - ret = merge_record_and_extra_keys(mp_sbuf.data, mp_sbuf.size, - extra_mp_sbuf.data, extra_mp_sbuf.size, - out_buf, out_size); - msgpack_sbuffer_destroy(&extra_mp_sbuf); - if (ret == 0) { - msgpack_sbuffer_destroy(&mp_sbuf); - return 0; - } - } - - return 0; -} - -/* - * Iterate decoders list and lookup for an existing context for 'key_name', - * if it does not exists, create and link a new one - */ -static struct flb_parser_dec *get_decoder_key_context(const char *key_name, int key_len, - struct mk_list *list) -{ - struct mk_list *head; - struct flb_parser_dec *dec = NULL; - - mk_list_foreach(head, list) { - dec = mk_list_entry(head, struct flb_parser_dec, _head); - - /* Check if the decoder matches the requested key name */ - if (flb_sds_cmp(dec->key, key_name, key_len) != 0) { - dec = NULL; - continue; - } - else { - break; - } - } - - if (!dec) { - dec = flb_malloc(sizeof(struct flb_parser_dec)); - if (!dec) { - flb_errno(); - return NULL; - } - - dec->key = flb_sds_create_len(key_name, key_len); - if (!dec->key) { - flb_errno(); - flb_free(dec); - return NULL; - } - - dec->buffer = flb_sds_create_size(FLB_PARSER_DEC_BUF_SIZE); - if (!dec->buffer) { - flb_errno(); - flb_sds_destroy(dec->key); - flb_free(dec); - return NULL; - } - dec->add_extra_keys = FLB_FALSE; - mk_list_init(&dec->rules); - mk_list_add(&dec->_head, list); - } - - return dec; -} - -struct mk_list *flb_parser_decoder_list_create(struct flb_cf_section *section) -{ - int c = 0; - int type; - int backend; - int size; - struct cfl_list *head; - struct mk_list *list = NULL; - struct mk_list *split; - struct flb_split_entry *decoder; - struct flb_split_entry *field; - struct flb_split_entry *action; - struct flb_parser_dec *dec; - struct flb_parser_dec_rule *dec_rule; - struct cfl_kvpair *entry; - - /* Global list to be referenced by parent parser definition */ - list = flb_malloc(sizeof(struct mk_list)); - if (!list) { - flb_errno(); - return NULL; - } - mk_list_init(list); - - cfl_list_foreach(head, §ion->properties->list) { - entry = cfl_list_entry(head, struct cfl_kvpair, _head); - - /* Lookup for specific Decode rules */ - if (strcasecmp(entry->key, "decode_field") == 0) { - type = FLB_PARSER_DEC_DEFAULT; - } - else if (strcasecmp(entry->key, "decode_field_as") == 0) { - type = FLB_PARSER_DEC_AS; - } - else { - continue; - } - - /* Split the value */ - split = flb_utils_split(entry->val->data.as_string, ' ', 3); - if (!split) { - flb_error("[parser] invalid number of parameters in decoder"); - flb_parser_decoder_list_destroy(list); - return NULL; - } - - /* We expect at least two values: decoder name and target field */ - size = mk_list_size(split); - if (size < 2) { - flb_error("[parser] invalid number of parameters in decoder"); - flb_utils_split_free(split); - flb_parser_decoder_list_destroy(list); - return NULL; - } - - /* - * Get the rule/entry references: - * - * decoder: specify the backend that handle decoding (json, escaped..) - * field : the 'key' where decoding should happen - * action : optional rules to follow on success or failure - */ - decoder = mk_list_entry_first(split, struct flb_split_entry, _head); - field = mk_list_entry_next(&decoder->_head, struct flb_split_entry, - _head, list); - if (size >= 3) { - action = mk_list_entry_next(&field->_head, struct flb_split_entry, - _head, list); - } - else { - action = NULL; - } - - /* Get decoder */ - if (strcasecmp(decoder->value, "json") == 0) { - backend = FLB_PARSER_DEC_JSON; - } - else if (strcasecmp(decoder->value, "escaped") == 0) { - backend = FLB_PARSER_DEC_ESCAPED; - } - else if (strcasecmp(decoder->value, "escaped_utf8") == 0) { - backend = FLB_PARSER_DEC_ESCAPED_UTF8; - } - else if (strcasecmp(decoder->value, "mysql_quoted") == 0) { - backend = FLB_PARSER_DEC_MYSQL_QUOTED; - } - else { - flb_error("[parser] field decoder '%s' unknown", decoder->value); - flb_utils_split_free(split); - flb_parser_decoder_list_destroy(list); - return NULL; - } - - /* Get the parent decoder that will hold the rules defined */ - dec = get_decoder_key_context(field->value, strlen(field->value), list); - if (!dec) { - /* Unexpected error */ - flb_error("[parser] unexpected error, could not get a decoder"); - flb_utils_split_free(split); - flb_parser_decoder_list_destroy(list); - return NULL; - } - - /* Create decoder context */ - dec_rule = flb_calloc(1, sizeof(struct flb_parser_dec_rule)); - if (!dec_rule) { - flb_errno(); - flb_utils_split_free(split); - flb_parser_decoder_list_destroy(list); - return NULL; - } - - if (type == FLB_PARSER_DEC_DEFAULT) { - dec->add_extra_keys = FLB_TRUE; - } - - dec_rule->type = type; - dec_rule->backend = backend; - if (action) { - if (strcasecmp(action->value, "try_next") == 0) { - dec_rule->action = FLB_PARSER_ACT_TRY_NEXT; - } - else if (strcasecmp(action->value, "do_next") == 0) { - dec_rule->action = FLB_PARSER_ACT_DO_NEXT; - } - else { - dec_rule->action = FLB_PARSER_ACT_NONE; - } - } - - /* Remove temporary split */ - flb_utils_split_free(split); - mk_list_add(&dec_rule->_head, &dec->rules); - c++; - } - - if (c == 0) { - flb_free(list); - return NULL; - } - - return list; -} - -int flb_parser_decoder_list_destroy(struct mk_list *list) -{ - int c = 0; - struct mk_list *head; - struct mk_list *r_head; - struct mk_list *tmp; - struct mk_list *r_tmp; - struct flb_parser_dec *dec; - struct flb_parser_dec_rule *dec_rule; - - mk_list_foreach_safe(head, tmp, list) { - dec = mk_list_entry(head, struct flb_parser_dec, _head); - - /* Destroy rules */ - mk_list_foreach_safe(r_head, r_tmp, &dec->rules) { - dec_rule = mk_list_entry(r_head, struct flb_parser_dec_rule, - _head); - mk_list_del(&dec_rule->_head); - flb_free(dec_rule); - } - - mk_list_del(&dec->_head); - flb_sds_destroy(dec->key); - flb_sds_destroy(dec->buffer); - flb_free(dec); - c++; - } - - flb_free(list); - return c; -} diff --git a/fluent-bit/src/flb_parser_json.c b/fluent-bit/src/flb_parser_json.c deleted file mode 100644 index 7add06e94..000000000 --- a/fluent-bit/src/flb_parser_json.c +++ /dev/null @@ -1,246 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#define _GNU_SOURCE -#include <time.h> - -#include <fluent-bit/flb_parser.h> -#include <fluent-bit/flb_pack.h> -#include <fluent-bit/flb_mem.h> -#include <fluent-bit/flb_parser_decoder.h> - -int flb_parser_json_do(struct flb_parser *parser, - const char *in_buf, size_t in_size, - void **out_buf, size_t *out_size, - struct flb_time *out_time) -{ - int i; - int skip; - int ret; - int slen; - int root_type; - int records; - double tmfrac = 0; - char *mp_buf = NULL; - char *time_key; - char *tmp_out_buf = NULL; - char tmp[255]; - size_t tmp_out_size = 0; - size_t off = 0; - size_t map_size; - size_t mp_size; - size_t len; - msgpack_sbuffer mp_sbuf; - msgpack_packer mp_pck; - msgpack_unpacked result; - msgpack_object map; - msgpack_object *k = NULL; - msgpack_object *v = NULL; - time_t time_lookup; - struct flb_tm tm = {0}; - struct flb_time *t; - size_t consumed; - - consumed = 0; - - /* Convert incoming in_buf JSON message to message pack format */ - ret = flb_pack_json_recs(in_buf, in_size, &mp_buf, &mp_size, &root_type, - &records, &consumed); - if (ret != 0) { - return -1; - } - - if (records != 1) { - flb_free(mp_buf); - - return -1; - } - - /* Make sure object is a map */ - msgpack_unpacked_init(&result); - if (msgpack_unpack_next(&result, mp_buf, mp_size, &off) == MSGPACK_UNPACK_SUCCESS) { - map = result.data; - if (map.type != MSGPACK_OBJECT_MAP) { - flb_free(mp_buf); - msgpack_unpacked_destroy(&result); - - return -1; - } - } - else { - if (mp_size > 0) { - flb_free(mp_buf); - } - - msgpack_unpacked_destroy(&result); - - return -1; - } - - /* Export results (might change later) */ - tmp_out_buf = mp_buf; - tmp_out_size = mp_size; - - /* Do we have some decoders set ? */ - if (parser->decoders) { - ret = flb_parser_decoder_do(parser->decoders, - mp_buf, mp_size, - &tmp_out_buf, &tmp_out_size); - if (ret == 0) { - /* re-process the unpack context */ - off = 0; - msgpack_unpacked_destroy(&result); - msgpack_unpacked_init(&result); - msgpack_unpack_next(&result, tmp_out_buf, tmp_out_size, &off); - map = result.data; - } - } - - /* Set the possible outgoing buffer */ - *out_buf = tmp_out_buf; - *out_size = tmp_out_size; - if (mp_buf != tmp_out_buf) { - flb_free(mp_buf); - mp_buf = NULL; - } - - /* Do time resolution ? */ - if (!parser->time_fmt) { - msgpack_unpacked_destroy(&result); - - return (int) consumed; - } - - if (parser->time_key) { - time_key = parser->time_key; - } - else { - time_key = "time"; - } - slen = strlen(time_key); - - /* Lookup time field */ - map_size = map.via.map.size; - skip = map_size; - for (i = 0; i < map_size; i++) { - k = &map.via.map.ptr[i].key; - v = &map.via.map.ptr[i].val; - - if (k->via.str.size != slen) { - continue; - } - - /* Ensure the pointer we are about to read is not NULL */ - if (k->via.str.ptr == NULL) { - if (mp_buf == tmp_out_buf) { - flb_free(mp_buf); - } - else { - flb_free(mp_buf); - flb_free(tmp_out_buf); - } - *out_buf = NULL; - msgpack_unpacked_destroy(&result); - - return -1; - } - - if (strncmp(k->via.str.ptr, time_key, k->via.str.size) == 0) { - /* We found the key, break the loop and keep the index */ - if (parser->time_keep == FLB_FALSE) { - skip = i; - break; - } - else { - skip = -1; - } - break; - } - - k = NULL; - v = NULL; - } - - /* No time_key field found */ - if (i >= map_size || !k || !v) { - msgpack_unpacked_destroy(&result); - - return (int) consumed; - } - - /* Ensure we have an accurate type */ - if (v->type != MSGPACK_OBJECT_STR) { - msgpack_unpacked_destroy(&result); - - return (int) consumed; - } - - /* Lookup time */ - ret = flb_parser_time_lookup(v->via.str.ptr, v->via.str.size, - 0, parser, &tm, &tmfrac); - if (ret == -1) { - len = v->via.str.size; - if (len > sizeof(tmp) - 1) { - len = sizeof(tmp) - 1; - } - memcpy(tmp, v->via.str.ptr, len); - tmp[len] = '\0'; - flb_warn("[parser:%s] invalid time format %s for '%s'", - parser->name, parser->time_fmt_full, tmp); - time_lookup = 0; - skip = map_size; - } - else { - time_lookup = flb_parser_tm2time(&tm); - } - - /* Compose a new map without the time_key field */ - msgpack_sbuffer_init(&mp_sbuf); - msgpack_packer_init(&mp_pck, &mp_sbuf, msgpack_sbuffer_write); - - if (parser->time_keep == FLB_FALSE && skip < map_size) { - msgpack_pack_map(&mp_pck, map_size - 1); - } - else { - msgpack_pack_map(&mp_pck, map_size); - } - - for (i = 0; i < map_size; i++) { - if (i == skip) { - continue; - } - - msgpack_pack_object(&mp_pck, map.via.map.ptr[i].key); - msgpack_pack_object(&mp_pck, map.via.map.ptr[i].val); - } - - /* Export the proper buffer */ - flb_free(tmp_out_buf); - - *out_buf = mp_sbuf.data; - *out_size = mp_sbuf.size; - - t = out_time; - t->tm.tv_sec = time_lookup; - t->tm.tv_nsec = (tmfrac * 1000000000); - - msgpack_unpacked_destroy(&result); - - return (int) consumed; -} diff --git a/fluent-bit/src/flb_parser_logfmt.c b/fluent-bit/src/flb_parser_logfmt.c deleted file mode 100644 index 8e6b46590..000000000 --- a/fluent-bit/src/flb_parser_logfmt.c +++ /dev/null @@ -1,326 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#define _GNU_SOURCE -#include <time.h> - -#include <fluent-bit/flb_parser.h> -#include <fluent-bit/flb_pack.h> -#include <fluent-bit/flb_mem.h> -#include <fluent-bit/flb_parser_decoder.h> -#include <fluent-bit/flb_unescape.h> -#include <fluent-bit/flb_mem.h> - -/* - * https://brandur.org/logfmt - * https://godoc.org/github.com/kr/logfmt - * - * ident_byte = any byte greater than ' ', excluding '=' and '"' - * string_byte = any byte excluding '"' and '\' - * garbage = !ident_byte - * ident = ident_byte, { ident byte } - * key = ident - * value = ident | '"', { string_byte | '\', '"' }, '"' - * pair = key, '=', value | key, '=' | key - * message = { garbage, pair }, garbage - */ - -static char ident_byte[256] = { - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 -}; - -static int logfmt_parser(struct flb_parser *parser, - const char *in_buf, size_t in_size, - msgpack_packer *tmp_pck, - char *time_key, size_t time_key_len, - time_t *time_lookup, double *tmfrac, - size_t *map_size) -{ - int ret; - struct flb_tm tm = {0}; - const unsigned char *key = NULL; - size_t key_len = 0; - const unsigned char *value = NULL; - size_t value_len = 0; - const unsigned char *c = (const unsigned char *)in_buf; - const unsigned char *end = c + in_size; - int last_byte; - int do_pack = FLB_TRUE; - int value_set = FLB_FALSE; - int value_str = FLB_FALSE; - int value_escape = FLB_FALSE; - - /* if map_size is 0 only count the number of k:v */ - if (*map_size == 0) { - do_pack = FLB_FALSE; - } - - while (c < end) { - /* garbage */ - while ((c < end) && !ident_byte[*c]) { - c++; - } - if (c == end) { - break; - } - /* key */ - key = c; - while ((c < end) && ident_byte[*c]) { - c++; - } - - key_len = c - key; - /* value */ - value_len = 0; - value_set = FLB_FALSE; - value_str = FLB_FALSE; - value_escape = FLB_FALSE; - - if (c < end && *c == '=') { - value_set = FLB_TRUE; - c++; - if (c < end) { - if (*c == '"') { - c++; - value = c; - value_str = FLB_TRUE; - while (c < end) { - if (*c != '\\' && *c!= '"') { - c++; - } - else if (*c == '\\') { - value_escape = FLB_TRUE; - c++; - if (c == end) { - break; - } - c++; - } - else { - break; - } - } - value_len = c - value; - if (c < end && *c == '\"') { - c++; - } - } - else { - value = c; - while ((c < end) && ident_byte[*c]) { - c++; - } - value_len = c - value; - } - } - } - - if (key_len > 0) { - int time_found = FLB_FALSE; - if (parser->logfmt_no_bare_keys && value_len == 0 && !value_set) { - if (!do_pack) { - *map_size = 0; - } - return 0; - } - - if (parser->time_fmt && key_len == time_key_len && - value_len > 0 && - !strncmp((const char *)key, time_key, key_len)) { - if (do_pack) { - ret = flb_parser_time_lookup((const char *) value, value_len, - 0, parser, &tm, tmfrac); - if (ret == -1) { - flb_error("[parser:%s] Invalid time format %s", - parser->name, parser->time_fmt_full); - return -1; - } - *time_lookup = flb_parser_tm2time(&tm); - } - time_found = FLB_TRUE; - } - - if (time_found == FLB_FALSE || parser->time_keep == FLB_TRUE) { - if (do_pack) { - if (parser->types_len != 0) { - flb_parser_typecast((const char*) key, key_len, - (const char*) value, value_len, - tmp_pck, - parser->types, - parser->types_len); - } - else { - msgpack_pack_str(tmp_pck, key_len); - msgpack_pack_str_body(tmp_pck, (const char *)key, key_len); - if (value_len == 0) { - if (value_str == FLB_TRUE) { - msgpack_pack_str(tmp_pck, 0); - } - else { - msgpack_pack_true(tmp_pck); - } - } - else { - if (value_escape == FLB_TRUE) { - int out_len; - char *out_str; - - out_str = flb_malloc(value_len + 1); - if (out_str == NULL) { - flb_errno(); - return -1; - } - out_str[0] = 0; - flb_unescape_string_utf8((const char *)value, - value_len, - out_str); - out_len = strlen(out_str); - - msgpack_pack_str(tmp_pck, out_len); - msgpack_pack_str_body(tmp_pck, - out_str, - out_len); - - flb_free(out_str); - } - else { - msgpack_pack_str(tmp_pck, value_len); - msgpack_pack_str_body(tmp_pck, - (const char *)value, - value_len); - } - } - } - } - else { - (*map_size)++; - } - } - } - - if (c == end) { - break; - } - - if (*c == '\r') { - c++; - if (c == end) { - break; - } - if (*c == '\n') { - c++; - } - break; - } - if (*c == '\n') { - c++; - break; - } - } - last_byte = (const char *)c - in_buf; - - return last_byte; -} - -int flb_parser_logfmt_do(struct flb_parser *parser, - const char *in_buf, size_t in_size, - void **out_buf, size_t *out_size, - struct flb_time *out_time) -{ - int ret; - time_t time_lookup; - double tmfrac = 0; - struct flb_time *t; - msgpack_sbuffer tmp_sbuf; - msgpack_packer tmp_pck; - char *dec_out_buf; - size_t dec_out_size; - size_t map_size; - char *time_key; - size_t time_key_len; - int last_byte; - - if (parser->time_key) { - time_key = parser->time_key; - } - else { - time_key = "time"; - } - time_key_len = strlen(time_key); - time_lookup = 0; - - /* count the number of key value pairs */ - map_size = 0; - logfmt_parser(parser, in_buf, in_size, NULL, - time_key, time_key_len, - &time_lookup, &tmfrac, &map_size); - if (map_size == 0) { - return -1; - } - - /* Prepare new outgoing buffer */ - msgpack_sbuffer_init(&tmp_sbuf); - msgpack_packer_init(&tmp_pck, &tmp_sbuf, msgpack_sbuffer_write); - msgpack_pack_map(&tmp_pck, map_size); - - last_byte = logfmt_parser(parser, in_buf, in_size, &tmp_pck, - time_key, time_key_len, - &time_lookup, &tmfrac, &map_size); - if (last_byte < 0) { - msgpack_sbuffer_destroy(&tmp_sbuf); - return last_byte; - } - - /* Export results */ - *out_buf = tmp_sbuf.data; - *out_size = tmp_sbuf.size; - - t = out_time; - t->tm.tv_sec = time_lookup; - t->tm.tv_nsec = (tmfrac * 1000000000); - - /* Check if some decoder was specified */ - if (parser->decoders) { - ret = flb_parser_decoder_do(parser->decoders, - tmp_sbuf.data, tmp_sbuf.size, - &dec_out_buf, &dec_out_size); - if (ret == 0) { - *out_buf = dec_out_buf; - *out_size = dec_out_size; - msgpack_sbuffer_destroy(&tmp_sbuf); - } - } - - return last_byte; -} diff --git a/fluent-bit/src/flb_parser_ltsv.c b/fluent-bit/src/flb_parser_ltsv.c deleted file mode 100644 index 8f38102cf..000000000 --- a/fluent-bit/src/flb_parser_ltsv.c +++ /dev/null @@ -1,269 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#define _GNU_SOURCE -#include <time.h> - -#include <fluent-bit/flb_parser.h> -#include <fluent-bit/flb_pack.h> -#include <fluent-bit/flb_mem.h> -#include <fluent-bit/flb_parser_decoder.h> - -/* - * http://ltsv.org - * - * ltsv = *(record NL) [record] - * record = [field *(TAB field)] - * field = label ":" field-value - * label = 1*lbyte - * field-value = *fbyte - * - * TAB = %x09 - * NL = [%x0D] %x0A - * lbyte = %x30-39 / %x41-5A / %x61-7A / "_" / "." / "-" ;; [0-9A-Za-z_.-] - * fbyte = %x01-08 / %x0B / %x0C / %x0E-FF - */ - -static char ltvs_label[256] = { - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, - 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, - 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -}; - -static char ltvs_field[256] = { - 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 1, 0, 0, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, -}; - - -static int ltsv_parser(struct flb_parser *parser, - const char *in_buf, size_t in_size, - msgpack_packer *tmp_pck, - char *time_key, size_t time_key_len, - time_t *time_lookup, double *tmfrac, - size_t *map_size) -{ - int ret; - struct flb_tm tm = {0}; - const unsigned char *label = NULL; - size_t label_len = 0; - const unsigned char *field = NULL; - size_t field_len = 0; - const unsigned char *c = (const unsigned char *)in_buf; - const unsigned char *end = c + in_size; - int last_byte; - int do_pack = FLB_TRUE; - - /* if map_size is 0 only count the number of k:v */ - if (*map_size == 0) { - do_pack = FLB_FALSE; - } - - while (c < end) { - label = c; - while ((c < end) && ltvs_label[*c]) { - c++; - } - label_len = c - label; - if (c == end) { - break; - } - - if (*c != ':') { - break; - } - c++; - - field = c; - if (c != end) { - while ((c < end) && ltvs_field[*c]) { - c++; - } - } - field_len = c - field; - - if (label_len > 0) { - int time_found = FLB_FALSE; - - if (parser->time_fmt && label_len == time_key_len && - field_len > 0 && - !strncmp((const char *)label, time_key, label_len)) { - if (do_pack) { - ret = flb_parser_time_lookup((const char *) field, field_len, - 0, parser, &tm, tmfrac); - if (ret == -1) { - flb_error("[parser:%s] Invalid time format %s", - parser->name, parser->time_fmt_full); - return -1; - } - *time_lookup = flb_parser_tm2time(&tm); - } - time_found = FLB_TRUE; - } - - if (time_found == FLB_FALSE || parser->time_keep == FLB_TRUE) { - if (do_pack) { - if (parser->types_len != 0) { - flb_parser_typecast((const char*) label, label_len, - (const char*) field, field_len, - tmp_pck, - parser->types, - parser->types_len); - } - else { - msgpack_pack_str(tmp_pck, label_len); - msgpack_pack_str_body(tmp_pck, (const char *)label, label_len); - msgpack_pack_str(tmp_pck, field_len); - msgpack_pack_str_body(tmp_pck, (const char *)field, field_len); - } - } - else { - (*map_size)++; - } - } - } - - if (c == end) { - break; - } - if (*c == '\t') { - c++; - } - if (c == end) { - break; - } - - if (*c == '\r') { - c++; - if (c == end) { - break; - } - if (*c == '\n') { - c++; - } - break; - } - if (*c == '\n') { - c++; - break; - } - } - last_byte = (const char *)c - in_buf; - - return last_byte; -} - -int flb_parser_ltsv_do(struct flb_parser *parser, - const char *in_buf, size_t in_size, - void **out_buf, size_t *out_size, - struct flb_time *out_time) -{ - int ret; - time_t time_lookup; - double tmfrac = 0; - struct flb_time *t; - msgpack_sbuffer tmp_sbuf; - msgpack_packer tmp_pck; - char *dec_out_buf; - size_t dec_out_size; - size_t map_size; - char *time_key; - size_t time_key_len; - int last_byte; - - if (parser->time_key) { - time_key = parser->time_key; - } - else { - time_key = "time"; - } - time_key_len = strlen(time_key); - time_lookup = 0; - - /* count the number of key value pairs */ - map_size = 0; - ltsv_parser(parser, in_buf, in_size, NULL, - time_key, time_key_len, - &time_lookup, &tmfrac, &map_size); - if (map_size == 0) { - return -1; - } - - /* Prepare new outgoing buffer */ - msgpack_sbuffer_init(&tmp_sbuf); - msgpack_packer_init(&tmp_pck, &tmp_sbuf, msgpack_sbuffer_write); - msgpack_pack_map(&tmp_pck, map_size); - - last_byte = ltsv_parser(parser, in_buf, in_size, &tmp_pck, - time_key, time_key_len, - &time_lookup, &tmfrac, &map_size); - if (last_byte < 0) { - msgpack_sbuffer_destroy(&tmp_sbuf); - return last_byte; - } - - /* Export results */ - *out_buf = tmp_sbuf.data; - *out_size = tmp_sbuf.size; - - t = out_time; - t->tm.tv_sec = time_lookup; - t->tm.tv_nsec = (tmfrac * 1000000000); - - /* Check if some decoder was specified */ - if (parser->decoders) { - ret = flb_parser_decoder_do(parser->decoders, - tmp_sbuf.data, tmp_sbuf.size, - &dec_out_buf, &dec_out_size); - if (ret == 0) { - *out_buf = dec_out_buf; - *out_size = dec_out_size; - msgpack_sbuffer_destroy(&tmp_sbuf); - } - } - - return last_byte; -} diff --git a/fluent-bit/src/flb_parser_regex.c b/fluent-bit/src/flb_parser_regex.c deleted file mode 100644 index efcc6fb60..000000000 --- a/fluent-bit/src/flb_parser_regex.c +++ /dev/null @@ -1,227 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#define _GNU_SOURCE -#include <time.h> - -#include <fluent-bit/flb_info.h> -#include <fluent-bit/flb_parser.h> -#include <fluent-bit/flb_parser_decoder.h> -#include <fluent-bit/flb_regex.h> -#include <fluent-bit/flb_str.h> - -#include <msgpack.h> - -/* don't do this at home */ -#define pack_uint16(buf, d) _msgpack_store16(buf, (uint16_t) d) -#define pack_uint32(buf, d) _msgpack_store32(buf, (uint32_t) d) - -struct regex_cb_ctx { - int num_skipped; - time_t time_lookup; - time_t time_now; - double time_frac; - struct flb_parser *parser; - msgpack_packer *pck; -}; - -static void cb_results(const char *name, const char *value, - size_t vlen, void *data) -{ - int len; - int ret; - double frac = 0; - char *time_key; - char tmp[255]; - struct regex_cb_ctx *pcb = data; - struct flb_parser *parser = pcb->parser; - struct flb_tm tm = {0}; - (void) data; - - if (vlen == 0 && parser->skip_empty) { - pcb->num_skipped++; - return; - } - - len = strlen(name); - - /* Check if there is a time lookup field */ - if (parser->time_fmt) { - if (parser->time_key) { - time_key = parser->time_key; - } - else { - time_key = "time"; - } - - if (strcmp(name, time_key) == 0) { - /* Lookup time */ - ret = flb_parser_time_lookup(value, vlen, - pcb->time_now, parser, &tm, &frac); - if (ret == -1) { - if (vlen > sizeof(tmp) - 1) { - vlen = sizeof(tmp) - 1; - } - memcpy(tmp, value, vlen); - tmp[vlen] = '\0'; - flb_warn("[parser:%s] invalid time format %s for '%s'", - parser->name, parser->time_fmt_full, tmp); - pcb->num_skipped++; - return; - } - - pcb->time_frac = frac; - pcb->time_lookup = flb_parser_tm2time(&tm); - - if (parser->time_keep == FLB_FALSE) { - pcb->num_skipped++; - return; - } - } - } - - if (parser->types_len != 0) { - flb_parser_typecast(name, len, - value, vlen, - pcb->pck, - parser->types, - parser->types_len); - } - else { - msgpack_pack_str(pcb->pck, len); - msgpack_pack_str_body(pcb->pck, name, len); - msgpack_pack_str(pcb->pck, vlen); - msgpack_pack_str_body(pcb->pck, value, vlen); - } -} - -int flb_parser_regex_do(struct flb_parser *parser, - const char *buf, size_t length, - void **out_buf, size_t *out_size, - struct flb_time *out_time) -{ - int ret; - int arr_size; - int last_byte; - ssize_t n; - size_t dec_out_size; - char *dec_out_buf; - char *tmp; - struct flb_regex_search result; - struct regex_cb_ctx pcb; - struct flb_time *t; - msgpack_sbuffer tmp_sbuf; - msgpack_packer tmp_pck; - - n = flb_regex_do(parser->regex, buf, length, &result); - if (n <= 0) { - return -1; - } - - /* Prepare new outgoing buffer */ - msgpack_sbuffer_init(&tmp_sbuf); - msgpack_packer_init(&tmp_pck, &tmp_sbuf, msgpack_sbuffer_write); - - /* Set a Map size with the exact number of matches returned by regex */ - arr_size = n; - msgpack_pack_map(&tmp_pck, arr_size); - - /* Callback context */ - pcb.pck = &tmp_pck; - pcb.parser = parser; - pcb.num_skipped = 0; - pcb.time_lookup = 0; - pcb.time_frac = 0; - pcb.time_now = 0; - - /* Iterate results and compose new buffer */ - last_byte = flb_regex_parse(parser->regex, &result, cb_results, &pcb); - if (last_byte == -1) { - msgpack_sbuffer_destroy(&tmp_sbuf); - return -1; - } - - /* - * There some special cases when the Parser have a 'time' handling - * requirement, meaning: lookup for this 'time' key and resolve the - * real date of the record. If so, the parser by default will - * keep the original 'time' key field found but in other scenarios - * it may ask to skip it. - * - * If a time lookup is specified and the parser ask to skip the record - * and the time key is found, we need to adjust the msgpack header - * map size, initially we set a size to include all keys found, but - * until now we just know we are not going to include it. - * - * In addition, keys without associated values are skipped too and we - * must take this into account in msgpack header map size adjustment. - * - * In order to avoid to create a new msgpack buffer and repack the - * map entries, we just position at the header byte and do the - * proper adjustment in our original buffer. Note that for cases - * where the map is large enough '<= 65535' or '> 65535' we have - * to use internal msgpack api functions since packing the bytes - * in Big-Endian is a requirement. - */ - if (pcb.num_skipped > 0) { - - arr_size = (n - pcb.num_skipped); - - tmp = tmp_sbuf.data; - uint8_t h = tmp[0]; - if (h >> 4 == 0x8) { /* 1000xxxx */ - *tmp = (uint8_t) 0x8 << 4 | ((uint8_t) arr_size); - } - else if (h == 0xde) { - tmp++; - pack_uint16(tmp, arr_size); - } - else if (h == 0xdf) { - tmp++; - pack_uint32(tmp, arr_size); - } - } - - /* Export results */ - *out_buf = tmp_sbuf.data; - *out_size = tmp_sbuf.size; - - t = out_time; - t->tm.tv_sec = pcb.time_lookup; - t->tm.tv_nsec = (pcb.time_frac * 1000000000); - - /* Check if some decoder was specified */ - if (parser->decoders) { - ret = flb_parser_decoder_do(parser->decoders, - tmp_sbuf.data, tmp_sbuf.size, - &dec_out_buf, &dec_out_size); - if (ret == 0) { - *out_buf = dec_out_buf; - *out_size = dec_out_size; - msgpack_sbuffer_destroy(&tmp_sbuf); - } - } - - /* - * The return the value >= 0, belongs to the LAST BYTE consumed by the - * regex engine. If the last byte is lower than string length, means - * there is more data to be processed (maybe it's a stream). - */ - return last_byte; -} diff --git a/fluent-bit/src/flb_pipe.c b/fluent-bit/src/flb_pipe.c deleted file mode 100644 index 57ed07834..000000000 --- a/fluent-bit/src/flb_pipe.c +++ /dev/null @@ -1,183 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * Fluent Bit core uses unnamed Unix pipes for signaling and general - * communication across components. When building on Windows this is - * problematic because Windows pipes are not selectable and only - * sockets are. - * - * This file aims to wrap around the required backend calls depending - * of the operating system. - * - * This file provides 4 interfaces: - * - * - flb_pipe_create : create a pair of connected file descriptors or sockets. - * - flb_pipe_destroy : destroy a pair of connected fds or sockets. - * - flb_pipe_close : close individual end of a pipe. - * - flb_pipe_set_nonblocking : make a socket nonblocking - * - * we need to have a 'closer' handler because for Windows a file descriptor - * is not a socket. - */ - -#include <fluent-bit/flb_compat.h> -#include <fluent-bit/flb_pipe.h> -#include <fluent-bit/flb_log.h> -#include <fluent-bit/flb_time.h> - -#ifdef _WIN32 - -/* - * Building on Windows means that Monkey library (lib/monkey) and it - * core runtime have been build with 'libevent' backend support, that - * library provide an abstraction to create a socketpairs. - * - * Creating a pipe on Fluent Bit @Windows, means create a socket pair. - */ - -int flb_pipe_create(flb_pipefd_t pipefd[2]) -{ - if (evutil_socketpair(AF_UNIX, SOCK_STREAM, 0, pipefd) == -1) { - perror("socketpair"); - return -1; - } - - return 0; -} - -void flb_pipe_destroy(flb_pipefd_t pipefd[2]) -{ - evutil_closesocket(pipefd[0]); - evutil_closesocket(pipefd[1]); -} - -int flb_pipe_close(flb_pipefd_t fd) -{ - return evutil_closesocket(fd); -} - -int flb_pipe_set_nonblocking(flb_pipefd_t fd) -{ - return evutil_make_socket_nonblocking(fd); -} -#else -/* All other flavors of Unix/BSD are OK */ - -#include <stdint.h> -#include <fcntl.h> - -int flb_pipe_create(flb_pipefd_t pipefd[2]) -{ - return pipe(pipefd); -} - -void flb_pipe_destroy(flb_pipefd_t pipefd[2]) -{ - close(pipefd[0]); - close(pipefd[1]); -} - -int flb_pipe_close(flb_pipefd_t fd) -{ - /* - * when chunk file is destroyed, the fd for file will be -1, we should avoid - * deleting chunk file with fd -1 - */ - if (fd == -1) { - return -1; - } - - return close(fd); -} - -int flb_pipe_set_nonblocking(flb_pipefd_t fd) -{ - int flags = fcntl(fd, F_GETFL); - if (flags < 0) - return -1; - if (flags & O_NONBLOCK) - return 0; - return fcntl(fd, F_SETFL, flags | O_NONBLOCK); -} -#endif - -/* Blocking read until receive 'count' bytes */ -ssize_t flb_pipe_read_all(int fd, void *buf, size_t count) -{ - ssize_t bytes; - size_t total = 0; - - do { - bytes = flb_pipe_r(fd, (char *) buf + total, count - total); - if (bytes == -1) { - if (FLB_PIPE_WOULDBLOCK()) { - /* - * This could happen, since this function goal is not to - * return until all data have been read, just sleep a little - * bit (0.05 seconds) - */ - flb_time_msleep(50); - continue; - } - return -1; - } - else if (bytes == 0) { - /* Broken pipe ? */ - flb_errno(); - return -1; - } - total += bytes; - - } while (total < count); - - return total; -} - -/* Blocking write until send 'count bytes */ -ssize_t flb_pipe_write_all(int fd, const void *buf, size_t count) -{ - ssize_t bytes; - size_t total = 0; - - do { - bytes = flb_pipe_w(fd, (const char *) buf + total, count - total); - if (bytes == -1) { - if (FLB_PIPE_WOULDBLOCK()) { - /* - * This could happen, since this function goal is not to - * return until all data have been read, just sleep a little - * bit (0.05 seconds) - */ - flb_time_msleep(50); - continue; - } - return -1; - } - else if (bytes == 0) { - /* Broken pipe ? */ - flb_errno(); - return -1; - } - total += bytes; - - } while (total < count); - - return total; -} diff --git a/fluent-bit/src/flb_plugin.c b/fluent-bit/src/flb_plugin.c deleted file mode 100644 index cd497f179..000000000 --- a/fluent-bit/src/flb_plugin.c +++ /dev/null @@ -1,452 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <fluent-bit/flb_compat.h> -#include <fluent-bit/flb_info.h> -#include <fluent-bit/flb_mem.h> -#include <fluent-bit/flb_log.h> -#include <fluent-bit/flb_error.h> -#include <fluent-bit/flb_kv.h> -#include <fluent-bit/flb_config_format.h> -#include <fluent-bit/flb_utils.h> -#include <fluent-bit/flb_plugin.h> -#include <fluent-bit/flb_plugin_proxy.h> - -#include <cfl/cfl_sds.h> -#include <cfl/cfl_variant.h> -#include <cfl/cfl_kvlist.h> - -#include <sys/types.h> -#include <sys/stat.h> - -#define PLUGIN_PREFIX "flb-" -#define PLUGIN_EXTENSION ".so" -#define PLUGIN_STRUCT_SUFFIX "_plugin" -#define PLUGIN_STR_MIN \ - ((sizeof(PLUGIN_PREFIX) - 1) + sizeof(PLUGIN_EXTENSION) - 1) - -static int is_input(char *name) -{ - if (strncmp(name, "in_", 3) == 0) { - return FLB_TRUE; - } - - return FLB_FALSE; -} - -static int is_filter(char *name) -{ - if (strncmp(name, "filter_", 7) == 0) { - return FLB_TRUE; - } - - return FLB_FALSE; -} - -static int is_processor(char *name) -{ - if (strncmp(name, "processor_", 10) == 0) { - return FLB_TRUE; - } - - return FLB_FALSE; -} - -static int is_output(char *name) -{ - if (strncmp(name, "out_", 4) == 0) { - return FLB_TRUE; - } - - return FLB_FALSE; -} - -static void *get_handle(const char *path) -{ - void *handle; - - handle = dlopen(path, RTLD_LAZY); - if (!handle) { - flb_error("[plugin] dlopen() %s", dlerror()); - return NULL; - } - - return handle; -} - -static void *load_symbol(void *dso_handle, const char *symbol) -{ - void *s; - - dlerror(); - s = dlsym(dso_handle, symbol); - if (dlerror() != NULL) { - return NULL; - } - return s; -} - -/* - * From a given path file (.so file), retrieve the expected structure name - * used to perform the plugin registration. - */ -static char *path_to_plugin_name(char *path) -{ - int len; - int o_len; - char *bname; - char *name; - char *p; - - /* Get the basename of the file */ - bname = basename(path); - if (!bname) { - flb_error("[plugin] could not resolve basename(3) of the plugin"); - return NULL; - } - len = strlen(bname); - - if (len < PLUGIN_STR_MIN) { - flb_error("[plugin] invalid plugin name: %s", bname); - return NULL; - } - - if (strncmp(bname, PLUGIN_PREFIX, sizeof(PLUGIN_PREFIX) - 1) != 0) { - flb_error("[plugin] invalid plugin prefix: %s", bname); - return NULL; - } - - if (strncmp(bname + len - (sizeof(PLUGIN_EXTENSION) - 1), - PLUGIN_EXTENSION, sizeof(PLUGIN_EXTENSION) - 1) != 0) { - flb_error("[plugin] invalid plugin extension: %s", bname); - return NULL; - } - - /* Get the expected structure name */ - name = flb_malloc(len + (sizeof(PLUGIN_STRUCT_SUFFIX) - 1) + 1); - if (!name) { - flb_errno(); - return NULL; - } - - /* Name without prefix */ - p = bname + (sizeof(PLUGIN_PREFIX) - 1); - o_len = len - (sizeof(PLUGIN_PREFIX) - 1) - (sizeof(PLUGIN_EXTENSION) - 1); - memcpy(name, p, o_len); - name[o_len] = '\0'; - - /* Validate expected plugin type */ - if (is_input(name) == FLB_FALSE && - is_processor(name) == FLB_FALSE && - is_filter(name) == FLB_FALSE && - is_output(name) == FLB_FALSE) { - flb_error("[plugin] invalid plugin type: %s", name); - flb_free(name); - return NULL; - } - - /* Append struct suffix */ - p = name + o_len; - memcpy(p, PLUGIN_STRUCT_SUFFIX, sizeof(PLUGIN_STRUCT_SUFFIX) - 1); - o_len += sizeof(PLUGIN_STRUCT_SUFFIX) - 1; - name[o_len] = '\0'; - - return name; -} - -static void destroy_plugin(struct flb_plugin *plugin) -{ - flb_sds_destroy(plugin->path); - dlclose(plugin->dso_handle); - mk_list_del(&plugin->_head); - flb_free(plugin); -} - -/* Creates the global plugin context for 'dynamic plugins' */ -struct flb_plugins *flb_plugin_create() -{ - struct flb_plugins *ctx; - - ctx = flb_malloc(sizeof(struct flb_plugins)); - if (!ctx) { - flb_errno(); - return NULL; - } - - mk_list_init(&ctx->input); - mk_list_init(&ctx->processor); - mk_list_init(&ctx->filter); - mk_list_init(&ctx->output); - - return ctx; -} - -int flb_plugin_load(char *path, struct flb_plugins *ctx, - struct flb_config *config) -{ - int type = -1; - void *dso_handle; - void *symbol = NULL; - char *plugin_stname; - struct flb_plugin *plugin; - struct flb_input_plugin *input; - struct flb_processor_plugin *processor; - struct flb_filter_plugin *filter; - struct flb_output_plugin *output; - - /* Open the shared object file: dlopen(3) */ - dso_handle = get_handle(path); - if (!dso_handle) { - return -1; - } - - /* - * Based on the shared object file name, compose the expected - * registration structure name. - */ - plugin_stname = path_to_plugin_name(path); - if (!plugin_stname) { - dlclose(dso_handle); - return -1; - } - - /* Get the registration structure */ - symbol = load_symbol(dso_handle, plugin_stname); - if (!symbol) { - flb_error("[plugin] cannot load plugin '%s', " - "registration structure is missing '%s'", - path, plugin_stname); - flb_free(plugin_stname); - dlclose(dso_handle); - return -1; - } - - /* Detect plugin type and link it to the main context */ - if (is_input(plugin_stname) == FLB_TRUE) { - type = FLB_PLUGIN_INPUT; - input = flb_malloc(sizeof(struct flb_input_plugin)); - if (!input) { - flb_errno(); - flb_free(plugin_stname); - dlclose(dso_handle); - return -1; - } - memcpy(input, symbol, sizeof(struct flb_input_plugin)); - mk_list_add(&input->_head, &config->in_plugins); - } - else if (is_processor(plugin_stname) == FLB_TRUE) { - type = FLB_PLUGIN_PROCESSOR; - processor = flb_malloc(sizeof(struct flb_processor_plugin)); - if (processor == NULL) { - flb_errno(); - flb_free(plugin_stname); - dlclose(dso_handle); - return -1; - } - memcpy(processor, symbol, sizeof(struct flb_processor_plugin)); - mk_list_add(&processor->_head, &config->processor_plugins); - } - else if (is_filter(plugin_stname) == FLB_TRUE) { - type = FLB_PLUGIN_FILTER; - filter = flb_malloc(sizeof(struct flb_filter_plugin)); - if (!filter) { - flb_errno(); - flb_free(plugin_stname); - dlclose(dso_handle); - return -1; - } - memcpy(filter, symbol, sizeof(struct flb_filter_plugin)); - mk_list_add(&filter->_head, &config->filter_plugins); - } - else if (is_output(plugin_stname) == FLB_TRUE) { - type = FLB_PLUGIN_OUTPUT; - output = flb_malloc(sizeof(struct flb_output_plugin)); - if (!output) { - flb_errno(); - flb_free(plugin_stname); - dlclose(dso_handle); - return -1; - } - memcpy(output, symbol, sizeof(struct flb_output_plugin)); - mk_list_add(&output->_head, &config->out_plugins); - } - flb_free(plugin_stname); - - if (type == -1) { - flb_error("[plugin] plugin type not defined on '%s'", path); - dlclose(dso_handle); - return -1; - } - - /* Create plugin context (internal reference only) */ - plugin = flb_malloc(sizeof(struct flb_plugin)); - if (!plugin) { - flb_errno(); - dlclose(dso_handle); - return -1; - } - - plugin->type = type; - plugin->path = flb_sds_create(path); - plugin->dso_handle = dso_handle; - - /* Link by type to the plugins parent context */ - if (type == FLB_PLUGIN_INPUT) { - mk_list_add(&plugin->_head, &ctx->input); - } - else if (type == FLB_PLUGIN_PROCESSOR) { - mk_list_add(&plugin->_head, &ctx->processor); - } - else if (type == FLB_PLUGIN_FILTER) { - mk_list_add(&plugin->_head, &ctx->filter); - } - else if (type == FLB_PLUGIN_OUTPUT) { - mk_list_add(&plugin->_head, &ctx->output); - } - - return 0; -} - -int flb_plugin_load_router(char *path, struct flb_config *config) -{ - int ret = -1; - char *bname; - - bname = basename(path); - - /* Is this a DSO C plugin ? */ - if (strncmp(bname, PLUGIN_PREFIX, sizeof(PLUGIN_PREFIX) - 1) == 0) { - ret = flb_plugin_load(path, config->dso_plugins, config); - if (ret == -1) { - flb_error("[plugin] error loading DSO C plugin: %s", path); - return -1; - } - } - else { -#ifdef FLB_HAVE_PROXY_GO - if (flb_plugin_proxy_create(path, 0, config) == NULL) { - flb_error("[plugin] error loading proxy plugin: %s", path); - return -1; - } -#else - flb_error("[plugin] unsupported plugin type at: %s", path); - return -1; -#endif - } - - return 0; -} - -/* Load plugins from a configuration file */ -int flb_plugin_load_config_file(const char *file, struct flb_config *config) -{ - int ret; - char tmp[PATH_MAX + 1]; - char *cfg = NULL; - struct mk_list *head; - struct cfl_list *head_e; - struct stat st; - struct flb_cf *cf; - struct flb_cf_section *section; - struct cfl_kvpair *entry; - -#ifndef FLB_HAVE_STATIC_CONF - ret = stat(file, &st); - if (ret == -1 && errno == ENOENT) { - /* Try to resolve the real path (if exists) */ - if (file[0] == '/') { - flb_utils_error(FLB_ERR_CFG_PLUGIN_FILE); - return -1; - } - - if (config->conf_path) { - snprintf(tmp, PATH_MAX, "%s%s", config->conf_path, file); - cfg = tmp; - } - } - else { - cfg = (char *) file; - } - - flb_debug("[plugin] opening configuration file %s", cfg); - - cf = flb_cf_create_from_file(NULL, cfg); -#else - cf = flb_config_static_open(file); -#endif - - if (!cf) { - return -1; - } - - /* read all 'plugins' sections */ - mk_list_foreach(head, &cf->sections) { - section = mk_list_entry(head, struct flb_cf_section, _head); - if (strcasecmp(section->name, "plugins") != 0) { - continue; - } - - cfl_list_foreach(head_e, §ion->properties->list) { - entry = cfl_list_entry(head_e, struct cfl_kvpair, _head); - if (strcasecmp(entry->key, "path") != 0) { - continue; - } - - /* Load plugin with router function */ - ret = flb_plugin_load_router(entry->val->data.as_string, config); - if (ret == -1) { - flb_cf_destroy(cf); - return -1; - } - } - } - - flb_cf_destroy(cf); - return 0; -} - -/* Destroy plugin context */ -void flb_plugin_destroy(struct flb_plugins *ctx) -{ - struct mk_list *tmp; - struct mk_list *head; - struct flb_plugin *plugin; - - mk_list_foreach_safe(head, tmp, &ctx->input) { - plugin = mk_list_entry(head, struct flb_plugin, _head); - destroy_plugin(plugin); - } - - mk_list_foreach_safe(head, tmp, &ctx->processor) { - plugin = mk_list_entry(head, struct flb_plugin, _head); - destroy_plugin(plugin); - } - - mk_list_foreach_safe(head, tmp, &ctx->filter) { - plugin = mk_list_entry(head, struct flb_plugin, _head); - destroy_plugin(plugin); - } - - mk_list_foreach_safe(head, tmp, &ctx->output) { - plugin = mk_list_entry(head, struct flb_plugin, _head); - destroy_plugin(plugin); - } - - flb_free(ctx); -} diff --git a/fluent-bit/src/flb_plugin_proxy.c b/fluent-bit/src/flb_plugin_proxy.c deleted file mode 100644 index 440c54525..000000000 --- a/fluent-bit/src/flb_plugin_proxy.c +++ /dev/null @@ -1,498 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <stdio.h> -#include <stdlib.h> -#include <sys/types.h> -#include <sys/stat.h> - -#include <monkey/mk_core.h> -#include <fluent-bit/flb_compat.h> -#include <fluent-bit/flb_info.h> -#include <fluent-bit/flb_log.h> -#include <fluent-bit/flb_mem.h> -#include <fluent-bit/flb_output.h> -#include <fluent-bit/flb_api.h> -#include <fluent-bit/flb_error.h> -#include <fluent-bit/flb_utils.h> -#include <fluent-bit/flb_plugin_proxy.h> -#include <fluent-bit/flb_input_log.h> - -/* Proxies */ -#include "proxy/go/go.h" - -#define PROXY_CALLBACK_TIME 1 /* 1 seconds */ - -static void proxy_cb_flush(struct flb_event_chunk *event_chunk, - struct flb_output_flush *out_flush, - struct flb_input_instance *i_ins, - void *out_context, - struct flb_config *config) -{ - int ret = FLB_ERROR; - struct flb_plugin_proxy_context *ctx = out_context; - (void) i_ins; - (void) config; - - -#ifdef FLB_HAVE_PROXY_GO - if (ctx->proxy->def->proxy == FLB_PROXY_GOLANG) { - flb_trace("[GO] entering go_flush()"); - ret = proxy_go_output_flush(ctx, - event_chunk->data, - event_chunk->size, - event_chunk->tag, - flb_sds_len(event_chunk->tag)); - } -#else - (void) ctx; -#endif - - if (ret != FLB_OK && ret != FLB_RETRY && ret != FLB_ERROR) { - FLB_OUTPUT_RETURN(FLB_ERROR); - } - - FLB_OUTPUT_RETURN(ret); -} - -static int flb_proxy_input_cb_collect(struct flb_input_instance *ins, - struct flb_config *config, void *in_context) -{ - int ret = FLB_OK; - size_t len = 0; - void *data = NULL; - struct flb_plugin_input_proxy_context *ctx = (struct flb_plugin_input_proxy_context *) in_context; - -#ifdef FLB_HAVE_PROXY_GO - if (ctx->proxy->def->proxy == FLB_PROXY_GOLANG) { - flb_trace("[GO] entering go_collect()"); - ret = proxy_go_input_collect(ctx->proxy, &data, &len); - - if (len == 0) { - flb_trace("[GO] No logs are ingested"); - return -1; - } - - if (ret == -1) { - flb_errno(); - return -1; - } - - flb_input_log_append(ins, NULL, 0, data, len); - - ret = proxy_go_input_cleanup(ctx->proxy, data); - if (ret == -1) { - flb_errno(); - return -1; - } - } -#endif - - return 0; -} - -static int flb_proxy_input_cb_init(struct flb_input_instance *ins, - struct flb_config *config, void *data) -{ - int ret = -1; - struct flb_plugin_input_proxy_context *ctx; - struct flb_plugin_proxy_context *pc; - - /* Allocate space for the configuration context */ - ctx = flb_malloc(sizeof(struct flb_plugin_input_proxy_context)); - if (!ctx) { - flb_errno(); - return -1; - } - - /* Before to initialize for proxy, set the proxy instance reference */ - pc = (struct flb_plugin_proxy_context *)(ins->context); - ctx->proxy = pc->proxy; - - /* Before to initialize, set the instance reference */ - pc->proxy->instance = ins; - - /* Based on 'proxy', use the proper handler */ - if (pc->proxy->def->proxy == FLB_PROXY_GOLANG) { -#ifdef FLB_HAVE_PROXY_GO - ret = proxy_go_input_init(pc->proxy); - - if (ret == -1) { - flb_error("Could not initialize proxy for threaded input plugin"); - goto init_error; - } -#else - flb_error("Could not find initializing function on proxy for threaded input plugin"); - goto init_error; -#endif - } - else { - flb_error("[proxy] unrecognized input proxy handler %i", - pc->proxy->def->proxy); - } - - /* Set the context */ - flb_input_set_context(ins, ctx); - - /* Collect upon data available on timer */ - ret = flb_input_set_collector_time(ins, - flb_proxy_input_cb_collect, - PROXY_CALLBACK_TIME, 0, - config); - - if (ret == -1) { - flb_error("Could not set collector for threaded proxy input plugin"); - goto init_error; - } - ctx->coll_fd = ret; - - return ret; - -init_error: - flb_free(ctx); - - return -1; -} - -static void flb_proxy_input_cb_pause(void *data, struct flb_config *config) -{ - struct flb_plugin_input_proxy_context *ctx = data; - - flb_input_collector_pause(ctx->coll_fd, ctx->proxy->instance); -} - -static void flb_proxy_input_cb_resume(void *data, struct flb_config *config) -{ - struct flb_plugin_input_proxy_context *ctx = data; - - flb_input_collector_resume(ctx->coll_fd, ctx->proxy->instance); -} - -static void flb_plugin_proxy_destroy(struct flb_plugin_proxy *proxy); - -static int flb_proxy_output_cb_exit(void *out_context, struct flb_config *config) -{ - struct flb_plugin_proxy_context *ctx = out_context; - struct flb_plugin_proxy *proxy = (ctx->proxy); - - if (!out_context) { - return 0; - } - - if (proxy->def->proxy == FLB_PROXY_GOLANG) { -#ifdef FLB_HAVE_PROXY_GO - proxy_go_output_destroy(ctx); -#endif - } - - flb_free(ctx); - return 0; -} - -static void flb_proxy_output_cb_destroy(struct flb_output_plugin *plugin) -{ - struct flb_plugin_proxy *proxy = (struct flb_plugin_proxy *) plugin->proxy; - /* cleanup */ - void (*cb_unregister)(struct flb_plugin_proxy_def *def); - - cb_unregister = flb_plugin_proxy_symbol(proxy, "FLBPluginUnregister"); - if (cb_unregister != NULL) { - cb_unregister(proxy->def); - } - - if (proxy->def->proxy == FLB_PROXY_GOLANG) { -#ifdef FLB_HAVE_PROXY_GO - proxy_go_output_unregister(proxy->data); -#endif - } - - flb_plugin_proxy_destroy(proxy); -} - -static int flb_proxy_input_cb_exit(void *in_context, struct flb_config *config) -{ - struct flb_plugin_input_proxy_context *ctx = in_context; - struct flb_plugin_proxy *proxy = (ctx->proxy); - - if (!in_context) { - return 0; - } - - if (proxy->def->proxy == FLB_PROXY_GOLANG) { -#ifdef FLB_HAVE_PROXY_GO - proxy_go_input_destroy(ctx); -#endif - } - - flb_free(ctx); - return 0; -} - -static void flb_proxy_input_cb_destroy(struct flb_input_plugin *plugin) -{ - struct flb_plugin_proxy *proxy = (struct flb_plugin_proxy *) plugin->proxy; - /* cleanup */ - void (*cb_unregister)(struct flb_plugin_proxy_def *def); - - cb_unregister = flb_plugin_proxy_symbol(proxy, "FLBPluginUnregister"); - if (cb_unregister != NULL) { - cb_unregister(proxy->def); - } - - if (proxy->def->proxy == FLB_PROXY_GOLANG) { -#ifdef FLB_HAVE_PROXY_GO - proxy_go_input_unregister(proxy->data); -#endif - } - - flb_plugin_proxy_destroy(proxy); -} - -static int flb_proxy_register_output(struct flb_plugin_proxy *proxy, - struct flb_plugin_proxy_def *def, - struct flb_config *config) -{ - struct flb_output_plugin *out; - - out = flb_calloc(1, sizeof(struct flb_output_plugin)); - if (!out) { - flb_errno(); - return -1; - } - - /* Plugin registration */ - out->type = FLB_OUTPUT_PLUGIN_PROXY; - out->proxy = proxy; - out->flags = def->flags; - out->name = def->name; - out->description = def->description; - mk_list_add(&out->_head, &config->out_plugins); - - /* - * Set proxy callbacks: external plugins which are not following - * the core plugins specs, have a different callback approach, so - * we put our proxy-middle callbacks to do the translation properly. - */ - out->cb_flush = proxy_cb_flush; - out->cb_exit = flb_proxy_output_cb_exit; - out->cb_destroy = flb_proxy_output_cb_destroy; - return 0; -} - -static int flb_proxy_register_input(struct flb_plugin_proxy *proxy, - struct flb_plugin_proxy_def *def, - struct flb_config *config) -{ - struct flb_input_plugin *in; - - in = flb_calloc(1, sizeof(struct flb_input_plugin)); - if (!in) { - flb_errno(); - return -1; - } - - /* Plugin registration */ - in->type = FLB_INPUT_PLUGIN_PROXY; - in->proxy = proxy; - in->flags = def->flags | FLB_INPUT_THREADED; - in->name = flb_strdup(def->name); - in->description = def->description; - mk_list_add(&in->_head, &config->in_plugins); - - /* - * Set proxy callbacks: external plugins which are not following - * the core plugins specs, have a different callback approach, so - * we put our proxy-middle callbacks to do the translation properly. - */ - in->cb_init = flb_proxy_input_cb_init; - in->cb_collect = flb_proxy_input_cb_collect; - in->cb_flush_buf = NULL; - in->cb_exit = flb_proxy_input_cb_exit; - in->cb_destroy = flb_proxy_input_cb_destroy; - in->cb_pause = flb_proxy_input_cb_pause; - in->cb_resume = flb_proxy_input_cb_resume; - return 0; -} - -void *flb_plugin_proxy_symbol(struct flb_plugin_proxy *proxy, - const char *symbol) -{ - void *s; - - dlerror(); - s = dlsym(proxy->dso_handler, symbol); - if (dlerror() != NULL) { - return NULL; - } - return s; -} - -int flb_plugin_proxy_register(struct flb_plugin_proxy *proxy, - struct flb_config *config) -{ - int ret; - int (*cb_register)(struct flb_plugin_proxy_def *); - struct flb_plugin_proxy_def *def = proxy->def; - - /* Lookup the registration callback */ - cb_register = flb_plugin_proxy_symbol(proxy, "FLBPluginRegister"); - if (!cb_register) { - return -1; - } - - /* - * Create a temporary definition used for registration. This definition - * aims to be be populated by plugin in the registration phase with: - * - * - plugin type (or proxy type, e.g: Golang) - * - plugin name - * - plugin description - */ - - /* Do the registration */ - ret = cb_register(def); - if (ret == -1) { - flb_free(def); - return -1; - } - - /* - * Each plugin proxy/type, have their own handler, based on the data - * provided in the registration invoke the proper handler. - */ - ret = -1; - if (def->proxy == FLB_PROXY_GOLANG) { -#ifdef FLB_HAVE_PROXY_GO - if (def->type == FLB_PROXY_OUTPUT_PLUGIN) { - ret = proxy_go_output_register(proxy, def); - } - else if (def->type == FLB_PROXY_INPUT_PLUGIN) { - ret = proxy_go_input_register(proxy, def); - } -#endif - } - if (ret == 0) { - /* - * We got a plugin that can do it job, now we need to create the - * real link to the 'output' interface - */ - if (def->type == FLB_PROXY_OUTPUT_PLUGIN) { - flb_proxy_register_output(proxy, def, config); - } - else if (def->type == FLB_PROXY_INPUT_PLUGIN) { - flb_proxy_register_input(proxy, def, config); - } - } - - return 0; -} - -int flb_plugin_proxy_output_init(struct flb_plugin_proxy *proxy, - struct flb_output_instance *o_ins, - struct flb_config *config) -{ - int ret = -1; - - /* Before to initialize, set the instance reference */ - proxy->instance = o_ins; - - /* Based on 'proxy', use the proper handler */ - if (proxy->def->proxy == FLB_PROXY_GOLANG) { -#ifdef FLB_HAVE_PROXY_GO - ret = proxy_go_output_init(proxy); -#endif - } - else { - flb_error("[proxy] unrecognized proxy handler %i", - proxy->def->proxy); - } - - return ret; -} - -struct flb_plugin_proxy *flb_plugin_proxy_create(const char *dso_path, int type, - struct flb_config *config) -{ - void *handle; - struct flb_plugin_proxy *proxy; - - /* Load shared library */ - handle = dlopen(dso_path, RTLD_LAZY); - if (!handle) { - flb_error("[proxy] error opening plugin %s: '%s'", - dso_path, dlerror()); - return NULL; - } - - /* Proxy Context */ - proxy = flb_malloc(sizeof(struct flb_plugin_proxy)); - if (!proxy) { - flb_errno(); - dlclose(handle); - return NULL; - } - - /* API Context */ - proxy->api = flb_api_create(); - if (!proxy->api) { - dlclose(handle); - flb_free(proxy); - return NULL; - } - - proxy->def = flb_malloc(sizeof(struct flb_plugin_proxy_def)); - if (!proxy->def) { - flb_errno(); - dlclose(handle); - flb_api_destroy(proxy->api); - flb_free(proxy); - return NULL; - } - - /* Set fields and add it to the list */ - proxy->def->type = type; - proxy->dso_handler = handle; - proxy->data = NULL; - mk_list_add(&proxy->_head, &config->proxies); - - /* Register plugin */ - flb_plugin_proxy_register(proxy, config); - - return proxy; -} - -static void flb_plugin_proxy_destroy(struct flb_plugin_proxy *proxy) -{ - flb_free(proxy->def); - flb_api_destroy(proxy->api); - dlclose(proxy->dso_handler); - mk_list_del(&proxy->_head); - flb_free(proxy); -} - -int flb_plugin_proxy_set(struct flb_plugin_proxy_def *def, int type, - int proxy, char *name, char *description) -{ - def->type = type; - def->proxy = proxy; - def->name = flb_strdup(name); - def->description = flb_strdup(description); - - return 0; -} diff --git a/fluent-bit/src/flb_processor.c b/fluent-bit/src/flb_processor.c deleted file mode 100644 index c46bc16a5..000000000 --- a/fluent-bit/src/flb_processor.c +++ /dev/null @@ -1,1129 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2023 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <fluent-bit/flb_info.h> -#include <fluent-bit/flb_log.h> -#include <fluent-bit/flb_env.h> -#include <fluent-bit/flb_mem.h> -#include <fluent-bit/flb_event.h> -#include <fluent-bit/flb_processor.h> -#include <fluent-bit/flb_processor_plugin.h> -#include <fluent-bit/flb_filter.h> -#include <fluent-bit/flb_kv.h> -#include <fluent-bit/flb_log_event_decoder.h> -#include <fluent-bit/flb_log_event_encoder.h> - -static int acquire_lock(pthread_mutex_t *lock, - size_t retry_limit, - size_t retry_delay) -{ - size_t retry_count; - int result; - - retry_count = 0; - - do { - result = pthread_mutex_lock(lock); - - if (result != 0) { - - if (result == EAGAIN) { - retry_count++; - - usleep(retry_delay); - } - else { - break; - } - } - } - while (result != 0 && - retry_count < retry_limit); - - if (result != 0) { - return FLB_FALSE; - } - - return FLB_TRUE; -} - -static int release_lock(pthread_mutex_t *lock, - size_t retry_limit, - size_t retry_delay) -{ - size_t retry_count; - int result; - - retry_count = 0; - - do { - result = pthread_mutex_unlock(lock); - - if (result != 0) { - - if (result == EAGAIN) { - retry_count++; - - usleep(retry_delay); - } - else { - break; - } - } - } - while (result != 0 && - retry_count < retry_limit); - - if (result != 0) { - return FLB_FALSE; - } - - return FLB_TRUE; -} - -/* - * A processor creates a chain of processing units for different telemetry data - * types such as logs, metrics and traces. - * - * From a design perspective, a Processor can be run independently from inputs, outputs - * or unit tests directly. - */ -struct flb_processor *flb_processor_create(struct flb_config *config, - char *name, - void *source_plugin_instance, - int source_plugin_type) -{ - struct flb_processor *proc; - - proc = flb_calloc(1, sizeof(struct flb_processor)); - - if (!proc) { - flb_errno(); - return NULL; - } - - proc->config = config; - proc->is_active = FLB_FALSE; - proc->data = source_plugin_instance; - proc->source_plugin_type = source_plugin_type; - - /* lists for types */ - mk_list_init(&proc->logs); - mk_list_init(&proc->metrics); - mk_list_init(&proc->traces); - - return proc; -} - -struct flb_processor_unit *flb_processor_unit_create(struct flb_processor *proc, - int event_type, - char *unit_name) -{ - int result; - struct mk_list *head; - int filter_event_type; - struct flb_filter_plugin *f = NULL; - struct flb_filter_instance *f_ins; - struct flb_config *config = proc->config; - struct flb_processor_unit *pu = NULL; - struct flb_processor_instance *processor_instance; - - /* - * Looking the processor unit by using it's name and type, the first list we - * will iterate are the common pipeline filters. - */ - mk_list_foreach(head, &config->filter_plugins) { - f = mk_list_entry(head, struct flb_filter_plugin, _head); - - filter_event_type = f->event_type; - - if (filter_event_type == 0) { - filter_event_type = FLB_FILTER_LOGS; - } - - /* skip filters which don't handle the required type */ - if ((event_type & filter_event_type) != 0) { - if (strcmp(f->name, unit_name) == 0) { - break; - } - } - - f = NULL; - } - - /* allocate and initialize processor unit context */ - pu = flb_calloc(1, sizeof(struct flb_processor_unit)); - - if (!pu) { - flb_errno(); - return NULL; - } - - pu->parent = proc; - pu->event_type = event_type; - pu->name = flb_sds_create(unit_name); - - if (!pu->name) { - flb_free(pu); - return NULL; - } - mk_list_init(&pu->unused_list); - - result = pthread_mutex_init(&pu->lock, NULL); - - if (result != 0) { - flb_sds_destroy(pu->name); - flb_free(pu); - - return NULL; - } - - /* If we matched a pipeline filter, create the speacial processing unit for it */ - if (f) { - /* create an instance of the filter */ - f_ins = flb_filter_new(config, unit_name, NULL); - - if (!f_ins) { - pthread_mutex_destroy(&pu->lock); - flb_sds_destroy(pu->name); - flb_free(pu); - - return NULL; - } - - f_ins->parent_processor = (void *) pu; - - /* matching rule: just set to workaround the pipeline initializer */ - f_ins->match = flb_sds_create("*"); - - if (f_ins->match == NULL) { - flb_filter_instance_destroy(f_ins); - - pthread_mutex_destroy(&pu->lock); - flb_sds_destroy(pu->name); - flb_free(pu); - - return NULL; - } - - /* unit type and context */ - pu->unit_type = FLB_PROCESSOR_UNIT_FILTER; - pu->ctx = f_ins; - - /* - * The filter was added to the linked list config->filters, since this filter - * won't run as part of the normal pipeline, we just unlink the node. - */ - mk_list_del(&f_ins->_head); - - /* link the filter to the unused list */ - mk_list_add(&f_ins->_head, &pu->unused_list); - } - else { - pu->unit_type = FLB_PROCESSOR_UNIT_NATIVE; - - /* create an instance of the processor */ - processor_instance = flb_processor_instance_create(config, unit_name, NULL); - - if (processor_instance == NULL) { - flb_error("[processor] error creating native processor instance %s", pu->name); - - pthread_mutex_destroy(&pu->lock); - flb_sds_destroy(pu->name); - flb_free(pu); - - return NULL; - } - - /* unit type and context */ - pu->ctx = (void *) processor_instance; - } - - /* Link the processor unit to the proper list */ - if (event_type == FLB_PROCESSOR_LOGS) { - mk_list_add(&pu->_head, &proc->logs); - } - else if (event_type == FLB_PROCESSOR_METRICS) { - mk_list_add(&pu->_head, &proc->metrics); - } - else if (event_type == FLB_PROCESSOR_TRACES) { - mk_list_add(&pu->_head, &proc->traces); - } - - pu->stage = proc->stage_count; - proc->stage_count++; - - return pu; -} - -int flb_processor_unit_set_property(struct flb_processor_unit *pu, const char *k, struct cfl_variant *v) -{ - struct cfl_variant *val; - int i; - int ret; - - if (pu->unit_type == FLB_PROCESSOR_UNIT_FILTER) { - - if (v->type == CFL_VARIANT_STRING) { - return flb_filter_set_property(pu->ctx, k, v->data.as_string); - } - else if (v->type == CFL_VARIANT_ARRAY) { - - for (i = 0; i < v->data.as_array->entry_count; i++) { - val = v->data.as_array->entries[i]; - ret = flb_filter_set_property(pu->ctx, k, val->data.as_string); - - if (ret == -1) { - return ret; - } - } - return 0; - } - } - - return flb_processor_instance_set_property( - (struct flb_processor_instance *) pu->ctx, - k, v->data.as_string); -} - -void flb_processor_unit_destroy(struct flb_processor_unit *pu) -{ - struct flb_processor *proc = pu->parent; - struct flb_config *config = proc->config; - - if (pu->unit_type == FLB_PROCESSOR_UNIT_FILTER) { - flb_filter_instance_exit(pu->ctx, config); - flb_filter_instance_destroy(pu->ctx); - } - else { - flb_processor_instance_exit( - (struct flb_processor_instance *) pu->ctx, - config); - - flb_processor_instance_destroy( - (struct flb_processor_instance *) pu->ctx); - } - - pthread_mutex_destroy(&pu->lock); - - flb_sds_destroy(pu->name); - flb_free(pu); -} - -/* Initialize a specific unit */ -int flb_processor_unit_init(struct flb_processor_unit *pu) -{ - int ret = -1; - struct flb_config; - struct flb_processor *proc = pu->parent; - - if (pu->unit_type == FLB_PROCESSOR_UNIT_FILTER) { - ret = flb_filter_init(proc->config, pu->ctx); - - if (ret == -1) { - flb_error("[processor] error initializing unit filter %s", pu->name); - return -1; - } - } - else { - ret = flb_processor_instance_init( - (struct flb_processor_instance *) pu->ctx, - proc->data, - 0, - proc->config); - - if (ret == -1) { - flb_error("[processor] error initializing unit native processor " - "%s", pu->name); - - return -1; - } - } - - return ret; -} - -/* Initialize the processor and all the units */ -int flb_processor_init(struct flb_processor *proc) -{ - int ret; - int count = 0; - struct mk_list *head; - struct flb_processor_unit *pu; - - /* Go through every unit and initialize it */ - mk_list_foreach(head, &proc->logs) { - pu = mk_list_entry(head, struct flb_processor_unit, _head); - ret = flb_processor_unit_init(pu); - - if (ret == -1) { - return -1; - } - count++; - } - - mk_list_foreach(head, &proc->metrics) { - pu = mk_list_entry(head, struct flb_processor_unit, _head); - ret = flb_processor_unit_init(pu); - - if (ret == -1) { - return -1; - } - count++; - } - - mk_list_foreach(head, &proc->traces) { - pu = mk_list_entry(head, struct flb_processor_unit, _head); - ret = flb_processor_unit_init(pu); - - if (ret == -1) { - return -1; - } - count++; - } - - if (count > 0) { - proc->is_active = FLB_TRUE; - } - return 0; -} - -int flb_processor_is_active(struct flb_processor *proc) -{ - if (proc->is_active) { - return FLB_TRUE; - } - - return FLB_FALSE; -} - -/* - * This function will run all the processor units for the given tag and data, note - * that depending of the 'type', 'data' can reference a msgpack for logs, a CMetrics - * context for metrics or a 'CTraces' context for traces. - */ -int flb_processor_run(struct flb_processor *proc, - size_t starting_stage, - int type, - const char *tag, size_t tag_len, - void *data, size_t data_size, - void **out_buf, size_t *out_size) -{ - int ret; - void *cur_buf; - size_t cur_size; - void *tmp_buf; - size_t tmp_size; - int decoder_result; - struct mk_list *head; - struct mk_list *list = NULL; - struct flb_log_event log_event; - struct flb_processor_unit *pu; - struct flb_filter_instance *f_ins; - struct flb_processor_instance *p_ins; - - if (type == FLB_PROCESSOR_LOGS) { - list = &proc->logs; - } - else if (type == FLB_PROCESSOR_METRICS) { - list = &proc->metrics; - } - else if (type == FLB_PROCESSOR_TRACES) { - list = &proc->traces; - } - - /* set current data buffer */ - cur_buf = data; - cur_size = data_size; - - /* iterate list units */ - mk_list_foreach(head, list) { - pu = mk_list_entry(head, struct flb_processor_unit, _head); - - /* This is meant to be used when filters or processors re-inject - * records in the pipeline. This way we can ensure that they will - * continue the process at the right stage. - */ - if (pu->stage < starting_stage) { - continue; - } - - tmp_buf = NULL; - tmp_size = 0; - - ret = acquire_lock(&pu->lock, - FLB_PROCESSOR_LOCK_RETRY_LIMIT, - FLB_PROCESSOR_LOCK_RETRY_DELAY); - - if (ret != FLB_TRUE) { - return -1; - } - - /* run the unit */ - if (pu->unit_type == FLB_PROCESSOR_UNIT_FILTER) { - /* get the filter context */ - f_ins = pu->ctx; - - /* run the filtering callback */ - ret = f_ins->p->cb_filter(cur_buf, cur_size, /* msgpack buffer */ - tag, tag_len, /* tag */ - &tmp_buf, &tmp_size, /* output buffer */ - f_ins, /* filter instance */ - proc->data, /* (input/output) instance context */ - f_ins->context, /* filter context */ - proc->config); - - /* - * The cb_filter() function return status tells us if something changed - * during it process. The possible values are: - * - * - FLB_FILTER_MODIFIED: the record was modified and the output buffer - * contains the new record. - * - * - FLB_FILTER_NOTOUCH: the record was not modified. - * - */ - if (ret == FLB_FILTER_MODIFIED) { - - /* release intermediate buffer */ - if (cur_buf != data) { - flb_free(cur_buf); - } - - /* - * if the content has been modified and the returned size is zero, it means - * the whole content has been dropped, on this case we just return since - * no more data exists to be processed. - */ - if (tmp_size == 0) { - *out_buf = NULL; - *out_size = 0; - - release_lock(&pu->lock, - FLB_PROCESSOR_LOCK_RETRY_LIMIT, - FLB_PROCESSOR_LOCK_RETRY_DELAY); - - return 0; - } - - /* set new buffer */ - cur_buf = tmp_buf; - cur_size = tmp_size; - } - else if (ret == FLB_FILTER_NOTOUCH) { - /* keep original data, do nothing */ - } - } - else { - /* get the processor context */ - p_ins = pu->ctx; - - ret = 0; - - /* run the process callback */ - if (type == FLB_PROCESSOR_LOGS) { - if (p_ins->p->cb_process_logs != NULL) { - flb_log_event_encoder_reset(p_ins->log_encoder); - - decoder_result = flb_log_event_decoder_init( - p_ins->log_decoder, cur_buf, cur_size); - - if (decoder_result != FLB_EVENT_DECODER_SUCCESS) { - flb_log_event_decoder_reset(p_ins->log_decoder, NULL, 0); - - if (cur_buf != data) { - flb_free(cur_buf); - } - - release_lock(&pu->lock, - FLB_PROCESSOR_LOCK_RETRY_LIMIT, - FLB_PROCESSOR_LOCK_RETRY_DELAY); - - return -1; - } - - ret = FLB_PROCESSOR_SUCCESS; - - do { - decoder_result = flb_log_event_decoder_next( - p_ins->log_decoder, - &log_event); - - if (decoder_result == FLB_EVENT_DECODER_SUCCESS) { - ret = p_ins->p->cb_process_logs(p_ins, - p_ins->log_encoder, - &log_event, - tag, tag_len); - } - } - while (decoder_result == FLB_EVENT_DECODER_SUCCESS && - ret == FLB_PROCESSOR_SUCCESS); - - flb_log_event_decoder_reset(p_ins->log_decoder, NULL, 0); - - if (cur_buf != data) { - flb_free(cur_buf); - } - - if (ret != FLB_PROCESSOR_SUCCESS) { - flb_log_event_encoder_reset(p_ins->log_encoder); - - release_lock(&pu->lock, - FLB_PROCESSOR_LOCK_RETRY_LIMIT, - FLB_PROCESSOR_LOCK_RETRY_DELAY); - - return -1; - } - - if (p_ins->log_encoder->output_length == 0) { - flb_log_event_encoder_reset(p_ins->log_encoder); - - *out_buf = NULL; - *out_size = 0; - - release_lock(&pu->lock, - FLB_PROCESSOR_LOCK_RETRY_LIMIT, - FLB_PROCESSOR_LOCK_RETRY_DELAY); - - return 0; - } - - flb_log_event_encoder_claim_internal_buffer_ownership(p_ins->log_encoder); - - /* set new buffer */ - cur_buf = p_ins->log_encoder->output_buffer; - cur_size = p_ins->log_encoder->output_length; - - flb_log_event_encoder_reset(p_ins->log_encoder); - } - } - else if (type == FLB_PROCESSOR_METRICS) { - - if (p_ins->p->cb_process_metrics != NULL) { - ret = p_ins->p->cb_process_metrics(p_ins, - (struct cmt *) cur_buf, - tag, - tag_len); - - if (ret != FLB_PROCESSOR_SUCCESS) { - release_lock(&pu->lock, - FLB_PROCESSOR_LOCK_RETRY_LIMIT, - FLB_PROCESSOR_LOCK_RETRY_DELAY); - - return -1; - } - } - } - else if (type == FLB_PROCESSOR_TRACES) { - - if (p_ins->p->cb_process_traces != NULL) { - ret = p_ins->p->cb_process_traces(p_ins, - (struct ctrace *) cur_buf, - tag, - tag_len); - - if (ret != FLB_PROCESSOR_SUCCESS) { - release_lock(&pu->lock, - FLB_PROCESSOR_LOCK_RETRY_LIMIT, - FLB_PROCESSOR_LOCK_RETRY_DELAY); - - return -1; - } - } - } - } - - release_lock(&pu->lock, - FLB_PROCESSOR_LOCK_RETRY_LIMIT, - FLB_PROCESSOR_LOCK_RETRY_DELAY); - } - - /* set output buffer */ - if (out_buf != NULL) { - *out_buf = cur_buf; - } - - if (out_size != NULL) { - *out_size = cur_size; - } - - return 0; -} - -void flb_processor_destroy(struct flb_processor *proc) -{ - struct mk_list *head; - struct mk_list *tmp; - struct flb_processor_unit *pu; - - mk_list_foreach_safe(head, tmp, &proc->logs) { - pu = mk_list_entry(head, struct flb_processor_unit, _head); - mk_list_del(&pu->_head); - flb_processor_unit_destroy(pu); - } - - mk_list_foreach_safe(head, tmp, &proc->metrics) { - pu = mk_list_entry(head, struct flb_processor_unit, _head); - mk_list_del(&pu->_head); - flb_processor_unit_destroy(pu); - } - - mk_list_foreach_safe(head, tmp, &proc->traces) { - pu = mk_list_entry(head, struct flb_processor_unit, _head); - mk_list_del(&pu->_head); - flb_processor_unit_destroy(pu); - } - flb_free(proc); -} - - -static int load_from_config_format_group(struct flb_processor *proc, int type, struct cfl_variant *val) -{ - int i; - int ret; - char *name; - struct cfl_variant *tmp; - struct cfl_array *array; - struct cfl_kvlist *kvlist; - struct cfl_kvpair *pair = NULL; - struct cfl_list *head; - struct flb_processor_unit *pu; - struct flb_filter_instance *f_ins; - - if (val->type != CFL_VARIANT_ARRAY) { - return -1; - } - - array = val->data.as_array; - for (i = 0; i < array->entry_count; i++) { - /* every entry in the array must be a map */ - tmp = array->entries[i]; - - if (tmp->type != CFL_VARIANT_KVLIST) { - return -1; - } - - kvlist = tmp->data.as_kvlist; - - /* get the processor name, this is a mandatory config field */ - tmp = cfl_kvlist_fetch(kvlist, "name"); - - if (!tmp) { - flb_error("processor configuration don't have a 'name' defined"); - return -1; - } - - /* create the processor unit and load all the properties */ - name = tmp->data.as_string; - pu = flb_processor_unit_create(proc, type, name); - - if (!pu) { - flb_error("cannot create '%s' processor unit", name); - return -1; - } - - /* iterate list of properties and set each one (skip name) */ - cfl_list_foreach(head, &kvlist->list) { - pair = cfl_list_entry(head, struct cfl_kvpair, _head); - - if (strcmp(pair->key, "name") == 0) { - continue; - } - - /* If filter plugin in processor unit has its own match rule, - * we must release the pre-allocated '*' match at first. - */ - if (pu->unit_type == FLB_PROCESSOR_UNIT_FILTER) { - - if (strcmp(pair->key, "match") == 0) { - f_ins = (struct flb_filter_instance *)pu->ctx; - - if (f_ins->match != NULL) { - flb_sds_destroy(f_ins->match); - f_ins->match = NULL; - } - } - } - - ret = flb_processor_unit_set_property(pu, pair->key, pair->val); - - if (ret == -1) { - flb_error("cannot set property '%s' for processor '%s'", pair->key, name); - return -1; - } - } - } - - return 0; - -} - -/* Load processors into an input instance */ -int flb_processors_load_from_config_format_group(struct flb_processor *proc, struct flb_cf_group *g) -{ - int ret; - struct cfl_variant *val; - - /* logs */ - val = cfl_kvlist_fetch(g->properties, "logs"); - - if (val) { - ret = load_from_config_format_group(proc, FLB_PROCESSOR_LOGS, val); - - if (ret == -1) { - flb_error("failed to load 'logs' processors"); - return -1; - } - } - - /* metrics */ - val = cfl_kvlist_fetch(g->properties, "metrics"); - - if (val) { - ret = load_from_config_format_group(proc, FLB_PROCESSOR_METRICS, val); - - if (ret == -1) { - flb_error("failed to load 'metrics' processors"); - return -1; - } - } - - /* traces */ - val = cfl_kvlist_fetch(g->properties, "traces"); - if (val) { - ret = load_from_config_format_group(proc, FLB_PROCESSOR_TRACES, val); - - if (ret == -1) { - flb_error("failed to load 'traces' processors"); - return -1; - } - } - - return 0; -} - - - - - - - - -static inline int prop_key_check(const char *key, const char *kv, int k_len) -{ - int len; - - len = strlen(key); - - if (strncasecmp(key, kv, k_len) == 0 && len == k_len) { - return 0; - } - - return -1; -} - -int flb_processor_instance_set_property(struct flb_processor_instance *ins, - const char *k, const char *v) -{ - int len; - int ret; - flb_sds_t tmp; - struct flb_kv *kv; - - len = strlen(k); - tmp = flb_env_var_translate(ins->config->env, v); - - if (!tmp) { - return -1; - } - - if (prop_key_check("alias", k, len) == 0 && tmp) { - ins->alias = tmp; - } - else if (prop_key_check("log_level", k, len) == 0 && tmp) { - ret = flb_log_get_level_str(tmp); - flb_sds_destroy(tmp); - - if (ret == -1) { - return -1; - } - ins->log_level = ret; - } - else { - /* - * Create the property, we don't pass the value since we will - * map it directly to avoid an extra memory allocation. - */ - kv = flb_kv_item_create(&ins->properties, (char *) k, NULL); - - if (!kv) { - - if (tmp) { - flb_sds_destroy(tmp); - } - return -1; - } - kv->val = tmp; - } - - return 0; -} - -const char *flb_processor_instance_get_property( - const char *key, - struct flb_processor_instance *ins) -{ - return flb_kv_get_key_value(key, &ins->properties); -} - -struct flb_processor_instance *flb_processor_instance_create( - struct flb_config *config, - const char *name, void *data) -{ - struct flb_processor_instance *instance; - struct flb_processor_plugin *plugin; - struct mk_list *head; - int id; - - if (name == NULL) { - return NULL; - } - - mk_list_foreach(head, &config->processor_plugins) { - plugin = mk_list_entry(head, struct flb_processor_plugin, _head); - - if (strcasecmp(plugin->name, name) == 0) { - break; - } - plugin = NULL; - } - - if (!plugin) { - return NULL; - } - - instance = flb_calloc(1, sizeof(struct flb_filter_instance)); - - if (!instance) { - flb_errno(); - return NULL; - } - - instance->config = config; - - /* Get an ID */ - id = 0; - - /* format name (with instance id) */ - snprintf(instance->name, sizeof(instance->name) - 1, - "%s.%i", plugin->name, id); - - instance->id = id; - instance->alias = NULL; - instance->p = plugin; - instance->data = data; - instance->log_level = -1; - - mk_list_init(&instance->properties); - - instance->log_encoder = flb_log_event_encoder_create( - FLB_LOG_EVENT_FORMAT_DEFAULT); - - if (instance->log_encoder == NULL) { - flb_plg_error(instance, "log event encoder initialization error"); - - flb_processor_instance_destroy(instance); - - instance = NULL; - } - - instance->log_decoder = flb_log_event_decoder_create(NULL, 0); - - if (instance->log_decoder == NULL) { - flb_plg_error(instance, "log event decoder initialization error"); - - flb_processor_instance_destroy(instance); - - instance = NULL; - } - - return instance; -} - -void flb_processor_instance_exit( - struct flb_processor_instance *ins, - struct flb_config *config) -{ - struct flb_processor_plugin *plugin; - - plugin = ins->p; - - if (plugin->cb_exit != NULL && - ins->context != NULL) { - plugin->cb_exit(ins); - } -} - -const char *flb_processor_instance_get_name(struct flb_processor_instance *ins) -{ - if (ins->alias) { - return ins->alias; - } - - return ins->name; -} - -int flb_processor_instance_check_properties( - struct flb_processor_instance *ins, - struct flb_config *config) -{ - int ret = 0; - struct mk_list *config_map; - struct flb_processor_plugin *p = ins->p; - - if (p->config_map) { - /* - * Create a dynamic version of the configmap that will be used by the specific - * instance in question. - */ - config_map = flb_config_map_create(config, p->config_map); - - if (!config_map) { - flb_error("[native processor] error loading config map for '%s' plugin", - p->name); - return -1; - } - ins->config_map = config_map; - - /* Validate incoming properties against config map */ - ret = flb_config_map_properties_check(ins->p->name, - &ins->properties, - ins->config_map); - - if (ret == -1) { - - if (config->program_name) { - flb_helper("try the command: %s -F %s -h\n", - config->program_name, ins->p->name); - } - return -1; - } - } - - return 0; -} - -int flb_processor_instance_init( - struct flb_processor_instance *ins, - void *source_plugin_instance, - int source_plugin_type, - struct flb_config *config) -{ - int ret; - char *name; - struct flb_processor_plugin *p; - - if (ins->log_level == -1 && - config->log != NULL) { - ins->log_level = config->log->level; - } - - p = ins->p; - - /* Get name or alias for the instance */ - name = (char *) flb_processor_instance_get_name(ins); - - /* CMetrics */ - ins->cmt = cmt_create(); - - if (!ins->cmt) { - flb_error("[processor] could not create cmetrics context: %s", - name); - - return -1; - } - - /* - * Before to call the initialization callback, make sure that the received - * configuration parameters are valid if the plugin is registering a config map. - */ - if (flb_processor_instance_check_properties(ins, config) == -1) { - return -1; - } - - /* Initialize the input */ - if (p->cb_init != NULL) { - ret = p->cb_init(ins, - source_plugin_instance, - source_plugin_type, - config); - - if (ret != 0) { - flb_error("[processor] failed initialize filter %s", ins->name); - - return -1; - } - } - - return 0; -} - -void flb_processor_instance_set_context( - struct flb_processor_instance *ins, - void *context) -{ - ins->context = context; -} - -void flb_processor_instance_destroy( - struct flb_processor_instance *ins) -{ - if (ins == NULL) { - return; - } - - /* destroy config map */ - if (ins->config_map != NULL) { - flb_config_map_destroy(ins->config_map); - } - - /* release properties */ - flb_kv_release(&ins->properties); - - /* Remove metrics */ -#ifdef FLB_HAVE_METRICS - if (ins->cmt != NULL) { - cmt_destroy(ins->cmt); - } -#endif - - if (ins->alias != NULL) { - flb_sds_destroy(ins->alias); - } - - if (ins->log_encoder != NULL) { - flb_log_event_encoder_destroy(ins->log_encoder); - } - - if (ins->log_decoder != NULL) { - flb_log_event_decoder_destroy(ins->log_decoder); - } - - flb_free(ins); -} diff --git a/fluent-bit/src/flb_ra_key.c b/fluent-bit/src/flb_ra_key.c deleted file mode 100644 index f167a7664..000000000 --- a/fluent-bit/src/flb_ra_key.c +++ /dev/null @@ -1,808 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <fluent-bit/flb_info.h> -#include <fluent-bit/flb_log.h> -#include <fluent-bit/flb_sds.h> -#include <fluent-bit/flb_mem.h> -#include <fluent-bit/flb_regex.h> -#include <fluent-bit/flb_ra_key.h> -#include <fluent-bit/record_accessor/flb_ra_parser.h> -#include <msgpack.h> -#include <limits.h> - -/* Map msgpack object into flb_ra_value representation */ -static int msgpack_object_to_ra_value(msgpack_object o, - struct flb_ra_value *result) -{ - result->o = o; - - /* Compose result with found value */ - if (o.type == MSGPACK_OBJECT_BOOLEAN) { - result->type = FLB_RA_BOOL; - result->val.boolean = o.via.boolean; - return 0; - } - else if (o.type == MSGPACK_OBJECT_POSITIVE_INTEGER || - o.type == MSGPACK_OBJECT_NEGATIVE_INTEGER) { - result->type = FLB_RA_INT; - result->val.i64 = o.via.i64; - return 0; - } - else if (o.type == MSGPACK_OBJECT_FLOAT32 || - o.type == MSGPACK_OBJECT_FLOAT) { - result->type = FLB_RA_FLOAT; - result->val.f64 = o.via.f64; - return 0; - } - else if (o.type == MSGPACK_OBJECT_STR) { - result->type = FLB_RA_STRING; - result->val.string = flb_sds_create_len((char *) o.via.str.ptr, - o.via.str.size); - - /* Handle cases where flb_sds_create_len fails */ - if (result->val.string == NULL) { - return -1; - } - return 0; - } - else if (o.type == MSGPACK_OBJECT_MAP) { - /* return boolean 'true', just denoting the existence of the key */ - result->type = FLB_RA_BOOL; - result->val.boolean = true; - return 0; - } - else if (o.type == MSGPACK_OBJECT_NIL) { - result->type = FLB_RA_NULL; - return 0; - } - - return -1; -} - -/* Return the entry position of key/val in the map */ -static int ra_key_val_id(flb_sds_t ckey, msgpack_object map) -{ - int i; - int map_size; - msgpack_object key; - - if (map.type != MSGPACK_OBJECT_MAP) { - return -1; - } - - map_size = map.via.map.size; - for (i = map_size - 1; i >= 0; i--) { - key = map.via.map.ptr[i].key; - - if (key.type != MSGPACK_OBJECT_STR) { - continue; - } - - /* Compare by length and by key name */ - if (flb_sds_cmp(ckey, key.via.str.ptr, key.via.str.size) != 0) { - continue; - } - - return i; - } - - return -1; -} - -static int msgpack_object_strcmp(msgpack_object o, char *str, int len) -{ - if (o.type != MSGPACK_OBJECT_STR) { - return -1; - } - - if (o.via.str.size != len) { - return -1; - } - - return strncmp(o.via.str.ptr, str, len); -} - -/* Lookup perfect match of sub-keys and map content */ -static int subkey_to_object(msgpack_object *map, struct mk_list *subkeys, - msgpack_object **out_key, msgpack_object **out_val) -{ - int i = 0; - int levels; - int matched = 0; - msgpack_object *found = NULL; - msgpack_object *key = NULL; - msgpack_object *val = NULL; - msgpack_object cur; - struct mk_list *head; - struct flb_ra_subentry *entry; - - /* Expected number of map levels in the map */ - levels = mk_list_size(subkeys); - - cur = *map; - - mk_list_foreach(head, subkeys) { - /* expected entry */ - entry = mk_list_entry(head, struct flb_ra_subentry, _head); - - /* Array Handling */ - if (entry->type == FLB_RA_PARSER_ARRAY_ID) { - /* check the current msgpack object is an array */ - if (cur.type != MSGPACK_OBJECT_ARRAY) { - return -1; - } - - /* Index limit and ensure no overflow */ - if (entry->array_id == INT_MAX || - cur.via.array.size < entry->array_id + 1) { - return -1; - } - - val = &cur.via.array.ptr[entry->array_id]; - cur = *val; - key = NULL; /* fill NULL since the type is array. */ - goto next; - } - - if (cur.type != MSGPACK_OBJECT_MAP) { - break; - } - - i = ra_key_val_id(entry->str, cur); - if (i == -1) { - found = NULL; - continue; - } - - key = &cur.via.map.ptr[i].key; - val = &cur.via.map.ptr[i].val; - - /* A bit obvious, but it's better to validate data type */ - if (key->type != MSGPACK_OBJECT_STR) { - found = NULL; - continue; - } - - found = key; - cur = cur.via.map.ptr[i].val; - - next: - matched++; - - if (levels == matched) { - break; - } - } - - /* No matches */ - if (!found || (matched > 0 && levels != matched)) { - return -1; - } - - *out_key = (msgpack_object *) key; - *out_val = (msgpack_object *) val; - - return 0; -} - -struct flb_ra_value *flb_ra_key_to_value(flb_sds_t ckey, - msgpack_object map, - struct mk_list *subkeys) -{ - int i; - int ret; - msgpack_object val; - msgpack_object *out_key; - msgpack_object *out_val; - struct flb_ra_value *result; - - /* Get the key position in the map */ - i = ra_key_val_id(ckey, map); - if (i == -1) { - return NULL; - } - - /* Reference entries */ - val = map.via.map.ptr[i].val; - - /* Create the result context */ - result = flb_calloc(1, sizeof(struct flb_ra_value)); - if (!result) { - flb_errno(); - return NULL; - } - result->o = val; - - if ((val.type == MSGPACK_OBJECT_MAP || val.type == MSGPACK_OBJECT_ARRAY) - && subkeys != NULL && mk_list_size(subkeys) > 0) { - - ret = subkey_to_object(&val, subkeys, &out_key, &out_val); - if (ret == 0) { - ret = msgpack_object_to_ra_value(*out_val, result); - if (ret == -1) { - flb_free(result); - return NULL; - } - return result; - } - else { - flb_free(result); - return NULL; - } - } - else { - ret = msgpack_object_to_ra_value(val, result); - if (ret == -1) { - flb_error("[ra key] cannot process key value"); - flb_free(result); - return NULL; - } - } - - return result; -} - -int flb_ra_key_value_get(flb_sds_t ckey, msgpack_object map, - struct mk_list *subkeys, - msgpack_object **start_key, - msgpack_object **out_key, msgpack_object **out_val) -{ - int i; - int ret; - msgpack_object val; - msgpack_object *o_key; - msgpack_object *o_val; - - /* Get the key position in the map */ - i = ra_key_val_id(ckey, map); - if (i == -1) { - return -1; - } - - /* Reference entries */ - *start_key = &map.via.map.ptr[i].key; - val = map.via.map.ptr[i].val; - - if ((val.type == MSGPACK_OBJECT_MAP || val.type == MSGPACK_OBJECT_ARRAY) - && subkeys != NULL && mk_list_size(subkeys) > 0) { - ret = subkey_to_object(&val, subkeys, &o_key, &o_val); - if (ret == 0) { - *out_key = o_key; - *out_val = o_val; - return 0; - } - } - else { - *out_key = &map.via.map.ptr[i].key; - *out_val = &map.via.map.ptr[i].val; - return 0; - } - - return -1; -} - -int flb_ra_key_strcmp(flb_sds_t ckey, msgpack_object map, - struct mk_list *subkeys, char *str, int len) -{ - int i; - int ret; - msgpack_object val; - msgpack_object *out_key; - msgpack_object *out_val; - - /* Get the key position in the map */ - i = ra_key_val_id(ckey, map); - if (i == -1) { - return -1; - } - - /* Reference map value */ - val = map.via.map.ptr[i].val; - - if ((val.type == MSGPACK_OBJECT_MAP || val.type == MSGPACK_OBJECT_ARRAY) - && subkeys != NULL && mk_list_size(subkeys) > 0) { - ret = subkey_to_object(&val, subkeys, &out_key, &out_val); - if (ret == 0) { - return msgpack_object_strcmp(*out_val, str, len); - } - else { - return -1; - } - } - - return msgpack_object_strcmp(val, str, len); -} - -int flb_ra_key_regex_match(flb_sds_t ckey, msgpack_object map, - struct mk_list *subkeys, struct flb_regex *regex, - struct flb_regex_search *result) -{ - int i; - int ret; - msgpack_object val; - msgpack_object *out_key; - msgpack_object *out_val; - - /* Get the key position in the map */ - i = ra_key_val_id(ckey, map); - if (i == -1) { - return -1; - } - - /* Reference map value */ - val = map.via.map.ptr[i].val; - - if ((val.type == MSGPACK_OBJECT_MAP || val.type == MSGPACK_OBJECT_ARRAY) - && subkeys != NULL && mk_list_size(subkeys) > 0) { - ret = subkey_to_object(&val, subkeys, &out_key, &out_val); - if (ret == 0) { - if (out_val->type != MSGPACK_OBJECT_STR) { - return -1; - } - - if (result) { - /* Regex + capture mode */ - return flb_regex_do(regex, - (char *) out_val->via.str.ptr, - out_val->via.str.size, - result); - } - else { - /* No capture */ - return flb_regex_match(regex, - (unsigned char *) out_val->via.str.ptr, - out_val->via.str.size); - } - } - return -1; - } - - if (val.type != MSGPACK_OBJECT_STR) { - return -1; - } - - if (result) { - /* Regex + capture mode */ - return flb_regex_do(regex, (char *) val.via.str.ptr, val.via.str.size, - result); - } - else { - /* No capture */ - return flb_regex_match(regex, (unsigned char *) val.via.str.ptr, - val.via.str.size); - } - - return -1; -} - -static int update_subkey(msgpack_object *obj, struct mk_list *subkeys, - int levels, int *matched, - msgpack_object *in_key, msgpack_object *in_val, - msgpack_packer *mp_pck); - - -static int update_subkey_array(msgpack_object *obj, struct mk_list *subkeys, - int levels, int *matched, - msgpack_object *in_key, msgpack_object *in_val, - msgpack_packer *mp_pck) -{ - struct flb_ra_subentry *entry; - int i; - int ret; - int size; - - entry = mk_list_entry_first(subkeys, struct flb_ra_subentry, _head); - - /* check the current msgpack object is an array */ - if (obj->type != MSGPACK_OBJECT_ARRAY) { - flb_error("%s: object is not array", __FUNCTION__); - return -1; - } - size = obj->via.array.size; - /* Index limit and ensure no overflow */ - if (entry->array_id == INT_MAX || - size < entry->array_id + 1) { - flb_trace("%s: out of index", __FUNCTION__); - return -1; - } - - msgpack_pack_array(mp_pck, size); - for (i=0; i<size; i++) { - if (i != entry->array_id) { - msgpack_pack_object(mp_pck, obj->via.array.ptr[i]); - continue; - } - *matched += 1; - if (levels == *matched) { - flb_trace("%s: update val matched=%d", __FUNCTION__, *matched); - /* update value */ - msgpack_pack_object(mp_pck, *in_val); - continue; - } - - if (subkeys->next == NULL) { - flb_trace("%s: end of subkey", __FUNCTION__); - return -1; - } - ret = update_subkey(&obj->via.array.ptr[i], subkeys->next, - levels, matched, - in_key, in_val, mp_pck); - if (ret < 0) { - return -1; - } - } - return 0; -} - -static int update_subkey_map(msgpack_object *obj, struct mk_list *subkeys, - int levels, int *matched, - msgpack_object *in_key, msgpack_object *in_val, - msgpack_packer *mp_pck) -{ - struct flb_ra_subentry *entry; - int i; - int ret_id; - int size; - int ret; - msgpack_object_kv kv; - - entry = mk_list_entry_first(subkeys, struct flb_ra_subentry, _head); - /* check the current msgpack object is a map */ - if (obj->type != MSGPACK_OBJECT_MAP) { - flb_trace("%s: object is not map", __FUNCTION__); - return -1; - } - size = obj->via.map.size; - - ret_id = ra_key_val_id(entry->str, *obj); - if (ret_id < 0) { - flb_trace("%s: not found", __FUNCTION__); - return -1; - } - - msgpack_pack_map(mp_pck, size); - for (i=0; i<size; i++) { - if (i != ret_id) { - msgpack_pack_object(mp_pck, obj->via.map.ptr[i].key); - msgpack_pack_object(mp_pck, obj->via.map.ptr[i].val); - continue; - } - *matched += 1; - if (levels == *matched) { - flb_trace("%s update key/val matched=%d", __FUNCTION__, *matched); - /* update key/value */ - kv = obj->via.map.ptr[i]; - if (in_key != NULL) { - kv.key = *in_key; - } - msgpack_pack_object(mp_pck, kv.key); - if (in_val != NULL) { - kv.val = *in_val; - } - msgpack_pack_object(mp_pck, kv.val); - - continue; - } - if (subkeys->next == NULL) { - flb_trace("%s: end of subkey", __FUNCTION__); - return -1; - } - msgpack_pack_object(mp_pck, obj->via.map.ptr[i].key); - ret = update_subkey(&(obj->via.map.ptr[i].val), subkeys->next, - levels, matched, - in_key, in_val, mp_pck); - if (ret < 0) { - return -1; - } - } - return 0; -} - -static int update_subkey(msgpack_object *obj, struct mk_list *subkeys, - int levels, int *matched, - msgpack_object *in_key, msgpack_object *in_val, - msgpack_packer *mp_pck) -{ - struct flb_ra_subentry *entry; - - entry = mk_list_entry_first(subkeys, struct flb_ra_subentry, _head); - - if (entry->type == FLB_RA_PARSER_ARRAY_ID) { - return update_subkey_array(obj, subkeys, - levels, matched, - in_key, in_val, mp_pck); - } - return update_subkey_map(obj, subkeys, levels, matched, in_key, in_val, mp_pck); -} - -int flb_ra_key_value_update(struct flb_ra_parser *rp, msgpack_object map, - msgpack_object *in_key, msgpack_object *in_val, - msgpack_packer *mp_pck) -{ - int kv_id; - int i; - int map_size; - int ret; - int levels; - int matched = 0; - - /* Get the key position in the map */ - kv_id = ra_key_val_id(rp->key->name, map); - if (kv_id == -1) { - return -1; - } - - levels = mk_list_size(rp->key->subkeys); - - map_size = map.via.map.size; - - msgpack_pack_map(mp_pck, map_size); - if (levels == 0) { - /* no subkeys */ - for (i=0; i<map_size; i++) { - if (i != kv_id) { - /* pack original key/val */ - msgpack_pack_object(mp_pck, map.via.map.ptr[i].key); - msgpack_pack_object(mp_pck, map.via.map.ptr[i].val); - continue; - } - - /* update key/val */ - if (in_key != NULL) { - msgpack_pack_object(mp_pck, *in_key); - } - else { - msgpack_pack_object(mp_pck, map.via.map.ptr[i].key); - } - if (in_val != NULL) { - msgpack_pack_object(mp_pck, *in_val); - } - else { - msgpack_pack_object(mp_pck, map.via.map.ptr[i].val); - } - } - return 0; - } - - for (i=0; i<map_size; i++) { - msgpack_pack_object(mp_pck, map.via.map.ptr[i].key); - if (i != kv_id) { - msgpack_pack_object(mp_pck, map.via.map.ptr[i].val); - continue; - } - ret = update_subkey(&(map.via.map.ptr[i].val), rp->key->subkeys, - levels, &matched, - in_key, in_val, mp_pck); - if (ret < 0) { - return -1; - } - } - - return 0; -} - -static int append_subkey(msgpack_object *obj, struct mk_list *subkeys, - int levels, int *matched, - msgpack_object *in_val, - msgpack_packer *mp_pck); - - -static int append_subkey_array(msgpack_object *obj, struct mk_list *subkeys, - int levels, int *matched, - msgpack_object *in_val, - msgpack_packer *mp_pck) -{ - struct flb_ra_subentry *entry; - int i; - int ret; - int size; - - /* check the current msgpack object is an array */ - if (obj->type != MSGPACK_OBJECT_ARRAY) { - flb_trace("%s: object is not array", __FUNCTION__); - return -1; - } - size = obj->via.array.size; - entry = mk_list_entry_first(subkeys, struct flb_ra_subentry, _head); - - if (levels == *matched) { - /* append val */ - msgpack_pack_array(mp_pck, size+1); - for (i=0; i<size; i++) { - msgpack_pack_object(mp_pck, obj->via.array.ptr[i]); - } - msgpack_pack_object(mp_pck, *in_val); - - *matched = -1; - return 0; - } - - /* Index limit and ensure no overflow */ - if (entry->array_id == INT_MAX || - size < entry->array_id + 1) { - flb_trace("%s: out of index", __FUNCTION__); - return -1; - } - - msgpack_pack_array(mp_pck, size); - for (i=0; i<size; i++) { - if (i != entry->array_id) { - msgpack_pack_object(mp_pck, obj->via.array.ptr[i]); - continue; - } - if (*matched >= 0) { - *matched += 1; - } - if (subkeys->next == NULL) { - flb_trace("%s: end of subkey", __FUNCTION__); - return -1; - } - ret = append_subkey(&obj->via.array.ptr[i], subkeys->next, - levels, matched, - in_val, mp_pck); - if (ret < 0) { - return -1; - } - } - return 0; -} - -static int append_subkey_map(msgpack_object *obj, struct mk_list *subkeys, - int levels, int *matched, - msgpack_object *in_val, - msgpack_packer *mp_pck) -{ - struct flb_ra_subentry *entry; - int i; - int ret_id; - int size; - int ret; - - /* check the current msgpack object is a map */ - if (obj->type != MSGPACK_OBJECT_MAP) { - flb_trace("%s: object is not map", __FUNCTION__); - return -1; - } - size = obj->via.map.size; - entry = mk_list_entry_first(subkeys, struct flb_ra_subentry, _head); - - if (levels == *matched) { - /* append val */ - msgpack_pack_map(mp_pck, size+1); - for (i=0; i<size; i++) { - msgpack_pack_object(mp_pck, obj->via.map.ptr[i].key); - msgpack_pack_object(mp_pck, obj->via.map.ptr[i].val); - } - msgpack_pack_str(mp_pck, flb_sds_len(entry->str)); - msgpack_pack_str_body(mp_pck, entry->str, flb_sds_len(entry->str)); - msgpack_pack_object(mp_pck, *in_val); - - *matched = -1; - return 0; - } - - - ret_id = ra_key_val_id(entry->str, *obj); - if (ret_id < 0) { - flb_trace("%s: not found", __FUNCTION__); - return -1; - } - - msgpack_pack_map(mp_pck, size); - for (i=0; i<size; i++) { - if (i != ret_id) { - msgpack_pack_object(mp_pck, obj->via.map.ptr[i].key); - msgpack_pack_object(mp_pck, obj->via.map.ptr[i].val); - continue; - } - - if (*matched >= 0) { - *matched += 1; - } - if (subkeys->next == NULL) { - flb_trace("%s: end of subkey", __FUNCTION__); - return -1; - } - msgpack_pack_object(mp_pck, obj->via.map.ptr[i].key); - ret = append_subkey(&(obj->via.map.ptr[i].val), subkeys->next, - levels, matched, - in_val, mp_pck); - if (ret < 0) { - return -1; - } - } - return 0; -} - -static int append_subkey(msgpack_object *obj, struct mk_list *subkeys, - int levels, int *matched, - msgpack_object *in_val, - msgpack_packer *mp_pck) -{ - struct flb_ra_subentry *entry; - - entry = mk_list_entry_first(subkeys, struct flb_ra_subentry, _head); - - if (entry->type == FLB_RA_PARSER_ARRAY_ID) { - return append_subkey_array(obj, subkeys, - levels, matched, - in_val, mp_pck); - } - return append_subkey_map(obj, subkeys, levels, matched, in_val, mp_pck); -} - -int flb_ra_key_value_append(struct flb_ra_parser *rp, msgpack_object map, - msgpack_object *in_val, msgpack_packer *mp_pck) -{ - int ref_level; - int map_size; - int i; - int kv_id; - int ret; - int matched = 0; - - map_size = map.via.map.size; - - /* Decrement since the last key doesn't exist */ - ref_level = mk_list_size(rp->key->subkeys) - 1; - if (ref_level < 0) { - /* no subkeys */ - msgpack_pack_map(mp_pck, map_size+1); - for (i=0; i<map_size; i++) { - msgpack_pack_object(mp_pck, map.via.map.ptr[i].key); - msgpack_pack_object(mp_pck, map.via.map.ptr[i].val); - } - msgpack_pack_str(mp_pck, flb_sds_len(rp->key->name)); - msgpack_pack_str_body(mp_pck, rp->key->name, flb_sds_len(rp->key->name)); - msgpack_pack_object(mp_pck, *in_val); - return 0; - } - - /* Get the key position in the map */ - kv_id = ra_key_val_id(rp->key->name, map); - if (kv_id == -1) { - return -1; - } - - msgpack_pack_map(mp_pck, map_size); - for (i=0; i<map_size; i++) { - msgpack_pack_object(mp_pck, map.via.map.ptr[i].key); - if (i != kv_id) { - msgpack_pack_object(mp_pck, map.via.map.ptr[i].val); - continue; - } - ret = append_subkey(&(map.via.map.ptr[i].val), rp->key->subkeys, - ref_level, &matched, - in_val, mp_pck); - if (ret < 0) { - return -1; - } - } - - return 0; -} - -void flb_ra_key_value_destroy(struct flb_ra_value *v) -{ - if (v->type == FLB_RA_STRING) { - flb_sds_destroy(v->val.string); - } - flb_free(v); -} diff --git a/fluent-bit/src/flb_random.c b/fluent-bit/src/flb_random.c deleted file mode 100644 index 2425ec258..000000000 --- a/fluent-bit/src/flb_random.c +++ /dev/null @@ -1,97 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <fluent-bit/flb_compat.h> -#include <fcntl.h> - -#ifdef FLB_HAVE_GETENTROPY -#include <unistd.h> -#endif -#ifdef FLB_HAVE_GETENTROPY_SYS_RANDOM -#include <sys/random.h> -#endif - -#define MAX_GETENTROPY_LEN 256 - -/* - * This module provides a random number generator for common use cases. - * - * On Windows, we use BCryptGenRandom() from CNG API. This function - * is available since Windows Vista, and should be compliant to the - * official recommendation. - * - * On other platforms, we use getentropy(3) if available, otherwise - * /dev/urandom as a secure random source. - */ - -int flb_random_bytes(unsigned char *buf, int len) -{ -#ifdef FLB_SYSTEM_WINDOWS - NTSTATUS ret; - ret = BCryptGenRandom(NULL, buf, len, BCRYPT_USE_SYSTEM_PREFERRED_RNG); - if (!BCRYPT_SUCCESS(ret)) { - return -1; - } - return 0; -#else - int fd; - ssize_t bytes; - -#if defined(FLB_HAVE_GETENTROPY) || defined(FLB_HAVE_GETENTROPY_SYS_RANDOM) - while (len > 0) { - if (len > MAX_GETENTROPY_LEN) { - bytes = MAX_GETENTROPY_LEN; - } - else { - bytes = len; - } - if (getentropy(buf, bytes) < 0) { -#ifdef ENOSYS - /* Fall back to urandom if the syscall is not available (Linux only) */ - if (errno == ENOSYS) { - goto try_urandom; - } -#endif - return -1; - } - len -= bytes; - buf += bytes; - } - return 0; - -try_urandom: -#endif /* FLB_HAVE_GETENTROPY || FLB_HAVE_GETENTROPY_SYS_RANDOM */ - fd = open("/dev/urandom", O_RDONLY); - if (fd == -1) { - return -1; - } - - while (len > 0) { - bytes = read(fd, buf, len); - if (bytes <= 0) { - close(fd); - return -1; - } - len -= bytes; - buf += bytes; - } - close(fd); - return 0; -#endif /* FLB_SYSTEM_WINDOWS */ -} diff --git a/fluent-bit/src/flb_record_accessor.c b/fluent-bit/src/flb_record_accessor.c deleted file mode 100644 index d84f7ea64..000000000 --- a/fluent-bit/src/flb_record_accessor.c +++ /dev/null @@ -1,906 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <fluent-bit/flb_info.h> -#include <fluent-bit/flb_mem.h> -#include <fluent-bit/flb_env.h> -#include <fluent-bit/flb_log.h> -#include <fluent-bit/flb_pack.h> -#include <fluent-bit/flb_sds.h> -#include <fluent-bit/flb_sds_list.h> -#include <fluent-bit/flb_record_accessor.h> -#include <fluent-bit/flb_ra_key.h> -#include <fluent-bit/record_accessor/flb_ra_parser.h> -#include <monkey/mk_core.h> -#include <msgpack.h> - -#include <ctype.h> - -static struct flb_ra_parser *ra_parse_string(struct flb_record_accessor *ra, - flb_sds_t buf, int start, int end) -{ - int len; - struct flb_ra_parser *rp; - - len = end - start; - rp = flb_ra_parser_string_create(buf + start, len); - if (!rp) { - return NULL; - } - - return rp; -} - -/* Create a parser context for a key map or function definition */ -static struct flb_ra_parser *ra_parse_meta(struct flb_record_accessor *ra, - flb_sds_t buf, int start, int end) -{ - int len; - struct flb_ra_parser *rp; - - len = end - start; - rp = flb_ra_parser_meta_create(buf + start, len); - if (!rp) { - return NULL; - } - - return rp; -} - -/* - * Supported data - * - * ${X} => environment variable - * $key, $key['x'], $key['x'][N]['z'] => record key value or array index - * $0, $1,..$9 => regex id - * $X() => built-in function - */ -static int ra_parse_buffer(struct flb_record_accessor *ra, flb_sds_t buf) -{ - int i; - int n; - int c; - int t; - int len; - int pre = 0; - int end = 0; - int quote_cnt; - struct flb_ra_parser *rp; - struct flb_ra_parser *rp_str = NULL; - - len = flb_sds_len(buf); - - for (i = 0; i < len; i++) { - if (buf[i] != '$') { - continue; - } - - /* - * Before to add the number entry, add the previous text - * before hitting this. - */ - if (i > pre) { - rp = ra_parse_string(ra, buf, pre, i); - if (!rp) { - return -1; - } - mk_list_add(&rp->_head, &ra->list); - } - pre = i; - - - n = i + 1; - if (n >= len) { - /* Finalize, nothing to do */ - break; - } - - /* - * If the next character is a digit like $0,$1,$2..$9, means the user wants to use - * the result of a regex capture. - * - * We support up to 10 regex ids [0-9] - */ - if (isdigit(buf[n])) { - /* Add REGEX_ID entry */ - c = atoi(buf + n); - rp = flb_ra_parser_regex_id_create(c); - if (!rp) { - return -1; - } - - mk_list_add(&rp->_head, &ra->list); - i++; - pre = i + 1; - continue; - } - - /* - * If the next 3 character are 'TAG', the user might want to include the tag or - * part of it (e.g: TAG[n]). - */ - if (n + 2 < len && strncmp(buf + n, "TAG", 3) == 0) { - /* Check if some [] was added */ - if (n + 4 < len) { - end = -1; - if (buf[n + 3] == '[') { - t = n + 3; - - /* Look for the ending ']' */ - end = mk_string_char_search(buf + t, ']', len - t); - if (end == 0) { - end = -1; - } - - /* continue processsing */ - c = atoi(buf + t + 1); - - rp = flb_ra_parser_tag_part_create(c); - if (!rp) { - return -1; - } - mk_list_add(&rp->_head, &ra->list); - - i = t + end + 1; - pre = i; - continue; - } - } - - /* Append full tag */ - rp = flb_ra_parser_tag_create(); - if (!rp) { - return -1; - } - mk_list_add(&rp->_head, &ra->list); - i = n + 3; - pre = n + 3; - continue; - } - - quote_cnt = 0; - for (end = i + 1; end < len; end++) { - if (buf[end] == '\'') { - ++quote_cnt; - } - else if (buf[end] == '.' && (quote_cnt & 0x01)) { - /* ignore '.' if it is inside a string/subkey */ - continue; - } - else if (buf[end] == '.' || buf[end] == ' ' || buf[end] == ',' || buf[end] == '"') { - break; - } - } - if (end > len) { - end = len; - } - - /* Parse the content, we use 'end' as the separator position */ - rp = ra_parse_meta(ra, buf, i, end); - if (!rp) { - return -1; - } - - /* Generate fixed length string */ - if (pre < i) { - rp_str = ra_parse_string(ra, buf, pre, i); - if (!rp_str) { - flb_ra_parser_destroy(rp); - return -1; - } - } - else { - rp_str = NULL; - } - - if (rp_str) { - mk_list_add(&rp_str->_head, &ra->list); - } - mk_list_add(&rp->_head, &ra->list); - pre = end; - i = end; - } - - /* Append remaining string */ - if ((i - 1 > end && pre < i) || i == 1 /*allow single character*/) { - end = flb_sds_len(buf); - rp_str = ra_parse_string(ra, buf, pre, end); - if (rp_str) { - mk_list_add(&rp_str->_head, &ra->list); - } - } - - return 0; -} - -void flb_ra_destroy(struct flb_record_accessor *ra) -{ - struct mk_list *tmp; - struct mk_list *head; - struct flb_ra_parser *rp; - - mk_list_foreach_safe(head, tmp, &ra->list) { - rp = mk_list_entry(head, struct flb_ra_parser, _head); - mk_list_del(&rp->_head); - flb_ra_parser_destroy(rp); - } - - if (ra->pattern) { - flb_sds_destroy(ra->pattern); - } - flb_free(ra); -} - -int flb_ra_subkey_count(struct flb_record_accessor *ra) -{ - struct mk_list *head; - struct flb_ra_parser *rp; - int ret = -1; - int tmp; - - if (ra == NULL) { - return -1; - } - mk_list_foreach(head, &ra->list) { - rp = mk_list_entry(head, struct flb_ra_parser, _head); - tmp = flb_ra_parser_subkey_count(rp); - if (tmp > ret) { - ret = tmp; - } - } - - return ret; -} - -struct flb_record_accessor *flb_ra_create(char *str, int translate_env) -{ - int ret; - size_t hint = 0; - char *p; - flb_sds_t buf = NULL; - struct flb_env *env; - struct mk_list *head; - struct flb_ra_parser *rp; - struct flb_record_accessor *ra; - - p = str; - if (translate_env == FLB_TRUE) { - /* - * Check if some environment variable has been created as part of the - * string. Upon running the environment variable will be pre-set in the - * string. - */ - env = flb_env_create(); - if (!env) { - flb_error("[record accessor] cannot create environment context"); - return NULL; - } - - /* Translate string */ - buf = flb_env_var_translate(env, str); - if (!buf) { - flb_error("[record accessor] cannot translate string"); - flb_env_destroy(env); - return NULL; - } - flb_env_destroy(env); - p = buf; - } - - /* Allocate context */ - ra = flb_calloc(1, sizeof(struct flb_record_accessor)); - if (!ra) { - flb_errno(); - flb_error("[record accessor] cannot create context"); - if (buf) { - flb_sds_destroy(buf); - } - return NULL; - } - ra->pattern = flb_sds_create(str); - if (!ra->pattern) { - flb_error("[record accessor] could not allocate pattern"); - flb_free(ra); - if (buf) { - flb_sds_destroy(buf); - } - return NULL; - } - - mk_list_init(&ra->list); - - /* - * The buffer needs to processed where we create a list of parts, basically - * a linked list of sds using 'slist' api. - */ - ret = ra_parse_buffer(ra, p); - if (buf) { - flb_sds_destroy(buf); - } - if (ret == -1) { - flb_ra_destroy(ra); - return NULL; - } - - /* Calculate a hint of an outgoing size buffer */ - mk_list_foreach(head, &ra->list) { - rp = mk_list_entry(head, struct flb_ra_parser, _head); - if (rp->key) { - if (rp->type == FLB_RA_PARSER_REGEX_ID) { - hint += 32; - } - else { - hint += flb_sds_len(rp->key->name); - } - } - } - ra->size_hint = hint + 128; - return ra; -} - -/* - flb_ra_create_str_from_list returns record accessor string from string list. - e.g. {"aa", "bb", "cc", NULL} -> "$aa['bb']['cc']" - Return value should be freed using flb_sds_destroy after using. - */ -flb_sds_t flb_ra_create_str_from_list(struct flb_sds_list *str_list) -{ - int i = 0; - int ret_i = 0; - int offset = 0; - - char *fmt = NULL; - char **strs = NULL; - flb_sds_t str; - flb_sds_t tmp_sds; - - if (str_list == NULL || flb_sds_list_size(str_list) == 0) { - return NULL; - } - - str = flb_sds_create_size(256); - if (str == NULL) { - flb_errno(); - return NULL; - } - - strs = flb_sds_list_create_str_array(str_list); - if (strs == NULL) { - flb_error("%s flb_sds_list_create_str_array failed", __FUNCTION__); - return NULL; - } - - while(strs[i] != NULL) { - if (i == 0) { - fmt = "$%s"; - } - else { - fmt = "['%s']"; - } - - ret_i = snprintf(str+offset, flb_sds_alloc(str)-offset-1, fmt, strs[i]); - if (ret_i > flb_sds_alloc(str)-offset-1) { - tmp_sds = flb_sds_increase(str, ret_i); - if (tmp_sds == NULL) { - flb_errno(); - flb_sds_list_destroy_str_array(strs); - flb_sds_destroy(str); - return NULL; - } - str = tmp_sds; - ret_i = snprintf(str+offset, flb_sds_alloc(str)-offset-1, fmt, strs[i]); - if (ret_i > flb_sds_alloc(str)-offset-1) { - flb_errno(); - flb_sds_list_destroy_str_array(strs); - flb_sds_destroy(str); - return NULL; - } - } - offset += ret_i; - i++; - } - flb_sds_list_destroy_str_array(strs); - - return str; -} - -struct flb_record_accessor *flb_ra_create_from_list(struct flb_sds_list *str_list, int translate_env) -{ - flb_sds_t tmp = NULL; - struct flb_record_accessor *ret = NULL; - - tmp = flb_ra_create_str_from_list(str_list); - if (tmp == NULL) { - flb_errno(); - return NULL; - } - - ret = flb_ra_create(tmp, translate_env); - flb_sds_destroy(tmp); - - return ret; -} - -void flb_ra_dump(struct flb_record_accessor *ra) -{ - struct mk_list *head; - struct flb_ra_parser *rp; - - mk_list_foreach(head, &ra->list) { - rp = mk_list_entry(head, struct flb_ra_parser, _head); - printf("\n"); - flb_ra_parser_dump(rp); - } -} - -static flb_sds_t ra_translate_regex_id(struct flb_ra_parser *rp, - struct flb_regex_search *result, - flb_sds_t buf) -{ - int ret; - ptrdiff_t start; - ptrdiff_t end; - flb_sds_t tmp; - - ret = flb_regex_results_get(result, rp->id, &start, &end); - if (ret == -1) { - return buf; - } - - tmp = flb_sds_cat(buf, result->str + start, end - start); - return tmp; -} - -static flb_sds_t ra_translate_tag(struct flb_ra_parser *rp, flb_sds_t buf, - char *tag, int tag_len) -{ - flb_sds_t tmp; - - tmp = flb_sds_cat(buf, tag, tag_len); - return tmp; -} - -static flb_sds_t ra_translate_tag_part(struct flb_ra_parser *rp, flb_sds_t buf, - char *tag, int tag_len) -{ - int i = 0; - int id = -1; - int end; - flb_sds_t tmp = buf; - - while (i < tag_len) { - end = mk_string_char_search(tag + i, '.', tag_len - i); - if (end == -1) { - if (i == 0) { - break; - } - end = tag_len - i; - } - id++; - if (rp->id == id) { - tmp = flb_sds_cat(buf, tag + i, end); - break; - } - - i += end + 1; - } - - /* No dots in the tag */ - if (rp->id == 0 && id == -1 && i < tag_len) { - tmp = flb_sds_cat(buf, tag, tag_len); - return tmp; - } - - return tmp; -} - -static flb_sds_t ra_translate_string(struct flb_ra_parser *rp, flb_sds_t buf) -{ - flb_sds_t tmp; - - tmp = flb_sds_cat(buf, rp->key->name, flb_sds_len(rp->key->name)); - return tmp; -} - -static flb_sds_t ra_translate_keymap(struct flb_ra_parser *rp, flb_sds_t buf, - msgpack_object map, int *found) -{ - int len; - char *js; - char str[32]; - flb_sds_t tmp = NULL; - struct flb_ra_value *v; - - /* Lookup key or subkey value */ - if (rp->key == NULL) { - *found = FLB_FALSE; - return buf; - } - - v = flb_ra_key_to_value(rp->key->name, map, rp->key->subkeys); - if (!v) { - *found = FLB_FALSE; - return buf; - } - else { - *found = FLB_TRUE; - } - - /* Based on data type, convert to it string representation */ - if (v->type == FLB_RA_BOOL) { - /* Check if is a map or a real bool */ - if (v->o.type == MSGPACK_OBJECT_MAP) { - /* Convert msgpack map to JSON string */ - js = flb_msgpack_to_json_str(1024, &v->o); - if (js) { - len = strlen(js); - tmp = flb_sds_cat(buf, js, len); - flb_free(js); - } - } - else if (v->o.type == MSGPACK_OBJECT_BOOLEAN) { - if (v->val.boolean) { - tmp = flb_sds_cat(buf, "true", 4); - } - else { - tmp = flb_sds_cat(buf, "false", 5); - } - } - } - else if (v->type == FLB_RA_INT) { - len = snprintf(str, sizeof(str) - 1, "%" PRId64, v->val.i64); - tmp = flb_sds_cat(buf, str, len); - } - else if (v->type == FLB_RA_FLOAT) { - len = snprintf(str, sizeof(str) - 1, "%f", v->val.f64); - if (len >= sizeof(str)) { - tmp = flb_sds_cat(buf, str, sizeof(str)-1); - } - else { - tmp = flb_sds_cat(buf, str, len); - } - } - else if (v->type == FLB_RA_STRING) { - tmp = flb_sds_cat(buf, v->val.string, flb_sds_len(v->val.string)); - } - else if (v->type == FLB_RA_NULL) { - tmp = flb_sds_cat(buf, "null", 4); - } - - flb_ra_key_value_destroy(v); - return tmp; -} - -/* - * Translate a record accessor buffer, tag and records are optional - * parameters. - * - * For safety, the function returns a newly created string that needs - * to be destroyed by the caller. - */ -flb_sds_t flb_ra_translate(struct flb_record_accessor *ra, - char *tag, int tag_len, - msgpack_object map, struct flb_regex_search *result) -{ - return flb_ra_translate_check(ra, tag, tag_len, map, result, FLB_FALSE); -} - -/* - * Translate a record accessor buffer, tag and records are optional - * parameters. - * - * For safety, the function returns a newly created string that needs - * to be destroyed by the caller. - * - * Returns NULL if `check` is FLB_TRUE and any key lookup in the record failed - */ -flb_sds_t flb_ra_translate_check(struct flb_record_accessor *ra, - char *tag, int tag_len, - msgpack_object map, struct flb_regex_search *result, - int check) -{ - flb_sds_t tmp = NULL; - flb_sds_t buf; - struct mk_list *head; - struct flb_ra_parser *rp; - int found = FLB_FALSE; - - buf = flb_sds_create_size(ra->size_hint); - if (!buf) { - flb_error("[record accessor] cannot create outgoing buffer"); - return NULL; - } - - mk_list_foreach(head, &ra->list) { - rp = mk_list_entry(head, struct flb_ra_parser, _head); - if (rp->type == FLB_RA_PARSER_STRING) { - tmp = ra_translate_string(rp, buf); - } - else if (rp->type == FLB_RA_PARSER_KEYMAP) { - tmp = ra_translate_keymap(rp, buf, map, &found); - if (check == FLB_TRUE && found == FLB_FALSE) { - flb_warn("[record accessor] translation failed, root key=%s", rp->key->name); - flb_sds_destroy(buf); - return NULL; - } - } - else if (rp->type == FLB_RA_PARSER_REGEX_ID && result) { - tmp = ra_translate_regex_id(rp, result, buf); - } - else if (rp->type == FLB_RA_PARSER_TAG && tag) { - tmp = ra_translate_tag(rp, buf, tag, tag_len); - } - else if (rp->type == FLB_RA_PARSER_TAG_PART && tag) { - tmp = ra_translate_tag_part(rp, buf, tag, tag_len); - } - - //else if (rp->type == FLB_RA_PARSER_FUNC) { - //tmp = ra_translate_func(rp, buf, tag, tag_len); - //} - - if (!tmp) { - flb_error("[record accessor] translation failed"); - flb_sds_destroy(buf); - return NULL; - } - if (tmp != buf) { - buf = tmp; - } - } - - return buf; -} - -/* - * If the record accessor rules do not generate content based on a keymap or - * regex, it's considered to be 'static', so the value returned will always be - * the same. - * - * If the 'ra' is static, return FLB_TRUE, otherwise FLB_FALSE. - */ -int flb_ra_is_static(struct flb_record_accessor *ra) -{ - struct mk_list *head; - struct flb_ra_parser *rp; - - mk_list_foreach(head, &ra->list) { - rp = mk_list_entry(head, struct flb_ra_parser, _head); - if (rp->type == FLB_RA_PARSER_STRING) { - continue; - } - else if (rp->type == FLB_RA_PARSER_KEYMAP) { - return FLB_FALSE; - } - else if (rp->type == FLB_RA_PARSER_REGEX_ID) { - return FLB_FALSE; - } - else if (rp->type == FLB_RA_PARSER_TAG) { - continue; - } - else if (rp->type == FLB_RA_PARSER_TAG_PART) { - continue; - } - } - - return FLB_TRUE; -} - -/* - * Compare a string value against the first entry of a record accessor component, used - * specifically when the record accessor refers to a single key name. - */ -int flb_ra_strcmp(struct flb_record_accessor *ra, msgpack_object map, - char *str, int len) -{ - struct flb_ra_parser *rp; - - rp = mk_list_entry_first(&ra->list, struct flb_ra_parser, _head); - return flb_ra_key_strcmp(rp->key->name, map, rp->key->subkeys, - rp->key->name, flb_sds_len(rp->key->name)); -} - -/* - * Check if a regular expression matches a record accessor key in the - * given map - */ -int flb_ra_regex_match(struct flb_record_accessor *ra, msgpack_object map, - struct flb_regex *regex, struct flb_regex_search *result) -{ - struct flb_ra_parser *rp; - - rp = mk_list_entry_first(&ra->list, struct flb_ra_parser, _head); - if (rp == NULL || rp->key == NULL) { - return -1; - } - return flb_ra_key_regex_match(rp->key->name, map, rp->key->subkeys, - regex, result); -} - - -static struct flb_ra_parser* get_ra_parser(struct flb_record_accessor *ra) -{ - struct flb_ra_parser *rp = NULL; - - if (mk_list_size(&ra->list) == 0) { - return NULL; - } - rp = mk_list_entry_first(&ra->list, struct flb_ra_parser, _head); - if (!rp->key) { - return NULL; - } - return rp; -} - -/* - * If 'record accessor' pattern matches an entry in the 'map', set the - * reference in 'out_key' and 'out_val' for the entries in question. - * - * Returns FLB_TRUE if the pattern matched a kv pair, otherwise it returns - * FLB_FALSE. - */ -int flb_ra_get_kv_pair(struct flb_record_accessor *ra, msgpack_object map, - msgpack_object **start_key, - msgpack_object **out_key, msgpack_object **out_val) -{ - struct flb_ra_parser *rp; - - rp = get_ra_parser(ra); - if (rp == NULL) { - return FLB_FALSE; - } - - return flb_ra_key_value_get(rp->key->name, map, rp->key->subkeys, - start_key, out_key, out_val); -} - -struct flb_ra_value *flb_ra_get_value_object(struct flb_record_accessor *ra, - msgpack_object map) -{ - struct flb_ra_parser *rp; - - rp = get_ra_parser(ra); - if (rp == NULL) { - return NULL; - } - - return flb_ra_key_to_value(rp->key->name, map, rp->key->subkeys); -} - -/** - * Update key and/or value of the map using record accessor. - * - * @param ra the record accessor to specify key/value pair - * @param map the original map. - * @param in_key the pointer to overwrite key. If NULL, key will not be updated. - * @param in_val the pointer to overwrite val. If NULL, val will not be updated. - * @param out_map the updated map. If the API fails, out_map will be NULL. - * - * @return result of the API. 0:success, -1:fail - */ - -int flb_ra_update_kv_pair(struct flb_record_accessor *ra, msgpack_object map, - void **out_map, size_t *out_size, - msgpack_object *in_key, msgpack_object *in_val) -{ - struct flb_ra_parser *rp; - msgpack_packer mp_pck; - msgpack_sbuffer mp_sbuf; - int ret; - - msgpack_object *s_key; - msgpack_object *o_key; - msgpack_object *o_val; - - if (in_key == NULL && in_val == NULL) { - /* no key and value. nothing to do */ - flb_error("%s: no inputs", __FUNCTION__); - return -1; - } - else if (ra == NULL || out_map == NULL || out_size == NULL) { - /* invalid input */ - flb_error("%s: invalid input", __FUNCTION__); - return -1; - } - else if ( flb_ra_get_kv_pair(ra, map, &s_key, &o_key, &o_val) != 0) { - /* key and value are not found */ - flb_error("%s: no value", __FUNCTION__); - return -1; - } - - rp = get_ra_parser(ra); - if (rp == NULL) { - return -1; - } - - msgpack_sbuffer_init(&mp_sbuf); - msgpack_packer_init(&mp_pck, &mp_sbuf, msgpack_sbuffer_write); - - ret = flb_ra_key_value_update(rp, map, in_key, in_val, &mp_pck); - if (ret < 0) { - msgpack_sbuffer_destroy(&mp_sbuf); - return -1; - } - *out_map = mp_sbuf.data; - *out_size = mp_sbuf.size; - - return 0; -} - -/** - * Add key and/or value of the map using record accessor. - * If key already exists, the API fails. - * - * @param ra the record accessor to specify key. - * @param map the original map. - * @param in_val the pointer to add val. - * @param out_map the updated map. If the API fails, out_map will be NULL. - * - * @return result of the API. 0:success, -1:fail - */ - -int flb_ra_append_kv_pair(struct flb_record_accessor *ra, msgpack_object map, - void **out_map, size_t *out_size, - msgpack_object *in_val) -{ - struct flb_ra_parser *rp; - msgpack_packer mp_pck; - msgpack_sbuffer mp_sbuf; - int ret; - - msgpack_object *s_key = NULL; - msgpack_object *o_key = NULL; - msgpack_object *o_val = NULL; - - if (in_val == NULL) { - /* no key and value. nothing to do */ - flb_error("%s: no value", __FUNCTION__); - return -1; - } - else if (ra == NULL || out_map == NULL|| out_size == NULL) { - /* invalid input */ - flb_error("%s: invalid input", __FUNCTION__); - return -1; - } - - flb_ra_get_kv_pair(ra, map, &s_key, &o_key, &o_val); - if (o_key != NULL && o_val != NULL) { - /* key and value already exist */ - flb_error("%s: already exist", __FUNCTION__); - return -1; - } - - rp = get_ra_parser(ra); - if (rp == NULL || rp->key == NULL) { - return -1; - } - - msgpack_sbuffer_init(&mp_sbuf); - msgpack_packer_init(&mp_pck, &mp_sbuf, msgpack_sbuffer_write); - - ret = flb_ra_key_value_append(rp, map, in_val, &mp_pck); - if (ret < 0) { - msgpack_sbuffer_destroy(&mp_sbuf); - return -1; - } - - *out_map = mp_sbuf.data; - *out_size = mp_sbuf.size; - - return 0; -} diff --git a/fluent-bit/src/flb_regex.c b/fluent-bit/src/flb_regex.c deleted file mode 100644 index b2a1a5d57..000000000 --- a/fluent-bit/src/flb_regex.c +++ /dev/null @@ -1,319 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <fluent-bit/flb_info.h> -#include <fluent-bit/flb_regex.h> -#include <fluent-bit/flb_log.h> -#include <fluent-bit/flb_mem.h> - -#include <string.h> -#include <onigmo.h> - -static int -cb_onig_named(const UChar *name, const UChar *name_end, - int ngroup_num, int *group_nums, - regex_t *reg, void *data) -{ - int i; - int gn; - struct flb_regex_search *s; - OnigRegion *region; - - s = (struct flb_regex_search *) data; - region = s->region; - - for (i = 0; i < ngroup_num; i++) { - gn = group_nums[i]; - onig_name_to_backref_number(reg, name, name_end, region); - - if (s->cb_match) { - s->cb_match((const char *)name, - s->str + region->beg[gn], - region->end[gn] - region->beg[gn], - s->data); - } - - if (region->end[gn] >= 0) { - s->last_pos = region->end[gn]; - } - } - - return 0; -} - -static OnigOptionType check_option(const char *start, const char *end, char **new_end) -{ - char *chr = NULL; - OnigOptionType option = ONIG_OPTION_NONE; - - if (start == NULL || end == NULL || new_end == NULL) { - return ONIG_OPTION_DEFAULT; - } else if (start[0] != '/') { - *new_end = NULL; - return ONIG_OPTION_DEFAULT; - } - - chr = strrchr(start, '/'); - if (chr == start || chr == end) { - *new_end = NULL; - return ONIG_OPTION_DEFAULT; - } - *new_end = chr; - - chr++; - while(chr != end && *chr != '\0') { - switch (*chr) { - case 'm': - option |= ONIG_OPTION_MULTILINE; - break; - case 'i': - option |= ONIG_OPTION_IGNORECASE; - break; - case 'o': - flb_debug("[regex:%s]: 'o' option is not supported.", __FUNCTION__); - break; - case 'x': - option |= ONIG_OPTION_EXTEND; - break; - default: - flb_debug("[regex:%s]: unknown option. use default.", __FUNCTION__); - *new_end = NULL; - return ONIG_OPTION_DEFAULT; - } - chr++; - } - - if (option == ONIG_OPTION_NONE) { - *new_end = NULL; - option = ONIG_OPTION_DEFAULT; - } - - return option; -} - -static int str_to_regex(const char *pattern, OnigRegex *reg) -{ - int ret; - size_t len; - const char *start; - const char *end; - char *new_end = NULL; - OnigErrorInfo einfo; - OnigOptionType option; - - len = strlen(pattern); - start = pattern; - end = pattern + len; - - option = check_option(start, end, &new_end); - - if (pattern[0] == '/' && pattern[len - 1] == '/') { - start++; - end--; - } - - if (new_end != NULL) { - /* pattern is /pat/option. new_end indicates a last '/'. */ - start++; - end = new_end; - } - - ret = onig_new(reg, - (const unsigned char *)start, (const unsigned char *)end, - option, - ONIG_ENCODING_UTF8, ONIG_SYNTAX_RUBY, &einfo); - - if (ret != ONIG_NORMAL) { - return -1; - } - return 0; -} - -/* Initialize backend library */ -int flb_regex_init() -{ - return onig_init(); -} - -struct flb_regex *flb_regex_create(const char *pattern) -{ - int ret; - struct flb_regex *r; - - /* Create context */ - r = flb_malloc(sizeof(struct flb_regex)); - if (!r) { - flb_errno(); - return NULL; - } - - /* Compile pattern */ - ret = str_to_regex(pattern, (OnigRegex*)&r->regex); - if (ret == -1) { - flb_free(r); - return NULL; - } - - return r; -} - -ssize_t flb_regex_do(struct flb_regex *r, const char *str, size_t slen, - struct flb_regex_search *result) -{ - int ret; - const char *start; - const char *end; - const char *range; - OnigRegion *region; - - region = onig_region_new(); - if (!region) { - flb_errno(); - result->region = NULL; - return -1; - } - - /* Search scope */ - start = str; - end = start + slen; - range = end; - - ret = onig_search(r->regex, - (const unsigned char *)str, - (const unsigned char *)end, - (const unsigned char *)start, - (const unsigned char *)range, - region, ONIG_OPTION_NONE); - if (ret == ONIG_MISMATCH) { - result->region = NULL; - onig_region_free(region, 1); - return -1; - } - else if (ret < 0) { - result->region = NULL; - onig_region_free(region, 1); - return -1; - } - - result->region = region; - result->str = str; - - ret = region->num_regs - 1; - - if (ret == 0) { - result->region = NULL; - onig_region_free(region, 1); - } - - return ret; -} - -int flb_regex_results_get(struct flb_regex_search *result, int i, - ptrdiff_t *start, ptrdiff_t *end) -{ - OnigRegion *region; - - region = (OnigRegion *) result->region; - if (!region) { - return -1; - } - - if (i >= region->num_regs) { - return -1; - } - - *start = region->beg[i]; - *end = region->end[i]; - - return 0; -} - -void flb_regex_results_release(struct flb_regex_search *result) -{ - onig_region_free(result->region, 1); -} - -int flb_regex_results_size(struct flb_regex_search *result) -{ - OnigRegion *region; - - region = (OnigRegion *) result->region; - if (!region) { - return -1; - } - - return region->num_regs; -} - -int flb_regex_match(struct flb_regex *r, unsigned char *str, size_t slen) -{ - int ret; - unsigned char *start; - unsigned char *end; - unsigned char *range; - - /* Search scope */ - start = (unsigned char *) str; - end = start + slen; - range = end; - - ret = onig_search(r->regex, str, end, start, range, NULL, ONIG_OPTION_NONE); - - if (ret == ONIG_MISMATCH) { - return 0; - } - else if (ret < 0) { - return ret; - } - return 1; -} - - -int flb_regex_parse(struct flb_regex *r, struct flb_regex_search *result, - void (*cb_match) (const char *, /* name */ - const char *, size_t, /* value */ - void *), /* caller data */ - void *data) -{ - int ret; - - result->data = data; - result->cb_match = cb_match; - result->last_pos = -1; - - ret = onig_foreach_name(r->regex, cb_onig_named, result); - onig_region_free(result->region, 1); - - if (ret == 0) { - return result->last_pos; - } - return -1; -} - -int flb_regex_destroy(struct flb_regex *r) -{ - onig_free(r->regex); - flb_free(r); - return 0; -} - -void flb_regex_exit() -{ - onig_end(); -} diff --git a/fluent-bit/src/flb_reload.c b/fluent-bit/src/flb_reload.c deleted file mode 100644 index 641fa4a66..000000000 --- a/fluent-bit/src/flb_reload.c +++ /dev/null @@ -1,517 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2023 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -#include <fluent-bit/flb_error.h> -#include <fluent-bit/flb_info.h> -#include <fluent-bit/flb_lib.h> -#include <fluent-bit/flb_mem.h> -#include <fluent-bit/flb_input.h> -#include <fluent-bit/flb_filter.h> -#include <fluent-bit/flb_output.h> -#include <fluent-bit/flb_custom.h> -#include <fluent-bit/flb_config.h> -#include <fluent-bit/flb_config_format.h> -#include <fluent-bit/flb_kv.h> -#include <fluent-bit/flb_utils.h> -#include <fluent-bit/flb_plugin.h> - -#include <cfl/cfl.h> -#include <cfl/cfl_sds.h> -#include <cfl/cfl_variant.h> -#include <cfl/cfl_kvlist.h> - -static int flb_input_propery_check_all(struct flb_config *config) -{ - int ret; - struct mk_list *tmp; - struct mk_list *head; - struct flb_input_instance *ins; - struct flb_input_plugin *p; - - /* Iterate all active input instance plugins */ - mk_list_foreach_safe(head, tmp, &config->inputs) { - ins = mk_list_entry(head, struct flb_input_instance, _head); - p = ins->p; - - /* Skip pseudo input plugins */ - if (!p) { - continue; - } - - /* Check net property */ - ret = flb_input_net_property_check(ins, config); - if (ret == -1) { - return -1; - } - - /* Check plugin property */ - ret = flb_input_plugin_property_check(ins, config); - if (ret == -1) { - return -1; - } - - /* destroy net config map (will be recreated at flb_start) */ - if (ins->net_config_map) { - flb_config_map_destroy(ins->net_config_map); - ins->net_config_map = NULL; - } - - /* destroy config map (will be recreated at flb_start) */ - if (ins->config_map) { - flb_config_map_destroy(ins->config_map); - ins->config_map = NULL; - } - } - - return 0; -} - -static int flb_output_propery_check_all(struct flb_config *config) -{ - int ret; - struct mk_list *tmp; - struct mk_list *head; - struct flb_output_instance *ins; - - /* Iterate all active input instance plugins */ - mk_list_foreach_safe(head, tmp, &config->outputs) { - ins = mk_list_entry(head, struct flb_output_instance, _head); - - /* Check net property */ - ret = flb_output_net_property_check(ins, config); - if (ret == -1) { - return -1; - } - - /* Check plugin property */ - ret = flb_output_plugin_property_check(ins, config); - if (ret == -1) { - return -1; - } - - /* destroy net config map (will be recreated at flb_start) */ - if (ins->net_config_map) { - flb_config_map_destroy(ins->net_config_map); - ins->net_config_map = NULL; - } - - /* destroy config map (will be recreated at flb_start) */ - if (ins->config_map) { - flb_config_map_destroy(ins->config_map); - ins->config_map = NULL; - } - } - - return 0; -} - -static int flb_filter_propery_check_all(struct flb_config *config) -{ - int ret; - struct mk_list *tmp; - struct mk_list *head; - struct flb_filter_instance *ins; - - /* Iterate all active input instance plugins */ - mk_list_foreach_safe(head, tmp, &config->filters) { - ins = mk_list_entry(head, struct flb_filter_instance, _head); - - if (flb_filter_match_property_existence(ins) == FLB_FALSE) { - flb_error("[filter] NO match rule for %s filter instance, halting to reload.", - ins->name); - return -1; - } - - /* Check plugin property */ - ret = flb_filter_plugin_property_check(ins, config); - if (ret == -1) { - return -1; - } - - /* destroy config map (will be recreated at flb_start) */ - if (ins->config_map) { - flb_config_map_destroy(ins->config_map); - ins->config_map = NULL; - } - } - - return 0; -} - -static int flb_custom_propery_check_all(struct flb_config *config) -{ - int ret; - struct mk_list *tmp; - struct mk_list *head; - struct flb_custom_instance *ins; - - /* Iterate all active input instance plugins */ - mk_list_foreach_safe(head, tmp, &config->customs) { - ins = mk_list_entry(head, struct flb_custom_instance, _head); - - /* Check plugin property */ - ret = flb_custom_plugin_property_check(ins, config); - if (ret == -1) { - return -1; - } - - /* destroy config map (will be recreated at flb_start) */ - if (ins->config_map) { - flb_config_map_destroy(ins->config_map); - ins->config_map = NULL; - } - } - - return 0; -} - -int flb_reload_property_check_all(struct flb_config *config) -{ - int ret = 0; - - /* Check properties of custom plugins */ - ret = flb_custom_propery_check_all(config); - if (ret == -1) { - flb_error("[reload] check properties for custom plugins is failed"); - - return -1; - } - - /* Check properties of input plugins */ - ret = flb_input_propery_check_all(config); - if (ret == -1) { - flb_error("[reload] check properties for input plugins is failed"); - - return -1; - } - - /* Check properties of filter plugins */ - ret = flb_filter_propery_check_all(config); - if (ret == -1) { - flb_error("[reload] check properties for filter plugins is failed"); - - return -1; - } - - /* Check properties of output plugins */ - ret = flb_output_propery_check_all(config); - if (ret == -1) { - flb_error("[reload] check properties for output plugins is failed"); - - return -1; - } - - return 0; -} - -/* - * Hot reload - * ---------- - * Reload a Fluent Bit instance by using a new 'config_format' context. - * - * 1. As a first step, the config format is validated against the 'config maps', - * this will check that all configuration properties are valid. - */ - -static int recreate_cf_section(struct flb_cf_section *s, struct flb_cf *cf) -{ - struct mk_list *head; - struct cfl_list *p_head; - struct cfl_kvpair *kv; - struct flb_cf_group *g; - struct flb_cf_section *new_s; - struct flb_cf_group *new_g; - struct cfl_variant *var = NULL; - - new_s = flb_cf_section_create(cf, s->name, flb_sds_len(s->name)); - if (cfl_list_size(&s->properties->list) > 0) { - cfl_list_foreach(p_head, &s->properties->list) { - var = NULL; - kv = cfl_list_entry(p_head, struct cfl_kvpair, _head); - var = flb_cf_section_property_add(cf, new_s->properties, - kv->key, cfl_sds_len(kv->key), - kv->val->data.as_string, cfl_sds_len(kv->val->data.as_string)); - - if (var == NULL) { - flb_error("[reload] recreating section '%s' property '%s' is failed", s->name, kv->key); - return -1; - } - } - } - - if (mk_list_size(&s->groups) <= 0) { - return 0; - } - - mk_list_foreach(head, &s->groups) { - g = mk_list_entry(head, struct flb_cf_group, _head); - new_g = flb_cf_group_create(cf, new_s, g->name, flb_sds_len(g->name)); - - if (cfl_list_size(&g->properties->list) > 0) { - cfl_list_foreach(p_head, &g->properties->list) { - var = NULL; - kv = cfl_list_entry(p_head, struct cfl_kvpair, _head); - var = flb_cf_section_property_add(cf, new_g->properties, - kv->key, cfl_sds_len(kv->key), - kv->val->data.as_string, cfl_sds_len(kv->val->data.as_string)); - if (var == NULL) { - flb_error("[reload] recreating group '%s' property '%s' is failed", g->name, kv->key); - return -1; - } - } - } - } - - return 0; -} - -int flb_reload_reconstruct_cf(struct flb_cf *src_cf, struct flb_cf *dest_cf) -{ - struct mk_list *head; - struct flb_cf_section *s; - struct flb_kv *kv; - - mk_list_foreach(head, &src_cf->sections) { - s = mk_list_entry(head, struct flb_cf_section, _head); - if (recreate_cf_section(s, dest_cf) != 0) { - return -1; - } - } - - /* Copy and store env. (For yaml cf.) */ - mk_list_foreach(head, &src_cf->env) { - kv = mk_list_entry(head, struct flb_kv, _head); - if (!flb_cf_env_property_add(dest_cf, - kv->key, cfl_sds_len(kv->key), - kv->val, cfl_sds_len(kv->val))) { - return -1; - } - - } - - /* Copy and store metas. (For old fluent-bit cf.) */ - mk_list_foreach(head, &src_cf->metas) { - kv = mk_list_entry(head, struct flb_kv, _head); - if (!flb_kv_item_create_len(&dest_cf->metas, - kv->key, cfl_sds_len(kv->key), - kv->val, cfl_sds_len(kv->val))) { - return -1; - } - - } - - return 0; -} - -#ifdef FLB_HAVE_STREAM_PROCESSOR -static int flb_reload_reconstruct_sp(struct flb_config *src, struct flb_config *dest) -{ - struct mk_list *head; - struct flb_slist_entry *e; - - /* Check for pre-configured Tasks (command line) */ - mk_list_foreach(head, &src->stream_processor_tasks) { - e = mk_list_entry(head, struct flb_slist_entry, _head); - flb_slist_add(&dest->stream_processor_tasks, e->str); - } - - return 0; -} -#endif - -static int flb_reload_reinstantiate_external_plugins(struct flb_config *src, struct flb_config *dest) -{ - int ret; - struct mk_list *head; - struct flb_slist_entry *e; - - /* Check for pre-configured Tasks (command line) */ - mk_list_foreach(head, &src->external_plugins) { - e = mk_list_entry(head, struct flb_slist_entry, _head); - flb_info("[reload] slist externals %s", e->str); - /* Load the new config format context to config context. */ - ret = flb_plugin_load_router(e->str, dest); - if (ret != 0) { - return -1; - } - flb_slist_add(&dest->external_plugins, e->str); - } - - return 0; -} - -int flb_reload(flb_ctx_t *ctx, struct flb_cf *cf_opts) -{ - int ret; - flb_sds_t file = NULL; - struct flb_config *old_config; - struct flb_config *new_config; - flb_ctx_t *new_ctx = NULL; - struct flb_cf *new_cf; - struct flb_cf *original_cf; - int verbose; - int reloaded_count = 0; - - if (ctx == NULL) { - flb_error("[reload] given flb context is NULL"); - return -2; - } - - old_config = ctx->config; - if (old_config->enable_hot_reload != FLB_TRUE) { - flb_warn("[reload] hot reloading is not enabled"); - return -3; - } - - if (old_config->ensure_thread_safety_on_hot_reloading) { - old_config->grace = -1; - } - - /* Normally, we should create a service section before using this cf - * context. However, this context of config format will be used - * for copying contents from other one. So, we just need to create - * a new cf instance here. - */ - new_cf = flb_cf_create(); - if (!new_cf) { - return -1; - } - - flb_info("reloading instance pid=%lu tid=%p", (long unsigned) getpid(), pthread_self()); - - if (old_config->conf_path_file) { - file = flb_sds_create(old_config->conf_path_file); - } - if (cf_opts != NULL) { - if (flb_reload_reconstruct_cf(cf_opts, new_cf) != 0) { - if (file != NULL) { - flb_sds_destroy(file); - } - flb_error("[reload] reconstruct cf failed"); - return -1; - } - } - - /* Create another instance */ - new_ctx = flb_create(); - if (new_ctx == NULL) { - if (file != NULL) { - flb_sds_destroy(file); - } - flb_cf_destroy(new_cf); - flb_error("[reload] creating flb context is failed. Reloading is halted"); - - return -1; - } - - new_config = new_ctx->config; - - /* Inherit verbose from the old ctx instance */ - verbose = ctx->config->verbose; - new_config->verbose = verbose; - /* Increment and store the number of hot reloaded times */ - reloaded_count = ctx->config->hot_reloaded_count + 1; - -#ifdef FLB_HAVE_STREAM_PROCESSOR - /* Inherit stream processor definitions from command line */ - flb_reload_reconstruct_sp(old_config, new_config); -#endif - - /* Create another config format context */ - if (file != NULL) { - new_cf = flb_cf_create_from_file(new_cf, file); - - if (!new_cf) { - flb_sds_destroy(file); - - return -1; - } - } - - /* Load external plugins via command line */ - if (mk_list_size(&old_config->external_plugins) > 0) { - ret = flb_reload_reinstantiate_external_plugins(old_config, new_config); - if (ret == -1) { - if (file != NULL) { - flb_sds_destroy(file); - } - flb_cf_destroy(new_cf); - flb_stop(new_ctx); - flb_destroy(new_ctx); - flb_error("[reload] reloaded config is invalid. Reloading is halted"); - - return -1; - } - } - - /* Load the new config format context to config context. */ - ret = flb_config_load_config_format(new_config, new_cf); - if (ret != 0) { - flb_sds_destroy(file); - flb_cf_destroy(new_cf); - flb_stop(new_ctx); - flb_destroy(new_ctx); - - flb_error("[reload] reloaded config format is invalid. Reloading is halted"); - - return -1; - } - - /* Validate plugin properites before fluent-bit stops the old context. */ - ret = flb_reload_property_check_all(new_config); - if (ret != 0) { - flb_sds_destroy(file); - flb_cf_destroy(new_cf); - flb_stop(new_ctx); - flb_destroy(new_ctx); - - flb_error("[reload] reloaded config is invalid. Reloading is halted"); - - return -1; - } - - /* Delete the original context of config format before replacing - * with the new one. */ - original_cf = new_config->cf_main; - flb_cf_destroy(original_cf); - - new_config->cf_main = new_cf; - new_config->cf_opts = cf_opts; - - if (file != NULL) { - new_config->conf_path_file = file; - } - - flb_info("[reload] stop everything of the old context"); - flb_stop(ctx); - flb_destroy(ctx); - - flb_info("[reload] start everything"); - - ret = flb_start(new_ctx); - - /* Store the new value of hot reloading times into the new context */ - if (ret == 0) { - new_config->hot_reloaded_count = reloaded_count; - flb_debug("[reload] hot reloaded %d time(s)", reloaded_count); - } - - return 0; -} diff --git a/fluent-bit/src/flb_ring_buffer.c b/fluent-bit/src/flb_ring_buffer.c deleted file mode 100644 index 77b6e86b1..000000000 --- a/fluent-bit/src/flb_ring_buffer.c +++ /dev/null @@ -1,205 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * This interface is a wrapper of the 'lwrb' ring buffer implementation: - * - * - https://github.com/MaJerle/lwrb - */ - -#include <fluent-bit/flb_info.h> -#include <fluent-bit/flb_mem.h> -#include <fluent-bit/flb_log.h> -#include <fluent-bit/flb_pipe.h> -#include <fluent-bit/flb_ring_buffer.h> -#include <fluent-bit/flb_engine_macros.h> - -#include <monkey/mk_core.h> - -#include <math.h> - -/* lwrb header */ -#include <lwrb/lwrb.h> - -static void flb_ring_buffer_remove_event_loop(struct flb_ring_buffer *rb); - -struct flb_ring_buffer *flb_ring_buffer_create(uint64_t size) -{ - lwrb_t *lwrb; - void * data_buf; - size_t data_size; - struct flb_ring_buffer *rb; - - rb = flb_calloc(1, sizeof(struct flb_ring_buffer)); - if (!rb) { - flb_errno(); - return NULL; - } - rb->data_size = size; - - /* lwrb context */ - lwrb = flb_malloc(sizeof(lwrb_t)); - if (!lwrb) { - flb_errno(); - flb_free(rb); - return NULL; - } - rb->ctx = lwrb; - - /* data buffer for backend library */ - data_size = 1 + (sizeof(uint8_t) * size); - data_buf = flb_calloc(1, data_size); - if (!data_buf) { - flb_errno(); - flb_free(rb); - flb_free(lwrb); - return NULL; - } - rb->data_buf = data_buf; - - /* initialize lwrb */ - lwrb_init(rb->ctx, data_buf, data_size); - - return rb; -} - -void flb_ring_buffer_destroy(struct flb_ring_buffer *rb) -{ - flb_ring_buffer_remove_event_loop(rb); - - if (rb->data_buf) { - flb_free(rb->data_buf); - } - - if (rb->ctx) { - flb_free(rb->ctx); - } - - flb_free(rb); -} - -int flb_ring_buffer_add_event_loop(struct flb_ring_buffer *rb, void *evl, uint8_t window_size) -{ - int result; - - if (window_size == 0) { - return -1; - } - else if (window_size > 100) { - window_size = 100; - } - - rb->data_window = (uint64_t) floor((rb->data_size * window_size) / 100); - - result = flb_pipe_create(rb->signal_channels); - - if (result) { - return -2; - } - - flb_pipe_set_nonblocking(rb->signal_channels[0]); - flb_pipe_set_nonblocking(rb->signal_channels[1]); - - rb->signal_event = (void *) flb_calloc(1, sizeof(struct mk_event)); - - if (rb->signal_event == NULL) { - flb_pipe_destroy(rb->signal_channels); - - return -2; - } - - MK_EVENT_ZERO(rb->signal_event); - - result = mk_event_add(evl, - rb->signal_channels[0], - FLB_ENGINE_EV_THREAD_INPUT, - MK_EVENT_READ, - rb->signal_event); - - if (result) { - flb_pipe_destroy(rb->signal_channels); - flb_free(rb->signal_event); - - rb->signal_event = NULL; - - return -3; - } - - rb->event_loop = evl; - - return 0; -} - -static void flb_ring_buffer_remove_event_loop(struct flb_ring_buffer *rb) -{ - if (rb->event_loop != NULL) { - mk_event_del(rb->event_loop, rb->signal_event); - flb_pipe_destroy(rb->signal_channels); - flb_free(rb->signal_event); - - rb->signal_event = NULL; - rb->data_window = 0; - rb->event_loop = NULL; - } -} - -int flb_ring_buffer_write(struct flb_ring_buffer *rb, void *ptr, size_t size) -{ - size_t used_size; - size_t ret; - size_t av; - - /* make sure there is enough space available */ - av = lwrb_get_free(rb->ctx); - if (av < size) { - return -1; - } - - /* write the content */ - ret = lwrb_write(rb->ctx, ptr, size); - if (ret == 0) { - return -1; - } - - if (!rb->flush_pending) { - used_size = rb->data_size - (av - size); - - if (used_size >= rb->data_window) { - rb->flush_pending = FLB_TRUE; - - flb_pipe_write_all(rb->signal_channels[1], ".", 1); - } - } - - return 0; -} - -int flb_ring_buffer_read(struct flb_ring_buffer *rb, void *ptr, size_t size) -{ - size_t ret; - - ret = lwrb_read(rb->ctx, ptr, size); - if (ret == 0) { - return -1; - } - - return 0; -} - - diff --git a/fluent-bit/src/flb_router.c b/fluent-bit/src/flb_router.c deleted file mode 100644 index 551e9c335..000000000 --- a/fluent-bit/src/flb_router.c +++ /dev/null @@ -1,271 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <fluent-bit/flb_info.h> -#include <fluent-bit/flb_mem.h> -#include <fluent-bit/flb_str.h> -#include <fluent-bit/flb_sds.h> -#include <fluent-bit/flb_input.h> -#include <fluent-bit/flb_input_chunk.h> -#include <fluent-bit/flb_output.h> -#include <fluent-bit/flb_config.h> -#include <fluent-bit/flb_router.h> - -#ifdef FLB_HAVE_REGEX -#include <onigmo.h> -#endif - -#include <string.h> - -/* wildcard support */ -/* tag and match should be null terminated. */ -static inline int router_match(const char *tag, int tag_len, - const char *match, - void *match_r) -{ - int ret = FLB_FALSE; - char *pos = NULL; - -#ifdef FLB_HAVE_REGEX - struct flb_regex *match_regex = match_r; - int n; - if (match_regex) { - n = onig_match(match_regex->regex, - (const unsigned char *) tag, - (const unsigned char *) tag + tag_len, - (const unsigned char *) tag, 0, - ONIG_OPTION_NONE); - if (n > 0) { - return 1; - } - } -#else - (void) match_r; -#endif - - while (match) { - if (*match == '*') { - while (*++match == '*'){ - /* skip successive '*' */ - } - if (*match == '\0') { - /* '*' is last of string */ - ret = 1; - break; - } - - while ((pos = strchr(tag, (int) *match))) { -#ifndef FLB_HAVE_REGEX - if (router_match(pos, tag_len, match, NULL)) { -#else - /* We don't need to pass the regex recursively, - * we matched in order above - */ - if (router_match(pos, tag_len, match, NULL)) { -#endif - ret = 1; - break; - } - tag = pos+1; - } - break; - } - else if (*tag != *match) { - /* mismatch! */ - break; - } - else if (*tag == '\0') { - /* end of tag. so matched! */ - ret = 1; - break; - } - tag++; - match++; - } - - return ret; -} - -int flb_router_match(const char *tag, int tag_len, const char *match, - void *match_regex) -{ - int ret; - flb_sds_t t; - - if (tag[tag_len] != '\0') { - t = flb_sds_create_len(tag, tag_len); - if (!t) { - return FLB_FALSE; - } - - ret = router_match(t, tag_len, match, match_regex); - flb_sds_destroy(t); - } - else { - ret = router_match(tag, tag_len, match, match_regex); - } - - return ret; -} - -/* Associate and input and output instances due to a previous match */ -int flb_router_connect(struct flb_input_instance *in, - struct flb_output_instance *out) -{ - struct flb_router_path *p; - - p = flb_malloc(sizeof(struct flb_router_path)); - if (!p) { - flb_errno(); - return -1; - } - - p->ins = out; - mk_list_add(&p->_head, &in->routes); - - return 0; -} - -int flb_router_connect_direct(struct flb_input_instance *in, - struct flb_output_instance *out) -{ - struct flb_router_path *p; - - p = flb_malloc(sizeof(struct flb_router_path)); - if (!p) { - flb_errno(); - return -1; - } - - p->ins = out; - mk_list_add(&p->_head, &in->routes_direct); - - return 0; -} - -/* - * This routine defines static routes for the plugins that have registered - * tags. It check where data should go before the service start running, each - * input 'instance' plugin will contain a list of destinations. - */ -int flb_router_io_set(struct flb_config *config) -{ - int in_count = 0; - int out_count = 0; - struct mk_list *i_head; - struct mk_list *o_head; - struct flb_input_instance *i_ins; - struct flb_output_instance *o_ins; - - /* Quick setup for 1:1 */ - in_count = mk_list_size(&config->inputs); - out_count = mk_list_size(&config->outputs); - - /* Mostly used for command line tests */ - if (in_count == 1 && out_count == 1) { - i_ins = mk_list_entry_first(&config->inputs, struct flb_input_instance, _head); - o_ins = mk_list_entry_first(&config->outputs, struct flb_output_instance, _head); - - if (!o_ins->match -#ifdef FLB_HAVE_REGEX - && !o_ins->match_regex -#endif - ) { - - o_ins->match = flb_sds_create_len("*", 1); - } - flb_router_connect(i_ins, o_ins); - return 0; - } - - /* N:M case, iterate all input instances */ - mk_list_foreach(i_head, &config->inputs) { - i_ins = mk_list_entry(i_head, struct flb_input_instance, _head); - if (!i_ins->p) { - continue; - } - - if (!i_ins->tag) { - flb_warn("[router] NO tag for %s input instance", - i_ins->name); - continue; - } - - - flb_trace("[router] input=%s tag=%s", i_ins->name, i_ins->tag); - - /* Try to find a match with output instances */ - mk_list_foreach(o_head, &config->outputs) { - o_ins = mk_list_entry(o_head, struct flb_output_instance, _head); - if (!o_ins->match -#ifdef FLB_HAVE_REGEX - && !o_ins->match_regex -#endif - ) { - flb_warn("[router] NO match for %s output instance", - o_ins->name); - continue; - } - - if (flb_router_match(i_ins->tag, i_ins->tag_len, o_ins->match -#ifdef FLB_HAVE_REGEX - , o_ins->match_regex -#else - , NULL -#endif - )) { - - flb_debug("[router] match rule %s:%s", - i_ins->name, o_ins->name); - flb_router_connect(i_ins, o_ins); - } - } - } - - return 0; -} - -void flb_router_exit(struct flb_config *config) -{ - struct mk_list *tmp; - struct mk_list *r_tmp; - struct mk_list *head; - struct mk_list *r_head; - struct flb_input_instance *in; - struct flb_router_path *r; - - /* Iterate input plugins */ - mk_list_foreach_safe(head, tmp, &config->inputs) { - in = mk_list_entry(head, struct flb_input_instance, _head); - - /* Iterate instance routes */ - mk_list_foreach_safe(r_head, r_tmp, &in->routes) { - r = mk_list_entry(r_head, struct flb_router_path, _head); - mk_list_del(&r->_head); - flb_free(r); - } - - /* Iterate instance routes direct */ - mk_list_foreach_safe(r_head, r_tmp, &in->routes_direct) { - r = mk_list_entry(r_head, struct flb_router_path, _head); - mk_list_del(&r->_head); - flb_free(r); - } - } -} diff --git a/fluent-bit/src/flb_routes_mask.c b/fluent-bit/src/flb_routes_mask.c deleted file mode 100644 index e9a21f5a7..000000000 --- a/fluent-bit/src/flb_routes_mask.c +++ /dev/null @@ -1,143 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <fluent-bit/flb_info.h> -#include <fluent-bit/flb_log.h> -#include <fluent-bit/flb_input.h> -#include <fluent-bit/flb_router.h> -#include <fluent-bit/flb_routes_mask.h> - - -/* - * Set the routes_mask for input chunk with a router_match on tag, return a - * non-zero value if any routes matched - */ -int flb_routes_mask_set_by_tag(uint64_t *routes_mask, - const char *tag, - int tag_len, - struct flb_input_instance *in) -{ - int has_routes = 0; - struct mk_list *o_head; - struct flb_output_instance *o_ins; - if (!in) { - return 0; - } - - /* Clear the bit field */ - memset(routes_mask, 0, sizeof(uint64_t) * FLB_ROUTES_MASK_ELEMENTS); - - /* Find all matching routes for the given tag */ - mk_list_foreach(o_head, &in->config->outputs) { - o_ins = mk_list_entry(o_head, - struct flb_output_instance, _head); - - if (flb_router_match(tag, tag_len, o_ins->match -#ifdef FLB_HAVE_REGEX - , o_ins->match_regex -#else - , NULL -#endif - )) { - flb_routes_mask_set_bit(routes_mask, o_ins->id); - has_routes = 1; - } - } - - return has_routes; -} - -/* - * Sets a single bit in an array of bitfields - * - * For example: Given a value of 35 this routine will set the - * 4th bit in the 2nd value of the bitfield array. - * - */ -void flb_routes_mask_set_bit(uint64_t *routes_mask, int value) -{ - int index; - uint64_t bit; - - if (value < 0 || value > FLB_ROUTES_MASK_MAX_VALUE) { - flb_warn("[routes_mask] Can't set bit (%d) past limits of bitfield", - value); - return; - } - - index = value / FLB_ROUTES_MASK_ELEMENT_BITS; - bit = 1ULL << (value % FLB_ROUTES_MASK_ELEMENT_BITS); - routes_mask[index] |= bit; -} - -/* - * Clears a single bit in an array of bitfields - * - * For example: Given a value of 68 this routine will clear the - * 4th bit in the 2nd value of the bitfield array. - * - */ -void flb_routes_mask_clear_bit(uint64_t *routes_mask, int value) -{ - int index; - uint64_t bit; - - if (value < 0 || value > FLB_ROUTES_MASK_MAX_VALUE) { - flb_warn("[routes_mask] Can't set bit (%d) past limits of bitfield", - value); - return; - } - - index = value / FLB_ROUTES_MASK_ELEMENT_BITS; - bit = 1ULL << (value % FLB_ROUTES_MASK_ELEMENT_BITS); - routes_mask[index] &= ~(bit); -} - -/* - * Checks the value of a single bit in an array of bitfields and returns a - * non-zero value if that bit is set. - * - * For example: Given a value of 68 this routine will return a non-zero value - * if the 4th bit in the 2nd value of the bitfield array is set. - * - */ -int flb_routes_mask_get_bit(uint64_t *routes_mask, int value) -{ - int index; - uint64_t bit; - - if (value < 0 || value > FLB_ROUTES_MASK_MAX_VALUE) { - flb_warn("[routes_mask] Can't get bit (%d) past limits of bitfield", - value); - return 0; - } - - index = value / FLB_ROUTES_MASK_ELEMENT_BITS; - bit = 1ULL << (value % FLB_ROUTES_MASK_ELEMENT_BITS); - return (routes_mask[index] & bit) != 0ULL; -} - -int flb_routes_mask_is_empty(uint64_t *routes_mask) -{ - uint64_t empty[FLB_ROUTES_MASK_ELEMENTS]; - - /* Clear the tmp bitfield */ - memset(empty, 0, sizeof(empty)); - return memcmp(routes_mask, empty, sizeof(empty)) == 0; -} diff --git a/fluent-bit/src/flb_scheduler.c b/fluent-bit/src/flb_scheduler.c deleted file mode 100644 index 91f706828..000000000 --- a/fluent-bit/src/flb_scheduler.c +++ /dev/null @@ -1,727 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <fluent-bit/flb_info.h> -#include <fluent-bit/flb_mem.h> -#include <fluent-bit/flb_coro.h> -#include <fluent-bit/flb_config.h> -#include <fluent-bit/flb_scheduler.h> -#include <fluent-bit/flb_pipe.h> -#include <fluent-bit/flb_engine.h> -#include <fluent-bit/flb_engine_dispatch.h> -#include <fluent-bit/flb_random.h> - -#include <sys/types.h> -#include <sys/stat.h> -#include <fcntl.h> - -FLB_TLS_DEFINE(struct flb_sched, flb_sched_ctx); - -void flb_sched_ctx_init() -{ - FLB_TLS_INIT(flb_sched_ctx); -} - -struct flb_sched *flb_sched_ctx_get() -{ - struct flb_sched *sched; - - sched = FLB_TLS_GET(flb_sched_ctx); - return sched; -} - -void flb_sched_ctx_set(struct flb_sched *sched) -{ - FLB_TLS_SET(flb_sched_ctx, sched); -} - -static inline double xmin(double a, double b) -{ - return a < b ? a : b; -} - -/* Consume an unsigned 64 bit number from fd */ -static inline int consume_byte(flb_pipefd_t fd) -{ - int ret; - uint64_t val; - - /* We need to consume the byte */ - ret = flb_pipe_r(fd, &val, sizeof(val)); -#if defined(__APPLE__) || __FreeBSD__ >= 12 - if (ret < 0) { -#else - if (ret <= 0) { -#endif - flb_errno(); - return -1; - } - - return 0; -} - -/* - * Generate an uniform random value between min and max. Original version - * taken from internet and modified to use /dev/urandom to set a seed on - * each call. Despites using the urandom device may add some overhead, - * this function is not called too often so it should not be an issue. - */ -static int random_uniform(int min, int max) -{ - int val; - int range; - int copies; - int limit; - int ra; - - if (flb_random_bytes((unsigned char *) &val, sizeof(int))) { - val = time(NULL); - } - srand(val); - - range = max - min + 1; - copies = (RAND_MAX / range); - limit = range * copies; - ra = -1; - - while (ra < 0 || ra >= limit) { - ra = rand(); - } - - return ra / copies + min; -} - - -/* - * Schedule a request that will be processed within the next - * FLB_SCHED_REQUEST_FRAME seconds. - */ -static int schedule_request_now(int seconds, - struct flb_sched_timer *timer, - struct flb_sched_request *request, - struct flb_config *config) -{ - flb_pipefd_t fd; - struct mk_event *event; - struct flb_sched *sched = config->sched; - - /* Initialize event */ - event = &timer->event; - event->mask = MK_EVENT_EMPTY; - event->status = MK_EVENT_NONE; - - /* Create a timeout into the main event loop */ - fd = mk_event_timeout_create(config->evl, seconds, 0, event); - event->priority = FLB_ENGINE_PRIORITY_CB_SCHED; - if (fd == -1) { - return -1; - } - request->fd = fd; - timer->timer_fd = fd; - - /* - * Note: mk_event_timeout_create() sets a type = MK_EVENT_NOTIFICATION by - * default, we need to overwrite this value so we can do a clean check - * into the Engine when the event is triggered. - */ - event->type = FLB_ENGINE_EV_SCHED; - mk_list_add(&request->_head, &sched->requests); - - return 0; -} - -/* - * Enqueue a request that will wait until it expected timeout reach the - * FLB_SCHED_REQUEST_FRAME interval. - */ -static int schedule_request_wait(struct flb_sched_request *request, - struct flb_config *config) -{ - struct flb_sched *sched = config->sched; - - mk_list_add(&request->_head, &sched->requests_wait); - return 0; -} - -/* - * Iterate requests_wait list looking for candidates to be promoted - * to the 'requests' list. - */ -static int schedule_request_promote(struct flb_sched *sched) -{ - int ret; - int next; - int passed; - time_t now; - struct mk_list *tmp; - struct mk_list *head; - struct mk_list failed_requests; - struct flb_sched_request *request; - - now = time(NULL); - mk_list_init(&failed_requests); - - mk_list_foreach_safe(head, tmp, &sched->requests_wait) { - request = mk_list_entry(head, struct flb_sched_request, _head); - - /* First check how many seconds have passed since the request creation */ - passed = (now - request->created); - ret = 0; - - /* If we passed the original time, schedule now for the next second */ - if (passed > request->timeout) { - mk_list_del(&request->_head); - ret = schedule_request_now(1, request->timer, request, sched->config); - if (ret != 0) { - mk_list_add(&request->_head, &failed_requests); - } - } - else if (passed + FLB_SCHED_REQUEST_FRAME >= request->timeout) { - /* Check if we should schedule within this frame */ - mk_list_del(&request->_head); - next = labs(passed - request->timeout); - ret = schedule_request_now(next, request->timer, request, sched->config); - if (ret != 0) { - mk_list_add(&request->_head, &failed_requests); - } - } - else { - continue; - } - - /* - * If the 'request' could not be scheduled, this could only happen due to memory - * exhaustion or running out of file descriptors. There is no much we can do - * at this time. - */ - if (ret == -1) { - flb_error("[sched] a 'retry request' could not be scheduled. the " - "system might be running out of memory or file " - "descriptors. The scheduler will do a retry later."); - } - } - - /* For each failed request, re-add them to the wait list */ - mk_list_foreach_safe(head, tmp, &failed_requests) { - request = mk_list_entry(head, struct flb_sched_request, _head); - mk_list_del(&request->_head); - mk_list_add(&request->_head, &sched->requests_wait); - } - - return 0; -} - -static double ipow(double base, int exp) -{ - double result = 1; - - for (;;) { - if (exp & 1) { - result *= base; - } - - exp >>= 1; - if (!exp) { - break; - } - base *= base; - } - - return result; -} - -/* - * The 'backoff full jitter' algorithm implements a capped backoff with a jitter - * to generate numbers to be used as 'wait times', this implementation is fully - * based on the following article: - * - * https://www.awsarchitectureblog.com/2015/03/backoff.html - */ -static int backoff_full_jitter(int base, int cap, int n) -{ - int temp; - - temp = xmin(cap, base * ipow(2, n)); - return random_uniform(base, temp); -} - -/* Schedule the 'retry' for a thread buffer flush */ -int flb_sched_request_create(struct flb_config *config, void *data, int tries) -{ - int ret; - int seconds; - struct flb_sched_timer *timer; - struct flb_sched_request *request; - - /* Allocate timer context */ - timer = flb_sched_timer_create(config->sched); - if (!timer) { - return -1; - } - - /* Allocate request node */ - request = flb_malloc(sizeof(struct flb_sched_request)); - if (!request) { - flb_errno(); - return -1; - } - - /* Link timer references */ - timer->type = FLB_SCHED_TIMER_REQUEST; - timer->data = request; - timer->event.mask = MK_EVENT_EMPTY; - - /* Get suggested wait_time for this request. If shutting down, set to 0. */ - if (config->is_shutting_down) { - seconds = 0; - } else { - seconds = backoff_full_jitter((int)config->sched_base, (int)config->sched_cap, - tries); - } - seconds += 1; - - /* Populare request */ - request->fd = -1; - request->created = time(NULL); - request->timeout = seconds; - request->data = data; - request->timer = timer; - - /* Request to be placed into the sched_requests_wait list */ - if (seconds > FLB_SCHED_REQUEST_FRAME) { - schedule_request_wait(request, config); - } - else { - ret = schedule_request_now(seconds, timer, request, config); - if (ret == -1) { - flb_error("[sched] 'retry request' could not be created. the " - "system might be running out of memory or file " - "descriptors."); - flb_sched_timer_destroy(timer); - flb_free(request); - return -1; - } - } - - return seconds; -} - -int flb_sched_request_destroy(struct flb_sched_request *req) -{ - struct flb_sched_timer *timer; - - if (!req) { - return 0; - } - - mk_list_del(&req->_head); - - timer = req->timer; - - /* - * We invalidate the timer since in the same event loop round - * an event associated to this timer can be present. Invalidation - * means the timer will do nothing and will be removed after - * the event loop round finish. - */ - flb_sched_timer_invalidate(timer); - - /* Remove request */ - flb_free(req); - - return 0; -} - -int flb_sched_request_invalidate(struct flb_config *config, void *data) -{ - struct mk_list *tmp; - struct mk_list *head; - struct flb_sched_request *request; - struct flb_sched *sched; - - sched = config->sched; - mk_list_foreach_safe(head, tmp, &sched->requests) { - request = mk_list_entry(head, struct flb_sched_request, _head); - if (request->data == data) { - flb_sched_request_destroy(request); - return 0; - } - } - - /* - * Clean up retry tasks that are scheduled more than 60s. - * Task might be destroyed when there are still retry - * scheduled but no thread is running for the task. - * - * We need to drop buffered chunks when the filesystem buffer - * limit is reached. We need to make sure that all requests - * should be destroyed to avoid invoke an invlidated request. - */ - mk_list_foreach_safe(head, tmp, &sched->requests_wait) { - request = mk_list_entry(head, struct flb_sched_request, _head); - if (request->data == data) { - flb_sched_request_destroy(request); - return 0; - } - } - - return -1; -} - -/* Handle a timeout event set by a previous flb_sched_request_create(...) */ -int flb_sched_event_handler(struct flb_config *config, struct mk_event *event) -{ - int ret; - struct flb_sched *sched; - struct flb_sched_timer *timer; - struct flb_sched_request *req; - - timer = (struct flb_sched_timer *) event; - if (timer->active == FLB_FALSE) { - return 0; - } - - if (timer->type == FLB_SCHED_TIMER_REQUEST) { - /* Map request struct */ - req = timer->data; - consume_byte(req->fd); - - /* Dispatch 'retry' */ - ret = flb_engine_dispatch_retry(req->data, config); - - /* Destroy this scheduled request, it's not longer required */ - if (ret == 0) { - flb_sched_request_destroy(req); - } - } - else if (timer->type == FLB_SCHED_TIMER_FRAME) { - sched = timer->data; -#ifndef __APPLE__ - consume_byte(sched->frame_fd); -#endif - schedule_request_promote(sched); - } - else if (timer->type == FLB_SCHED_TIMER_CB_ONESHOT) { - consume_byte(timer->timer_fd); - flb_sched_timer_cb_disable(timer); - timer->cb(config, timer->data); - flb_sched_timer_cb_destroy(timer); - } - else if (timer->type == FLB_SCHED_TIMER_CB_PERM) { - consume_byte(timer->timer_fd); - timer->cb(config, timer->data); - } - - return 0; -} - -/* - * Create a timer that once it expire, it triggers the defined callback - * upon creation. This interface is for generic purposes and not specific - * for re-tries. - * - * use-case: invoke function A() after M milliseconds. - */ -int flb_sched_timer_cb_create(struct flb_sched *sched, int type, int ms, - void (*cb)(struct flb_config *, void *), - void *data, struct flb_sched_timer **out_timer) -{ - int fd; - time_t sec; - long nsec; - struct mk_event *event; - struct flb_sched_timer *timer; - - if (type != FLB_SCHED_TIMER_CB_ONESHOT && type != FLB_SCHED_TIMER_CB_PERM) { - flb_error("[sched] invalid callback timer type %i", type); - return -1; - } - - timer = flb_sched_timer_create(sched); - if (!timer) { - return -1; - } - - timer->type = type; - timer->data = data; - timer->cb = cb; - - /* Initialize event */ - event = &timer->event; - event->mask = MK_EVENT_EMPTY; - event->status = MK_EVENT_NONE; - - /* Convert from milliseconds to seconds and nanoseconds */ - sec = (ms / 1000); - nsec = ((ms % 1000) * 1000000); - - /* Create the frame timer */ - fd = mk_event_timeout_create(sched->evl, sec, nsec, event); - event->priority = FLB_ENGINE_PRIORITY_CB_TIMER; - if (fd == -1) { - flb_error("[sched] cannot do timeout_create()"); - flb_sched_timer_destroy(timer); - return -1; - } - - /* - * Note: mk_event_timeout_create() sets a type = MK_EVENT_NOTIFICATION by - * default, we need to overwrite this value so we can do a clean check - * into the Engine when the event is triggered. - */ - event->type = FLB_ENGINE_EV_SCHED; - timer->timer_fd = fd; - - if (out_timer != NULL) { - *out_timer = timer; - } - - return 0; -} - -/* Disable notifications, used before to destroy the context */ -int flb_sched_timer_cb_disable(struct flb_sched_timer *timer) -{ - if (timer->timer_fd != -1) { - mk_event_timeout_destroy(timer->sched->evl, &timer->event); - - timer->timer_fd = -1; - } - - return 0; -} - -int flb_sched_timer_cb_destroy(struct flb_sched_timer *timer) -{ - flb_sched_timer_destroy(timer); - - return 0; -} - -/* Initialize the Scheduler */ -struct flb_sched *flb_sched_create(struct flb_config *config, - struct mk_event_loop *evl) -{ - flb_pipefd_t fd; - struct mk_event *event; - struct flb_sched *sched; - struct flb_sched_timer *timer; - - sched = flb_calloc(1, sizeof(struct flb_sched)); - if (!sched) { - flb_errno(); - return NULL; - } - - sched->config = config; - sched->evl = evl; - - /* Initialize lists */ - mk_list_init(&sched->requests); - mk_list_init(&sched->requests_wait); - mk_list_init(&sched->timers); - mk_list_init(&sched->timers_drop); - - /* Create the frame timer who enqueue 'requests' for future time */ - timer = flb_sched_timer_create(sched); - if (!timer) { - flb_free(sched); - return NULL; - } - - timer->type = FLB_SCHED_TIMER_FRAME; - timer->data = sched; - - /* Initialize event */ - event = &timer->event; - event->mask = MK_EVENT_EMPTY; - event->status = MK_EVENT_NONE; - - /* Create the frame timer */ - fd = mk_event_timeout_create(evl, FLB_SCHED_REQUEST_FRAME, 0, - event); - event->priority = FLB_ENGINE_PRIORITY_CB_SCHED; - if (fd == -1) { - flb_sched_timer_destroy(timer); - flb_free(sched); - return NULL; - } - sched->frame_fd = fd; - - /* - * Note: mk_event_timeout_create() sets a type = MK_EVENT_NOTIFICATION by - * default, we need to overwrite this value so we can do a clean check - * into the Engine when the event is triggered. - */ - event->type = FLB_ENGINE_EV_SCHED_FRAME; - - return sched; -} - -/* Release all resources used by the Scheduler */ -int flb_sched_destroy(struct flb_sched *sched) -{ - int c = 0; - struct mk_list *tmp; - struct mk_list *head; - struct flb_sched_timer *timer; - struct flb_sched_request *request; - - if (!sched) { - return 0; - } - - mk_list_foreach_safe(head, tmp, &sched->requests) { - request = mk_list_entry(head, struct flb_sched_request, _head); - flb_sched_request_destroy(request); - c++; /* evil counter */ - } - - /* Delete requests on wait list */ - mk_list_foreach_safe(head, tmp, &sched->requests_wait) { - request = mk_list_entry(head, struct flb_sched_request, _head); - flb_sched_request_destroy(request); - c++; /* evil counter */ - } - - /* Delete timers */ - mk_list_foreach_safe(head, tmp, &sched->timers) { - timer = mk_list_entry(head, struct flb_sched_timer, _head); - flb_sched_timer_destroy(timer); - c++; - } - - /* Delete timers drop list */ - mk_list_foreach_safe(head, tmp, &sched->timers_drop) { - timer = mk_list_entry(head, struct flb_sched_timer, _head); - flb_sched_timer_destroy(timer); - c++; - } - - flb_free(sched); - return c; -} - -/* Create a timer context */ -struct flb_sched_timer *flb_sched_timer_create(struct flb_sched *sched) -{ - struct flb_sched_timer *timer; - - /* Create timer context */ - timer = flb_calloc(1, sizeof(struct flb_sched_timer)); - if (!timer) { - flb_errno(); - return NULL; - } - MK_EVENT_ZERO(&timer->event); - - timer->timer_fd = -1; - timer->config = sched->config; - timer->sched = sched; - timer->data = NULL; - - /* Active timer (not invalidated) */ - timer->active = FLB_TRUE; - mk_list_add(&timer->_head, &sched->timers); - - return timer; -} - -void flb_sched_timer_invalidate(struct flb_sched_timer *timer) -{ - flb_sched_timer_cb_disable(timer); - - timer->active = FLB_FALSE; - - mk_list_del(&timer->_head); - mk_list_add(&timer->_head, &timer->sched->timers_drop); -} - -/* Destroy a timer context */ -int flb_sched_timer_destroy(struct flb_sched_timer *timer) -{ - flb_sched_timer_cb_disable(timer); - - mk_list_del(&timer->_head); - flb_free(timer); - - return 0; -} - -/* Used by the engine to cleanup pending timers waiting to be destroyed */ -int flb_sched_timer_cleanup(struct flb_sched *sched) -{ - int c = 0; - struct mk_list *tmp; - struct mk_list *head; - struct flb_sched_timer *timer; - - mk_list_foreach_safe(head, tmp, &sched->timers_drop) { - timer = mk_list_entry(head, struct flb_sched_timer, _head); - flb_sched_timer_destroy(timer); - c++; - } - - return c; -} - -int flb_sched_retry_now(struct flb_config *config, - struct flb_task_retry *retry) -{ - int ret; - struct flb_sched_timer *timer; - struct flb_sched_request *request; - - /* Allocate timer context */ - timer = flb_sched_timer_create(config->sched); - if (!timer) { - return -1; - } - - /* Allocate request node */ - request = flb_malloc(sizeof(struct flb_sched_request)); - if (!request) { - flb_errno(); - flb_sched_timer_destroy(timer); - return -1; - } - - /* Link timer references */ - timer->type = FLB_SCHED_TIMER_REQUEST; - timer->data = request; - timer->event.mask = MK_EVENT_EMPTY; - - /* Populate request */ - request->fd = -1; - request->created = time(NULL); - request->timeout = 0; - request->data = retry; - request->timer = timer; - - ret = schedule_request_now(0 /* seconds */, timer, request, config); - if (ret == -1) { - flb_error("[sched] 'retry-now request' could not be created. the " - "system might be running out of memory or file " - "descirptors."); - flb_sched_timer_destroy(timer); - flb_free(request); - return -1; - } - return 0; -} diff --git a/fluent-bit/src/flb_sds.c b/fluent-bit/src/flb_sds.c deleted file mode 100644 index 2cb562c86..000000000 --- a/fluent-bit/src/flb_sds.c +++ /dev/null @@ -1,500 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * The following SDS interface is a clone/strip-down version of the original - * SDS library created by Antirez at https://github.com/antirez/sds. - */ - -#include <fluent-bit/flb_compat.h> -#include <fluent-bit/flb_info.h> -#include <fluent-bit/flb_mem.h> -#include <fluent-bit/flb_log.h> -#include <fluent-bit/flb_sds.h> -#include <fluent-bit/flb_utf8.h> - -#include <stdarg.h> -#include <ctype.h> - -static flb_sds_t sds_alloc(size_t size) -{ - void *buf; - flb_sds_t s; - struct flb_sds *head; - - buf = flb_malloc(FLB_SDS_HEADER_SIZE + size + 1); - if (!buf) { - flb_errno(); - return NULL; - } - - head = buf; - head->len = 0; - head->alloc = size; - - s = head->buf; - *s = '\0'; - - return s; -} - -flb_sds_t flb_sds_create_len(const char *str, int len) -{ - flb_sds_t s; - struct flb_sds *head; - - s = sds_alloc(len); - if (!s) { - return NULL; - } - - if (str) { - memcpy(s, str, len); - s[len] = '\0'; - - head = FLB_SDS_HEADER(s); - head->len = len; - } - return s; -} - -flb_sds_t flb_sds_create(const char *str) -{ - size_t len; - - if (!str) { - len = 0; - } - else { - len = strlen(str); - } - - return flb_sds_create_len(str, len); -} - -flb_sds_t flb_sds_create_size(size_t size) -{ - return sds_alloc(size); -} - -/* Increase SDS buffer size 'len' bytes */ -flb_sds_t flb_sds_increase(flb_sds_t s, size_t len) -{ - size_t new_size; - struct flb_sds *head; - flb_sds_t out; - void *tmp; - - out = s; - new_size = (FLB_SDS_HEADER_SIZE + flb_sds_alloc(s) + len + 1); - head = FLB_SDS_HEADER(s); - tmp = flb_realloc(head, new_size); - if (!tmp) { - flb_errno(); - return NULL; - } - head = (struct flb_sds *) tmp; - head->alloc += len; - out = head->buf; - - return out; -} - -flb_sds_t flb_sds_cat(flb_sds_t s, const char *str, int len) -{ - size_t avail; - struct flb_sds *head; - flb_sds_t tmp = NULL; - - avail = flb_sds_avail(s); - if (avail < len) { - tmp = flb_sds_increase(s, len); - if (!tmp) { - return NULL; - } - s = tmp; - } - memcpy((char *) (s + flb_sds_len(s)), str, len); - - head = FLB_SDS_HEADER(s); - head->len += len; - s[head->len] = '\0'; - - return s; -} - - -/* - * remove empty spaces on left/right from sds buffer 's' and return the new length - * of the content. - */ -int flb_sds_trim(flb_sds_t s) -{ - unsigned int i; - unsigned int len; - char *left = 0, *right = 0; - char *buf; - - if (!s) { - return -1; - } - - len = flb_sds_len(s); - if (len == 0) { - return 0; - } - - buf = s; - left = buf; - - /* left spaces */ - while (left) { - if (isspace(*left)) { - left++; - } - else { - break; - } - } - - right = buf + (len - 1); - /* Validate right v/s left */ - if (right < left) { - buf[0] = '\0'; - return -1; - } - - /* Move back */ - while (right != buf){ - if (isspace(*right)) { - right--; - } - else { - break; - } - } - - len = (right - left) + 1; - for (i=0; i<len; i++) { - buf[i] = (char) left[i]; - } - buf[i] = '\0'; - flb_sds_len_set(buf, i); - - return i; -} - -int flb_sds_cat_safe(flb_sds_t *buf, const char *str, int len) -{ - flb_sds_t tmp; - - tmp = flb_sds_cat(*buf, str, len); - if (!tmp) { - return -1; - } - *buf = tmp; - return 0; -} - -flb_sds_t flb_sds_cat_esc(flb_sds_t s, const char *str, int len, - char *esc, size_t esc_size) -{ - size_t avail; - struct flb_sds *head; - flb_sds_t tmp = NULL; - uint32_t c; - int i; - - avail = flb_sds_avail(s); - if (avail < len) { - tmp = flb_sds_increase(s, len); - if (!tmp) { - return NULL; - } - s = tmp; - } - head = FLB_SDS_HEADER(s); - - for (i = 0; i < len; i++) { - if (flb_sds_avail(s) < 8) { - tmp = flb_sds_increase(s, 8); - if (tmp == NULL) { - return NULL; - } - s = tmp; - head = FLB_SDS_HEADER(s); - } - c = (unsigned char) str[i]; - if (esc != NULL && c < esc_size && esc[c] != 0) { - s[head->len++] = '\\'; - s[head->len++] = esc[c]; - } - else { - s[head->len++] = c; - } - } - - s[head->len] = '\0'; - - return s; -} - - -flb_sds_t flb_sds_copy(flb_sds_t s, const char *str, int len) -{ - size_t avail; - struct flb_sds *head; - flb_sds_t tmp = NULL; - - avail = flb_sds_alloc(s); - if (avail < len) { - tmp = flb_sds_increase(s, len); - if (!tmp) { - return NULL; - } - s = tmp; - } - memcpy((char *) s, str, len); - - head = FLB_SDS_HEADER(s); - head->len = len; - s[head->len] = '\0'; - - return s; -} - -flb_sds_t flb_sds_cat_utf8 (flb_sds_t *sds, const char *str, int str_len) -{ - static const char int2hex[] = "0123456789abcdef"; - int i; - int b; - int ret; - int hex_bytes; - uint32_t cp; - uint32_t state = 0; - unsigned char c; - const uint8_t *p; - struct flb_sds *head; - flb_sds_t tmp; - flb_sds_t s; - - s = *sds; - head = FLB_SDS_HEADER(s); - - if (flb_sds_avail(s) <= str_len) { - tmp = flb_sds_increase(s, str_len); - if (tmp == NULL) { - return NULL; - } - *sds = s = tmp; - head = FLB_SDS_HEADER(s); - } - - for (i = 0; i < str_len; i++) { - if (flb_sds_avail(s) < 8) { - tmp = flb_sds_increase(s, 8); - if (tmp == NULL) { - return NULL; - } - *sds = s = tmp; - head = FLB_SDS_HEADER(s); - } - - c = (unsigned char)str[i]; - if (c == '\\' || c == '"') { - s[head->len++] = '\\'; - s[head->len++] = c; - } - else if (c >= '\b' && c <= '\r') { - s[head->len++] = '\\'; - switch (c) { - case '\n': - s[head->len++] = 'n'; - break; - case '\t': - s[head->len++] = 't'; - break; - case '\b': - s[head->len++] = 'b'; - break; - case '\f': - s[head->len++] = 'f'; - break; - case '\r': - s[head->len++] = 'r'; - break; - case '\v': - s[head->len++] = 'u'; - s[head->len++] = '0'; - s[head->len++] = '0'; - s[head->len++] = '0'; - s[head->len++] = 'b'; - break; - } - } - else if (c < 32 || c == 0x7f) { - s[head->len++] = '\\'; - s[head->len++] = 'u'; - s[head->len++] = '0'; - s[head->len++] = '0'; - s[head->len++] = int2hex[ (unsigned char) ((c & 0xf0) >> 4)]; - s[head->len++] = int2hex[ (unsigned char) (c & 0x0f)]; - } - else if (c >= 0x80) { - hex_bytes = flb_utf8_len(str + i); - state = FLB_UTF8_ACCEPT; - cp = 0; - for (b = 0; b < hex_bytes; b++) { - p = (const unsigned char *) str + i + b; - if (p >= (unsigned char *) (str + str_len)) { - break; - } - ret = flb_utf8_decode(&state, &cp, *p); - if (ret == 0) { - break; - } - } - - if (state != FLB_UTF8_ACCEPT) { - /* Invalid UTF-8 hex, just skip utf-8 bytes */ - flb_warn("[pack] invalid UTF-8 bytes, skipping"); - break; - } - - s[head->len++] = '\\'; - s[head->len++] = 'u'; - if (cp > 0xFFFF) { - c = (unsigned char) ((cp & 0xf00000) >> 20); - if (c > 0) { - s[head->len++] = int2hex[c]; - } - c = (unsigned char) ((cp & 0x0f0000) >> 16); - if (c > 0) { - s[head->len++] = int2hex[c]; - } - } - s[head->len++] = int2hex[ (unsigned char) ((cp & 0xf000) >> 12)]; - s[head->len++] = int2hex[ (unsigned char) ((cp & 0x0f00) >> 8)]; - s[head->len++] = int2hex[ (unsigned char) ((cp & 0xf0) >> 4)]; - s[head->len++] = int2hex[ (unsigned char) (cp & 0x0f)]; - i += (hex_bytes - 1); - } - else { - s[head->len++] = c; - } - } - - s[head->len] = '\0'; - - return s; -} - -flb_sds_t flb_sds_printf(flb_sds_t *sds, const char *fmt, ...) -{ - va_list ap; - int len = strlen(fmt)*2; - int size; - flb_sds_t tmp = NULL; - flb_sds_t s; - struct flb_sds *head; - - if (len < 64) len = 64; - - s = *sds; - if (flb_sds_avail(s) < len) { - tmp = flb_sds_increase(s, len - flb_sds_avail(s)); - if (!tmp) { - return NULL; - } - *sds = s = tmp; - } - - va_start(ap, fmt); - size = vsnprintf((char *) (s + flb_sds_len(s)), flb_sds_avail(s), fmt, ap); - if (size < 0) { - flb_warn("[%s] buggy vsnprintf return %d", __FUNCTION__, size); - va_end(ap); - return NULL; - } - va_end(ap); - - if (size >= flb_sds_avail(s)) { - tmp = flb_sds_increase(s, size - flb_sds_avail(s) + 1); - if (!tmp) { - return NULL; - } - *sds = s = tmp; - - va_start(ap, fmt); - size = vsnprintf((char *) (s + flb_sds_len(s)), flb_sds_avail(s), fmt, ap); - if (size > flb_sds_avail(s)) { - flb_warn("[%s] vsnprintf is insatiable ", __FUNCTION__); - va_end(ap); - return NULL; - } - va_end(ap); - } - - head = FLB_SDS_HEADER(s); - head->len += size; - s[head->len] = '\0'; - - return s; -} - -void flb_sds_destroy(flb_sds_t s) -{ - struct flb_sds *head; - - if (!s) { - return; - } - - head = FLB_SDS_HEADER(s); - flb_free(head); -} - -/* - * flb_sds_snprintf is a wrapper of snprintf. - * The difference is that this function can increase the buffer of flb_sds_t. - */ -int flb_sds_snprintf(flb_sds_t *str, size_t size, const char *fmt, ...) -{ - va_list va; - flb_sds_t tmp; - int ret; - - retry_snprintf: - va_start(va, fmt); - ret = vsnprintf(*str, size, fmt, va); - if (ret > size) { - tmp = flb_sds_increase(*str, ret-size); - if (tmp == NULL) { - return -1; - } - *str = tmp; - size = ret; - va_end(va); - goto retry_snprintf; - } - va_end(va); - - flb_sds_len_set(*str, ret); - return ret; -} diff --git a/fluent-bit/src/flb_sds_list.c b/fluent-bit/src/flb_sds_list.c deleted file mode 100644 index 2757ceaac..000000000 --- a/fluent-bit/src/flb_sds_list.c +++ /dev/null @@ -1,185 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <fluent-bit.h> -#include <fluent-bit/flb_sds_list.h> - -size_t flb_sds_list_size(struct flb_sds_list *list) -{ - if (list == NULL) { - return 0; - } - return mk_list_size(&list->strs); -} - -struct flb_sds_list *flb_sds_list_create() -{ - struct flb_sds_list *ret = NULL; - - ret = flb_calloc(1, sizeof(struct flb_sds_list)); - if (ret == NULL) { - return NULL; - } - - mk_list_init(&ret->strs); - - return ret; -} - -int flb_sds_list_del(struct flb_sds_list_entry* entry) -{ - if (entry == NULL) { - return -1; - } - if (entry->str != NULL) { - flb_sds_destroy(entry->str); - } - mk_list_del(&entry->_head); - flb_free(entry); - - return 0; -} - - -int flb_sds_list_destroy(struct flb_sds_list *list) -{ - struct mk_list *tmp = NULL; - struct mk_list *head = NULL; - struct flb_sds_list_entry *entry = NULL; - - if (list == NULL) { - return -1; - } - - mk_list_foreach_safe(head, tmp, &list->strs) { - entry = mk_list_entry(head, struct flb_sds_list_entry, _head); - flb_sds_list_del(entry); - } - flb_free(list); - - return 0; -} - -int flb_sds_list_add(struct flb_sds_list* list, char *in_str, size_t in_size) -{ - flb_sds_t str; - struct flb_sds_list_entry *entry = NULL; - - if (list == NULL || in_str == NULL || in_size == 0) { - return -1; - } - - str = flb_sds_create_len(in_str, in_size); - if (str == NULL) { - return -1; - } - - entry = flb_malloc(sizeof(struct flb_sds_list_entry)); - if (entry == NULL) { - flb_errno(); - flb_sds_destroy(str); - return -1; - } - entry->str = str; - - mk_list_add(&entry->_head, &list->strs); - - return 0; -} - -int flb_sds_list_destroy_str_array(char **array) -{ - char **str = array; - int i = 0; - - if (array == NULL) { - return -1; - } - while(str[i] != NULL) { - flb_free(str[i]); - i++; - } - flb_free(array); - - return 0; -} - - -/* - This function allocates NULL terminated string array from list. - The array should be destroyed by flb_sds_list_destroy_str_array. -*/ -char **flb_sds_list_create_str_array(struct flb_sds_list *list) -{ - int i = 0; - size_t size; - char **ret = NULL; - struct mk_list *tmp = NULL; - struct mk_list *head = NULL; - struct flb_sds_list_entry *entry = NULL; - - if (list == NULL) { - return NULL; - } - - size = flb_sds_list_size(list); - if (size == 0) { - return NULL; - } - - ret = flb_malloc(sizeof(char*) * (size + 1)); - if (ret == NULL) { - flb_errno(); - return NULL; - } - - mk_list_foreach_safe(head, tmp, &list->strs) { - entry = mk_list_entry(head, struct flb_sds_list_entry, _head); - if (entry == NULL) { - flb_free(ret); - return NULL; - } - ret[i] = flb_malloc(flb_sds_len(entry->str)+1); - if (ret[i] == NULL) { - flb_free(ret); - return NULL; - } - strncpy(ret[i], entry->str, flb_sds_len(entry->str)); - ret[i][flb_sds_len(entry->str)] = '\0'; - i++; - } - ret[i] = NULL; - - return ret; -} - -int flb_sds_list_del_last_entry(struct flb_sds_list* list) -{ - struct flb_sds_list_entry *entry = NULL; - - if (list == NULL || flb_sds_list_size(list) == 0) { - return -1; - } - - entry = mk_list_entry_last(&list->strs, struct flb_sds_list_entry, _head); - if (entry == NULL) { - return -1; - } - return flb_sds_list_del(entry); -} diff --git a/fluent-bit/src/flb_signv4.c b/fluent-bit/src/flb_signv4.c deleted file mode 100644 index a4283dc05..000000000 --- a/fluent-bit/src/flb_signv4.c +++ /dev/null @@ -1,1245 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * - * AWS Signv4 documentation - * - * https://docs.aws.amazon.com/general/latest/gr/sigv4-create-canonical-request.html - */ - -#include <fluent-bit/flb_info.h> -#include <fluent-bit/flb_kv.h> -#include <fluent-bit/flb_sds.h> -#include <fluent-bit/flb_utils.h> -#include <fluent-bit/flb_hmac.h> -#include <fluent-bit/flb_hash.h> -#include <fluent-bit/flb_http_client.h> -#include <fluent-bit/flb_signv4.h> -#include <fluent-bit/flb_aws_credentials.h> - -#include <stdlib.h> -#include <ctype.h> - -static flb_sds_t sha256_to_hex(unsigned char *sha256) -{ - int i; - flb_sds_t hex; - flb_sds_t tmp; - - hex = flb_sds_create_size(64); - if (!hex) { - flb_error("[signv4] cannot allocate buffer to convert sha256 to hex"); - return NULL; - } - - for (i = 0; i < 32; i++) { - tmp = flb_sds_printf(&hex, "%02x", sha256[i]); - if (!tmp) { - flb_error("[signedv4] error formatting sha256 -> hex"); - flb_sds_destroy(hex); - return NULL; - } - hex = tmp; - } - - return hex; -} - -static int hmac_sha256_sign(unsigned char out[32], - unsigned char *key, size_t key_len, - unsigned char *msg, size_t msg_len) -{ - int result; - - result = flb_hmac_simple(FLB_HASH_SHA256, - key, key_len, - msg, msg_len, - out, 32); - - if (result != FLB_CRYPTO_SUCCESS) { - return -1; - } - - return 0; -} - -static int kv_key_cmp(const void *a_arg, const void *b_arg) -{ - int ret; - struct flb_kv *kv_a = *(struct flb_kv **) a_arg; - struct flb_kv *kv_b = *(struct flb_kv **) b_arg; - - ret = strcmp(kv_a->key, kv_b->key); - if (ret == 0) { - /* - * NULL pointer is allowed in kv_a->val and kv_b->val. - * Handle NULL pointer cases before passing to strcmp. - */ - if (kv_a->val == NULL && kv_b->val == NULL) { - ret = 0; - } - else if (kv_a->val == NULL) { - ret = -1; - } - else if (kv_b->val == NULL) { - ret = 1; - } - else { - ret = strcmp(kv_a->val, kv_b->val); - } - } - - return ret; -} - -static inline int to_encode(char c) -{ - if ((c >= 48 && c <= 57) || /* 0-9 */ - (c >= 65 && c <= 90) || /* A-Z */ - (c >= 97 && c <= 122) || /* a-z */ - (c == '-' || c == '_' || c == '.' || c == '~' || c == '/' || - c == '=')) { - return FLB_FALSE; - } - - return FLB_TRUE; -} - -static inline int to_encode_path(char c) -{ - if ((c >= 48 && c <= 57) || /* 0-9 */ - (c >= 65 && c <= 90) || /* A-Z */ - (c >= 97 && c <= 122) || /* a-z */ - (c == '-' || c == '_' || c == '.' || c == '~' || c == '/')) { - return FLB_FALSE; - } - - return FLB_TRUE; -} - -flb_sds_t flb_signv4_uri_normalize_path(char *uri, size_t len) -{ - char *p; - int end_slash = FLB_FALSE; - struct mk_list *tmp; - struct mk_list *prev; - struct mk_list *head; - struct mk_list *split; - struct flb_split_entry *entry; - flb_sds_t out; - - if (len == 0) { - return NULL; - } - - out = flb_sds_create_len(uri, len+1); - if (!out) { - return NULL; - } - out[len] = '\0'; - - if (uri[len - 1] == '/') { - end_slash = FLB_TRUE; - } - - split = flb_utils_split(out, '/', -1); - if (!split) { - flb_sds_destroy(out); - return NULL; - } - - p = out; - *p++ = '/'; - - mk_list_foreach_safe(head, tmp, split) { - entry = mk_list_entry(head, struct flb_split_entry, _head); - if (entry->len == 1 && *entry->value == '.') { - flb_utils_split_free_entry(entry); - } - else if (entry->len == 2 && memcmp(entry->value, "..", 2) == 0) { - prev = head->prev; - if (prev != split) { - entry = mk_list_entry(prev, struct flb_split_entry, _head); - flb_utils_split_free_entry(entry); - } - entry = mk_list_entry(head, struct flb_split_entry, _head); - flb_utils_split_free_entry(entry); - } - } - - mk_list_foreach(head, split) { - entry = mk_list_entry(head, struct flb_split_entry, _head); - memcpy(p, entry->value, entry->len); - p += entry->len; - - if (head->next != split) { - *p++ = '/'; - } - } - - len = (p - out); - if (end_slash == FLB_TRUE && out[len - 1] != '/') { - *p++ = '/'; - } - - flb_utils_split_free(split); - - flb_sds_len_set(out, p - out); - out[p - out] = '\0'; - - return out; -} - -static flb_sds_t uri_encode(const char *uri, size_t len) -{ - int i; - flb_sds_t buf = NULL; - flb_sds_t tmp = NULL; - int is_query_string = FLB_FALSE; - int do_encode = FLB_FALSE; - - buf = flb_sds_create_size(len * 2); - if (!buf) { - flb_error("[signv4] cannot allocate buffer for URI encoding"); - return NULL; - } - - for (i = 0; i < len; i++) { - if (uri[i] == '?') { - is_query_string = FLB_TRUE; - } - do_encode = FLB_FALSE; - - if (is_query_string == FLB_FALSE && to_encode_path(uri[i]) == FLB_TRUE) { - do_encode = FLB_TRUE; - } - if (is_query_string == FLB_TRUE && to_encode(uri[i]) == FLB_TRUE) { - do_encode = FLB_TRUE; - } - if (do_encode == FLB_TRUE) { - tmp = flb_sds_printf(&buf, "%%%02X", (unsigned char) *(uri + i)); - if (!tmp) { - flb_error("[signv4] error formatting special character"); - flb_sds_destroy(buf); - return NULL; - } - buf = tmp; - continue; - } - - /* Direct assignment, just copy the character */ - if (buf) { - tmp = flb_sds_cat(buf, uri + i, 1); - if (!tmp) { - flb_error("[signv4] error composing outgoing buffer"); - flb_sds_destroy(buf); - return NULL; - } - buf = tmp; - } - } - - return buf; -} - -/* - * Encodes URI parameters, which can not have "/" characters in them - * (This happens in an STS request, the role ARN has a slash and is - * given as a query parameter). - */ -static flb_sds_t uri_encode_params(const char *uri, size_t len) -{ - int i; - flb_sds_t buf = NULL; - flb_sds_t tmp = NULL; - - buf = flb_sds_create_size(len * 2); - if (!buf) { - flb_error("[signv4] cannot allocate buffer for URI encoding"); - return NULL; - } - - for (i = 0; i < len; i++) { - if (to_encode(uri[i]) == FLB_TRUE || uri[i] == '/') { - tmp = flb_sds_printf(&buf, "%%%02X", (unsigned char) *(uri + i)); - if (!tmp) { - flb_error("[signv4] error formatting special character"); - flb_sds_destroy(buf); - return NULL; - } - continue; - } - - /* Direct assignment, just copy the character */ - if (buf) { - tmp = flb_sds_cat(buf, uri + i, 1); - if (!tmp) { - flb_error("[signv4] error composing outgoing buffer"); - flb_sds_destroy(buf); - return NULL; - } - buf = tmp; - } - } - - return buf; -} - -/* - * Convert URL encoded params (query string or POST payload) to a sorted - * key/value linked list - */ -static flb_sds_t url_params_format(char *params) -{ - int i; - int ret; - int len; - int items; - char *p; - struct mk_list list; - struct mk_list split; - struct mk_list *h_tmp; - struct mk_list *head; - struct flb_slist_entry *e; - flb_sds_t key; - flb_sds_t val; - flb_sds_t tmp; - flb_sds_t buf = NULL; - struct flb_kv *kv; - struct flb_kv **arr; - - mk_list_init(&list); - mk_list_init(&split); - - ret = flb_slist_split_string(&split, params, '&', -1); - if (ret == -1) { - flb_error("[signv4] error processing given query string"); - flb_slist_destroy(&split); - flb_kv_release(&list); - return NULL; - } - - mk_list_foreach_safe(head, h_tmp, &split) { - e = mk_list_entry(head, struct flb_slist_entry, _head); - p = strchr(e->str, '='); - if (!p) { - continue; - } - - len = (p - e->str); - p++; - - /* URI encode every key and value */ - key = uri_encode_params(e->str, len); - len++; - val = uri_encode_params(p, flb_sds_len(e->str) - len); - if (!key || !val) { - flb_error("[signv4] error encoding uri for query string"); - if (key) { - flb_sds_destroy(key); - } - if (val) { - flb_sds_destroy(val); - } - flb_slist_destroy(&split); - flb_kv_release(&list); - return NULL; - } - - /* - * If key length is 0 then a problem occurs because val - * will not be set flb_kv_item_create_len, which eventually - * results in issues since kv->val will be equal to NULL. - * Thus, check here whether key length is satisfied - */ - if (flb_sds_len(key) == 0) { - flb_sds_destroy(key); - flb_sds_destroy(val); - flb_slist_destroy(&split); - flb_kv_release(&list); - return NULL; - } - - kv = flb_kv_item_create_len(&list, - key, flb_sds_len(key), - val, flb_sds_len(val)); - flb_sds_destroy(key); - flb_sds_destroy(val); - - if (!kv) { - flb_error("[signv4] error processing key/value from query string"); - flb_slist_destroy(&split); - flb_kv_release(&list); - return NULL; - } - } - flb_slist_destroy(&split); - - /* Sort the kv list of parameters */ - items = mk_list_size(&list); - if (items == 0) { - flb_kv_release(&list); - return flb_sds_create(""); - } - - arr = flb_calloc(1, sizeof(struct flb_kv *) * items); - if (!arr) { - flb_errno(); - flb_kv_release(&list); - return NULL; - } - - i = 0; - mk_list_foreach(head, &list) { - kv = mk_list_entry(head, struct flb_kv, _head); - arr[i] = kv; - i++; - } - /* sort headers by key */ - qsort(arr, items, sizeof(struct flb_kv *), kv_key_cmp); - - /* Format query string parameters */ - buf = flb_sds_create_size(items * 64); - if (!buf) { - flb_kv_release(&list); - flb_free(arr); - return NULL; - } - - for (i = 0; i < items; i++) { - kv = (struct flb_kv *) arr[i]; - if (i + 1 < items) { - if (kv->val == NULL) { - tmp = flb_sds_printf(&buf, "%s=&", - kv->key); - } - else { - tmp = flb_sds_printf(&buf, "%s=%s&", - kv->key, kv->val); - } - } - else { - if (kv->val == NULL) { - tmp = flb_sds_printf(&buf, "%s=", - kv->key); - } - else { - tmp = flb_sds_printf(&buf, "%s=%s", - kv->key, kv->val); - } - } - if (!tmp) { - flb_error("[signv4] error allocating value"); - } - buf = tmp; - } - - flb_kv_release(&list); - flb_free(arr); - - return buf; -} - -/* - * Given an original list of kv headers with 'in_list' as the list headed, - * generate new entries on 'out_list' considering lower case headers key, - * sorted by keys and values and merged duplicates. - */ -void headers_sanitize(struct mk_list *in_list, struct mk_list *out_list) -{ - int x; - char *v_start; - char *v_end; - char *val; - struct mk_list *head; - struct mk_list *c_head; - struct mk_list *tmp; - struct mk_list out_tmp; - struct flb_kv *kv; - struct flb_kv *c_kv; - flb_sds_t t; - - mk_list_init(&out_tmp); - - /* Create lowercase key headers in the temporal list */ - mk_list_foreach(head, in_list) { - kv = mk_list_entry(head, struct flb_kv, _head); - - /* Sanitize value */ - v_start = kv->val; - v_end = kv->val + flb_sds_len(kv->val); - while (*v_start == ' ' || *v_start == '\t') { - v_start++; - } - while (*v_end == ' ' || *v_end == '\t') { - v_end--; - } - - /* - * The original headers might have upper case characters, for safety just - * make a copy of them so we can lowercase them if required. - */ - kv = flb_kv_item_create_len(&out_tmp, - kv->key, flb_sds_len(kv->key), - v_start, v_end - v_start); - if (kv == NULL) { - continue; - } - for (x = 0; x < flb_sds_len(kv->key); x++) { - kv->key[x] = tolower(kv->key[x]); - } - - /* - * trim: kv->val alreay have a copy of the original value, now we need - * to look for double empty spaces in the middle of the value and do - * proper adjustments. - */ - val = kv->val; - while (v_start < v_end) { - if (*v_start == ' ') { - if (v_start < v_end && *(v_start + 1) == ' ') { - v_start++; - continue; - } - } - *val = *v_start; - v_start++; - val++; - } - *val = '\0'; - flb_sds_len_set(kv->val, val - kv->val); - } - - /* Find and merge duplicates */ - mk_list_foreach_safe(head, tmp, &out_tmp) { - kv = mk_list_entry(head, struct flb_kv, _head); - - /* Check if this kv exists in out_list */ - c_kv = NULL; - mk_list_foreach(c_head, out_list) { - c_kv = mk_list_entry(c_head, struct flb_kv, _head); - if (strcmp(kv->key, c_kv->key) == 0) { - break; - } - c_kv = NULL; - } - - /* if c_kv is set, means the key already exists in the outgoing list */ - if (c_kv) { - t = flb_sds_printf(&c_kv->val, ",%s", kv->val); - c_kv->val = t; - flb_kv_item_destroy(kv); - } - else { - mk_list_del(&kv->_head); - mk_list_add(&kv->_head, out_list); - } - } -} - -/* - * Task 1: Create a canonical request - * ================================== - * - * https://docs.aws.amazon.com/general/latest/gr/sigv4-create-canonical-request.html - * - * CanonicalRequest = - * HTTPRequestMethod + '\n' + - * CanonicalURI + '\n' + - * CanonicalQueryString + '\n' + - * CanonicalHeaders + '\n' + - * SignedHeaders + '\n' + - * HexEncode(Hash(RequestPayload)) - */ -static flb_sds_t flb_signv4_canonical_request(struct flb_http_client *c, - int normalize_uri, - int amz_date_header, - char *amzdate, - char *security_token, - int s3_mode, - struct mk_list *excluded_headers, - flb_sds_t *signed_headers) -{ - int i; - int len; - int items; - int all_items; - int excluded_items; - int post_params = FLB_FALSE; - int result; - size_t size; - int skip_header; - char *val; - struct flb_kv **arr; - flb_sds_t cr; - flb_sds_t uri; - flb_sds_t tmp = NULL; - flb_sds_t params = NULL; - flb_sds_t payload_hash = NULL; - struct flb_kv *kv; - struct mk_list list_tmp; - struct mk_list *head; - struct mk_list *head_2; - struct flb_slist_entry *sle; - unsigned char sha256_buf[64] = {0}; - - /* Size hint */ - size = strlen(c->uri) + (mk_list_size(&c->headers) * 64) + 256; - - cr = flb_sds_create_size(size); - if (!cr) { - flb_error("[signv4] cannot allocate buffer"); - return NULL; - } - - switch (c->method) { - case FLB_HTTP_GET: - tmp = flb_sds_cat(cr, "GET\n", 4); - break; - case FLB_HTTP_POST: - tmp = flb_sds_cat(cr, "POST\n", 5); - break; - case FLB_HTTP_PUT: - tmp = flb_sds_cat(cr, "PUT\n", 4); - break; - case FLB_HTTP_HEAD: - tmp = flb_sds_cat(cr, "HEAD\n", 5); - break; - }; - - if (!tmp) { - flb_error("[signv4] invalid processing of HTTP method"); - flb_sds_destroy(cr); - return NULL; - } - - cr = tmp; - - /* Our URI already contains the query string, so do the proper adjustments */ - if (c->query_string) { - len = (c->query_string - c->uri) - 1; - } - else { - len = strlen(c->uri); - } - - /* - * URI normalization is required by certain AWS service, for hence the caller - * plugin is responsible to enable/disable this flag. If set the URI in the - * canonical request will be normalized. - */ - if (normalize_uri == FLB_TRUE) { - tmp = flb_signv4_uri_normalize_path((char *) c->uri, len); - if (!tmp) { - flb_error("[signv4] error normalizing path"); - flb_sds_destroy(cr); - return NULL; - } - len = flb_sds_len(tmp); - } - else { - tmp = (char *) c->uri; - } - - /* Do URI encoding (rfc3986) */ - uri = uri_encode(tmp, len); - if (tmp != c->uri) { - flb_sds_destroy(tmp); - } - if (!uri) { - /* error composing outgoing buffer */ - flb_sds_destroy(cr); - return NULL; - } - - tmp = flb_sds_cat(cr, uri, flb_sds_len(uri)); - if (!tmp) { - flb_error("[signv4] error concatenating encoded URI"); - flb_sds_destroy(uri); - flb_sds_destroy(cr); - return NULL; - } - cr = tmp; - flb_sds_destroy(uri); - - tmp = flb_sds_cat(cr, "\n", 1); - if (!tmp) { - flb_error("[signv4] error concatenating encoded URI break line"); - flb_sds_destroy(cr); - return NULL; - } - cr = tmp; - - /* Canonical Query String */ - tmp = NULL; - if (c->query_string) { - params = url_params_format((char *) c->query_string); - if (!params) { - flb_sds_destroy(cr); - return NULL; - } - tmp = flb_sds_cat(cr, params, flb_sds_len(params)); - if (!tmp) { - flb_error("[signv4] error concatenating query string"); - flb_sds_destroy(params); - flb_sds_destroy(cr); - return NULL; - } - flb_sds_destroy(params); - cr = tmp; - } - - /* - * If the original HTTP method is POST and we have some urlencoded parameters - * as payload, we must handle them as we did for the query string. - */ - if (c->method == FLB_HTTP_POST && c->body_len > 0) { - val = (char *) flb_kv_get_key_value("Content-Type", &c->headers); - if (val) { - if (strstr(val, "application/x-www-form-urlencoded")) { - params = url_params_format((char *) c->body_buf); - if (!params) { - flb_error("[signv4] error processing POST payload params"); - flb_sds_destroy(cr); - return NULL; - } - tmp = flb_sds_cat(cr, params, flb_sds_len(params)); - if (!tmp) { - flb_error("[signv4] error concatenating POST payload params"); - flb_sds_destroy(params); - flb_sds_destroy(cr); - return NULL; - } - cr = tmp; - flb_sds_destroy(params); - post_params = FLB_TRUE; - } - } - } - - /* query string / POST separator */ - tmp = flb_sds_cat(cr, "\n", 1); - if (!tmp) { - flb_error("[signv4] error adding params breakline separator"); - flb_sds_destroy(cr); - return NULL; - } - cr = tmp; - - /* - * Calculate payload hash. - * This is added at the end of all canonical requests, unless - * S3_MODE_UNSIGNED_PAYLOAD is set. - * If we're using S3_MODE_SIGNED_PAYLOAD, then the hash is added to the - * canonical headers. - */ - if (s3_mode == S3_MODE_UNSIGNED_PAYLOAD) { - payload_hash = flb_sds_create("UNSIGNED-PAYLOAD"); - } else { - if (c->body_len > 0 && post_params == FLB_FALSE) { - result = flb_hash_simple(FLB_HASH_SHA256, - (unsigned char *) c->body_buf, c->body_len, - sha256_buf, sizeof(sha256_buf)); - } - else { - result = flb_hash_simple(FLB_HASH_SHA256, - (unsigned char *) NULL, 0, - sha256_buf, sizeof(sha256_buf)); - } - - if (result != FLB_CRYPTO_SUCCESS) { - flb_error("[signv4] error hashing payload"); - flb_sds_destroy(cr); - return NULL; - } - - payload_hash = flb_sds_create_size(64); - if (!payload_hash) { - flb_error("[signv4] error formatting hashed payload"); - flb_sds_destroy(cr); - return NULL; - } - for (i = 0; i < 32; i++) { - tmp = flb_sds_printf(&payload_hash, "%02x", - (unsigned char) sha256_buf[i]); - if (!tmp) { - flb_error("[signv4] error formatting hashed payload"); - flb_sds_destroy(cr); - flb_sds_destroy(payload_hash); - return NULL; - } - payload_hash = tmp; - } - } - - /* - * Canonical Headers - * - * Add the required custom headers: - * - * - x-amz-date - * - x-amz-security-token (if set) - * - x-amz-content-sha256 (if S3_MODE_SIGNED_PAYLOAD) - */ - mk_list_init(&list_tmp); - - /* include x-amz-date header ? */ - if (amz_date_header == FLB_TRUE) { - len = strlen(amzdate); - flb_http_add_header(c, "x-amz-date", 10, amzdate, len); - } - - /* x-amz-security_token */ - if (security_token) { - len = strlen(security_token); - flb_http_add_header(c, "x-amz-security-token", 20, security_token, len); - } - - if (s3_mode == S3_MODE_SIGNED_PAYLOAD) { - flb_http_add_header(c, "x-amz-content-sha256", 20, payload_hash, 64); - } - - headers_sanitize(&c->headers, &list_tmp); - - /* - * For every header registered, append it to the temporal array so we can sort them - * later. - */ - all_items = mk_list_size(&list_tmp); - excluded_items = 0; - size = (sizeof(struct flb_kv *) * (all_items)); - arr = flb_calloc(1, size); - if (!arr) { - flb_errno(); - flb_kv_release(&list_tmp); - flb_sds_destroy(cr); - flb_sds_destroy(payload_hash); - return NULL; - } - - /* Compose temporal array to sort headers */ - i = 0; - mk_list_foreach(head, &list_tmp) { - kv = mk_list_entry(head, struct flb_kv, _head); - - /* Skip excluded headers */ - if (excluded_headers) { - skip_header = FLB_FALSE; - mk_list_foreach(head_2, excluded_headers) { - sle = mk_list_entry(head_2, struct flb_slist_entry, _head); - if (flb_sds_casecmp(kv->key, sle->str, flb_sds_len(sle->str)) == 0) { - - /* Skip header */ - excluded_items++; - skip_header = FLB_TRUE; - break; - } - } - if (skip_header) { - continue; - } - } - - arr[i] = kv; - i++; - } - - /* Count items */ - items = all_items - excluded_items; - - /* Sort the headers from the temporal array */ - qsort(arr, items, sizeof(struct flb_kv *), kv_key_cmp); - - /* Iterate sorted headers and append them to the outgoing buffer */ - for (i = 0; i < items; i++) { - kv = (struct flb_kv *) arr[i]; - tmp = flb_sds_printf(&cr, "%s:%s\n", kv->key, kv->val); - if (!tmp) { - flb_error("[signv4] error composing canonical headers"); - flb_free(arr); - flb_kv_release(&list_tmp); - flb_sds_destroy(cr); - flb_sds_destroy(payload_hash); - return NULL; - } - cr = tmp; - } - - /* Add required breakline */ - tmp = flb_sds_printf(&cr, "\n"); - if (!tmp) { - flb_error("[signv4] error adding extra breakline separator"); - flb_free(arr); - flb_kv_release(&list_tmp); - flb_sds_destroy(cr); - flb_sds_destroy(payload_hash); - return NULL; - } - cr = tmp; - - /* Signed Headers for canonical request context */ - for (i = 0; i < items; i++) { - kv = (struct flb_kv *) arr[i]; - - /* Check if this is the last header, if so add breakline separator */ - if (i + 1 == items) { - tmp = flb_sds_printf(&cr, "%s\n", kv->key); - } - else { - tmp = flb_sds_printf(&cr, "%s;", kv->key); - } - if (!tmp) { - flb_error("[signv4] error composing canonical signed headers"); - flb_free(arr); - flb_kv_release(&list_tmp); - flb_sds_destroy(cr); - flb_sds_destroy(payload_hash); - return NULL; - } - cr = tmp; - } - - /* Signed Headers for authorization header (Task 4) */ - for (i = 0; i < items; i++) { - kv = (struct flb_kv *) arr[i]; - - /* Check if this is the last header, if so add breakline separator */ - if (i + 1 == items) { - tmp = flb_sds_printf(signed_headers, "%s", kv->key); - } - else { - tmp = flb_sds_printf(signed_headers, "%s;", kv->key); - } - if (!tmp) { - flb_error("[signv4] error composing auth signed headers"); - flb_free(arr); - flb_kv_release(&list_tmp); - flb_sds_destroy(cr); - flb_sds_destroy(payload_hash); - return NULL; - } - *signed_headers = tmp; - } - - flb_free(arr); - flb_kv_release(&list_tmp); - - /* Add Payload Hash */ - tmp = flb_sds_printf(&cr, "%s", payload_hash); - if (!tmp) { - flb_error("[signv4] error adding payload hash"); - flb_sds_destroy(cr); - flb_sds_destroy(payload_hash); - return NULL; - } - cr = tmp; - flb_sds_destroy(payload_hash); - - return cr; -} - -/* - * Task 2 - * ====== - * https://docs.aws.amazon.com/general/latest/gr/sigv4-create-string-to-sign.html - */ -static flb_sds_t flb_signv4_string_to_sign(struct flb_http_client *c, - flb_sds_t cr, char *amzdate, - char *datestamp, char *service, - char *region) -{ - int i; - int result; - flb_sds_t tmp; - flb_sds_t sign; - unsigned char sha256_buf[64] = {0}; - - sign = flb_sds_create_size(256); - if (!sign) { - flb_error("[signv4] cannot create buffer for signature"); - return NULL; - } - - /* Hashing Algorithm */ - tmp = flb_sds_cat(sign, "AWS4-HMAC-SHA256\n", 17); - if (!tmp) { - flb_error("[signv4] cannot add algorithm to signature"); - flb_sds_destroy(sign); - return NULL; - } - sign = tmp; - - /* Amazon date */ - tmp = flb_sds_printf(&sign, "%s\n", amzdate); - if (!tmp) { - flb_error("[signv4] cannot add amz-date to signature"); - flb_sds_destroy(sign); - return NULL; - } - sign = tmp; - - /* Credentials Scope */ - tmp = flb_sds_printf(&sign, "%s/%s/%s/aws4_request\n", - datestamp, region, service); - if (!tmp) { - flb_error("[signv4] cannot add credentials scope to signature"); - flb_sds_destroy(sign); - return NULL; - } - - /* Hash of Canonical Request */ - result = flb_hash_simple(FLB_HASH_SHA256, - (unsigned char *) cr, flb_sds_len(cr), - sha256_buf, sizeof(sha256_buf)); - - if (result != FLB_CRYPTO_SUCCESS) { - flb_error("[signv4] error hashing canonical request"); - flb_sds_destroy(sign); - return NULL; - } - - for (i = 0; i < 32; i++) { - tmp = flb_sds_printf(&sign, "%02x", (unsigned char) sha256_buf[i]); - if (!tmp) { - flb_error("[signv4] error formatting hashed canonical request"); - flb_sds_destroy(sign); - return NULL; - } - sign = tmp; - } - - return sign; -} - -/* - * Task 3 - * ====== - * - * https://docs.aws.amazon.com/general/latest/gr/sigv4-calculate-signature.html - * - * TODO: The signing key could be cached to improve performance - */ -static flb_sds_t flb_signv4_calculate_signature(flb_sds_t string_to_sign, - char *datestamp, char *service, - char *region, char *secret_key) -{ - int len; - int klen = 32; - flb_sds_t tmp; - flb_sds_t key; - unsigned char key_date[32]; - unsigned char key_region[32]; - unsigned char key_service[32]; - unsigned char key_signing[32]; - unsigned char signature[32]; - - /* Compose initial key */ - key = flb_sds_create_size(256); - if (!key) { - flb_error("[signv4] cannot create buffer for signature calculation"); - return NULL; - } - - tmp = flb_sds_printf(&key, "AWS4%s", secret_key); - if (!tmp) { - flb_error("[signv4] error formatting initial key"); - flb_sds_destroy(key); - return NULL; - } - key = tmp; - - /* key_date */ - len = strlen(datestamp); - hmac_sha256_sign(key_date, (unsigned char *) key, flb_sds_len(key), - (unsigned char *) datestamp, len); - flb_sds_destroy(key); - - /* key_region */ - len = strlen(region); - hmac_sha256_sign(key_region, key_date, klen, (unsigned char *) region, len); - - /* key_service */ - len = strlen(service); - hmac_sha256_sign(key_service, key_region, klen, (unsigned char *) service, len); - - /* key_signing */ - hmac_sha256_sign(key_signing, key_service, klen, - (unsigned char *) "aws4_request", 12); - - /* Signature */ - hmac_sha256_sign(signature, key_signing, klen, - (unsigned char *) string_to_sign, flb_sds_len(string_to_sign)); - - return sha256_to_hex(signature); -} - -/* - * Task 4 - * ====== - * - * https://docs.aws.amazon.com/general/latest/gr/sigv4-add-signature-to-request.html - */ -static flb_sds_t flb_signv4_add_authorization(struct flb_http_client *c, - char *access_key, - char *datestamp, - char *region, char *service, - flb_sds_t signed_headers, - flb_sds_t signature) -{ - int ret; - int len; - flb_sds_t tmp; - flb_sds_t header_value; - - header_value = flb_sds_create_size(512); - if (!header_value) { - flb_error("[signv4] cannot allocate buffer for authorization header"); - return NULL; - } - - tmp = flb_sds_printf(&header_value, - "AWS4-HMAC-SHA256 Credential=%s/%s/%s/%s/aws4_request, " - "SignedHeaders=%s, Signature=%s", - access_key, datestamp, region, service, - signed_headers, signature); - if (!tmp) { - flb_error("[signv4] error composing authorization header"); - flb_sds_destroy(header_value); - return NULL; - } - header_value = tmp; - - len = flb_sds_len(header_value); - ret = flb_http_add_header(c, "Authorization", 13, header_value, len); - if (ret == -1) { - flb_error("[signv4] could not add authorization header"); - flb_sds_destroy(header_value); - return NULL; - - } - - /* Return the composed final header for testing if required */ - return header_value; -} - -flb_sds_t flb_signv4_do(struct flb_http_client *c, int normalize_uri, - int amz_date_header, - time_t t_now, - char *region, char *service, - int s3_mode, - struct mk_list *unsigned_headers, - struct flb_aws_provider *provider) -{ - char amzdate[32]; - char datestamp[32]; - struct tm *gmt; - flb_sds_t cr; - flb_sds_t string_to_sign; - flb_sds_t signature; - flb_sds_t signed_headers; - flb_sds_t auth_header; - struct flb_aws_credentials *creds; - - creds = provider->provider_vtable->get_credentials(provider); - if (!creds) { - flb_error("[signv4] Provider returned no credentials, service=%s", - service); - return NULL; - } - - gmt = flb_calloc(1, sizeof(struct tm)); - if (!gmt) { - flb_errno(); - flb_aws_credentials_destroy(creds); - return NULL; - } - - if (!gmtime_r(&t_now, gmt)) { - flb_error("[signv4] error converting given unix timestamp"); - flb_free(gmt); - flb_aws_credentials_destroy(creds); - return NULL; - } - - strftime(amzdate, sizeof(amzdate) - 1, "%Y%m%dT%H%M%SZ", gmt); - strftime(datestamp, sizeof(datestamp) - 1, "%Y%m%d", gmt); - flb_free(gmt); - - /* Task 1: canonical request */ - signed_headers = flb_sds_create_size(256); - if (!signed_headers) { - flb_error("[signedv4] cannot allocate buffer for auth signed headers"); - flb_aws_credentials_destroy(creds); - return NULL; - } - - cr = flb_signv4_canonical_request(c, normalize_uri, - amz_date_header, amzdate, - creds->session_token, s3_mode, - unsigned_headers, - &signed_headers); - if (!cr) { - flb_error("[signv4] failed canonical request"); - flb_sds_destroy(signed_headers); - flb_aws_credentials_destroy(creds); - return NULL; - } - - /* Task 2: string to sign */ - string_to_sign = flb_signv4_string_to_sign(c, cr, amzdate, - datestamp, service, region); - if (!string_to_sign) { - flb_error("[signv4] failed string to sign"); - flb_sds_destroy(cr); - flb_sds_destroy(signed_headers); - flb_aws_credentials_destroy(creds); - return NULL; - } - flb_sds_destroy(cr); - - /* Task 3: calculate the signature */ - signature = flb_signv4_calculate_signature(string_to_sign, datestamp, - service, region, - creds->secret_access_key); - if (!signature) { - flb_error("[signv4] failed calculate_string"); - flb_sds_destroy(signed_headers); - flb_sds_destroy(string_to_sign); - flb_aws_credentials_destroy(creds); - return NULL; - } - flb_sds_destroy(string_to_sign); - - /* Task 4: add signature to HTTP request */ - auth_header = flb_signv4_add_authorization(c, - creds->access_key_id, - datestamp, region, service, - signed_headers, signature); - flb_sds_destroy(signed_headers); - flb_sds_destroy(signature); - flb_aws_credentials_destroy(creds); - - if (!auth_header) { - flb_error("[signv4] error creating authorization header"); - return NULL; - } - - return auth_header; -} diff --git a/fluent-bit/src/flb_slist.c b/fluent-bit/src/flb_slist.c deleted file mode 100644 index 512fe4522..000000000 --- a/fluent-bit/src/flb_slist.c +++ /dev/null @@ -1,356 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <fluent-bit/flb_info.h> -#include <fluent-bit/flb_log.h> -#include <fluent-bit/flb_mem.h> -#include <fluent-bit/flb_sds.h> -#include <fluent-bit/flb_slist.h> - -/* Initialize slist */ -int flb_slist_create(struct mk_list *list) -{ - mk_list_init(list); - return 0; -} - -/* Append 'len' bytes of 'str' as a new string node into the list */ -int flb_slist_add_n(struct mk_list *head, const char *str, int len) -{ - struct flb_slist_entry *e; - - e = flb_malloc(sizeof(struct flb_slist_entry)); - if (!e) { - flb_errno(); - return -1; - } - - e->str = flb_sds_create_len(str, len); - if (!e->str) { - flb_free(e); - return -1; - } - - mk_list_add(&e->_head, head); - return 0; -} - -int flb_slist_add_sds(struct mk_list *head, flb_sds_t str) -{ - struct flb_slist_entry *e; - - e = flb_malloc(sizeof(struct flb_slist_entry)); - if (!e) { - flb_errno(); - return -1; - } - - e->str = str; - mk_list_add(&e->_head, head); - return 0; -} - -/* Append NULL terminated string as a new node into the list */ -int flb_slist_add(struct mk_list *head, const char *str) -{ - int len; - - if (!str) { - return -1; - } - - len = strlen(str); - if (len <= 0) { - return -1; - } - - return flb_slist_add_n(head, str, len); -} - -static inline int token_unescape(char *token) -{ - char *in = token; - char *out = token; - - while (*in) { - if ((in[0] == '\\') && (in[1] == '"')) { - *out = in[1]; - out++; - in += 2; - } - else { - *out = *in; - out++; - in++; - } - } - *out = 0; - return out - token; -} - -static flb_sds_t token_retrieve(char **str) -{ - int len; - int quoted = FLB_FALSE; - char *p; - char *start; - char *prev; - flb_sds_t out = NULL; - - if (!*str) { - return NULL; - } - - p = *str; - - /* Skip empty spaces */ - while (*p == ' ') { - p++; - } - start = p; - - if (*p == '"') { - quoted = FLB_TRUE; - p++; - start = p; - while (1) { - while (*p && *p != '"') { - p++; - } - - if (!*p) { - goto exit; - } - - prev = p - 1; - if (*prev == '\\') { - p++; - continue; - } - goto exit; - } - } - - while (*p && *p != ' ') { - p++; - } - - exit: - if (*p) { - out = flb_sds_create_len(start, p - start); - if (!out) { - *str = NULL; - return NULL; - } - if (quoted == FLB_TRUE) { - len = token_unescape(out); - flb_sds_len_set(out, len); - } - p++; - - while (*p && *p == ' ') { - p++; - } - *str = p; - } - else { - if (p > start) { - out = flb_sds_create(start); - } - *str = NULL; - } - - return out; -} - -int flb_slist_split_tokens(struct mk_list *list, const char *str, int max_split) -{ - int count = 0; - char *p; - char *buf; - flb_sds_t tmp = NULL; - - buf = (char *) str; - while ((tmp = token_retrieve(&buf))) { - flb_slist_add_sds(list, tmp); - if (!buf) { - break; - } - count++; - - /* Append remaining string if we use a maximum number of tokens */ - if (count >= max_split && max_split > 0) { - p = buf; - while (*p == ' ') { - p++; - } - - if (*p) { - flb_slist_add(list, p); - } - break; - } - } - - return 0; -} - -/* - * Split a string using a separator, every splitted content is appended to the end of - * the slist list head. - */ -int flb_slist_split_string(struct mk_list *list, const char *str, - int separator, int max_split) -{ - int i = 0; - int ret; - int count = 0; - int val_len; - int len; - int end; - char *p_init; - char *p_end; - - if (!str) { - return -1; - } - - len = strlen(str); - while (i < len) { - end = mk_string_char_search(str + i, separator, len - i); - if (end < 0) { - end = len - i; - } - else if ((end + i) == i) { - i++; - continue; - } - - p_init = (char *) str + i; - p_end = p_init + end - 1; - - /* Skip empty spaces */ - while (*p_init == ' ') { - p_init++; - } - - while (*p_end == ' ' && p_end >= p_init) { - p_end--; - } - - if (p_init > p_end) { - goto next; - } - - if (p_init == p_end) { - if (*p_init == ' ') { - goto next; - } - val_len = 1; - } - else { - val_len = (p_end - p_init) + 1; - } - - if (val_len == 0) { - goto next; - } - - ret = flb_slist_add_n(list, p_init, val_len); - if (ret == -1) { - return -1; - } - count++; - - /* Append remaining string as a new node ? */ - if (count >= max_split && max_split > 0) { - p_end = p_init + end; - if (*p_end == separator) { - p_end++; - } - while (*p_end == ' ') { - p_end++; - } - - if ((p_end - str) >= len) { - break; - } - - ret = flb_slist_add(list, p_end); - if (ret == -1) { - return -1; - } - count++; - break; - } - - next: - i += end + 1; - } - - return count; -} - -void flb_slist_dump(struct mk_list *list) -{ - struct mk_list *head; - struct flb_slist_entry *e; - - printf("[slist %p]\n", list); - mk_list_foreach(head, list) { - e = mk_list_entry(head, struct flb_slist_entry, _head); - printf(" - '%s'\n", e->str); - } -} - -void flb_slist_destroy(struct mk_list *list) -{ - struct mk_list *tmp; - struct mk_list *head; - struct flb_slist_entry *e; - - mk_list_foreach_safe(head, tmp, list) { - e = mk_list_entry(head, struct flb_slist_entry, _head); - flb_sds_destroy(e->str); - mk_list_del(&e->_head); - flb_free(e); - } -} - -/* Return the entry in position number 'n' */ -struct flb_slist_entry *flb_slist_entry_get(struct mk_list *list, int n) -{ - int i = 0; - struct mk_list *head; - struct flb_slist_entry *e; - - if (!list || mk_list_size(list) == 0) { - return NULL; - } - - mk_list_foreach(head, list) { - if (i == n) { - e = mk_list_entry(head, struct flb_slist_entry, _head); - return e; - } - i++; - } - - return NULL; -} diff --git a/fluent-bit/src/flb_snappy.c b/fluent-bit/src/flb_snappy.c deleted file mode 100644 index 865e18399..000000000 --- a/fluent-bit/src/flb_snappy.c +++ /dev/null @@ -1,344 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <fluent-bit/flb_info.h> -#include <fluent-bit/flb_mem.h> -#include <fluent-bit/flb_log.h> - -#include <cfl/cfl.h> -#include <cfl/cfl_list.h> -#include <cfl/cfl_checksum.h> - -#include <fluent-bit/flb_snappy.h> - -#include <snappy.h> - -int flb_snappy_compress(char *in_data, size_t in_len, - char **out_data, size_t *out_len) -{ - struct snappy_env snappy_env; - char *tmp_data; - size_t tmp_len; - int result; - - tmp_len = snappy_max_compressed_length(in_len); - - tmp_data = flb_malloc(tmp_len); - - if (tmp_data == NULL) { - flb_errno(); - - return -1; - } - - result = snappy_init_env(&snappy_env); - - if (result != 0) { - flb_free(tmp_data); - - return -2; - } - - result = snappy_compress(&snappy_env, in_data, in_len, tmp_data, &tmp_len); - - if (result != 0) { - flb_free(tmp_data); - - return -3; - } - - snappy_free_env(&snappy_env); - - *out_data = tmp_data; - *out_len = tmp_len; - - return 0; -} - -int flb_snappy_uncompress(char *in_data, size_t in_len, - char **out_data, size_t *out_len) -{ - char *tmp_data; - size_t tmp_len; - int result; - - result = snappy_uncompressed_length(in_data, in_len, &tmp_len); - - if (result == 0) { - return -1; - } - - tmp_data = flb_malloc(tmp_len); - - if (tmp_data == NULL) { - flb_errno(); - - return -2; - } - - result = snappy_uncompress(in_data, in_len, tmp_data); - - if (result != 0) { - flb_free(tmp_data); - - return -3; - } - - *out_data = tmp_data; - *out_len = tmp_len; - - return 0; -} - -static uint32_t calculate_checksum(char *buffer, size_t length) -{ - uint32_t checksum; - - checksum = cfl_checksum_crc32c((unsigned char *) buffer, length); - - return ((checksum >> 15) | - (checksum << 17)) + 0xa282ead8; -} - -int flb_snappy_uncompress_framed_data(char *in_data, size_t in_len, - char **out_data, size_t *out_len) -{ - uint32_t decompressed_data_checksum; - size_t stream_identifier_length; - size_t uncompressed_chunk_count; - int stream_identifier_found; - char *aggregated_data_buffer; - size_t aggregated_data_length = 0; - size_t aggregated_data_offset; - size_t compressed_chunk_count; - struct cfl_list *iterator_backup; - uint32_t frame_checksum; - char *frame_buffer; - size_t frame_length; - char *frame_body; - unsigned char frame_type; - struct cfl_list *iterator; - int result; - size_t offset; - struct cfl_list chunks; - struct flb_snappy_data_chunk *chunk; - - if (*((uint8_t *) in_data) != FLB_SNAPPY_FRAME_TYPE_STREAM_IDENTIFIER) { - return flb_snappy_uncompress(in_data, in_len, out_data, out_len); - } - - if (out_data == NULL) { - return -1; - } - - if (out_len == NULL) { - return -1; - } - - *out_data = NULL; - *out_len = 0; - - cfl_list_init(&chunks); - - compressed_chunk_count = 0; - uncompressed_chunk_count = 0; - - stream_identifier_found = FLB_FALSE; - stream_identifier_length = strlen(FLB_SNAPPY_STREAM_IDENTIFIER_STRING); - - result = 0; - offset = 0; - - while (offset < in_len && result == 0) { - frame_buffer = &in_data[offset]; - - frame_type = *((uint8_t *) &frame_buffer[0]); - - frame_length = *((uint32_t *) &frame_buffer[1]); - frame_length &= 0x00FFFFFF; - - frame_body = &frame_buffer[4]; - - if (frame_length > FLB_SNAPPY_FRAME_SIZE_LIMIT) { - result = -2; - } - else if (frame_type == FLB_SNAPPY_FRAME_TYPE_STREAM_IDENTIFIER) { - if (!stream_identifier_found) { - if (frame_length == stream_identifier_length) { - result = strncmp(frame_body, - FLB_SNAPPY_STREAM_IDENTIFIER_STRING, - stream_identifier_length); - - if (result == 0) { - stream_identifier_found = FLB_TRUE; - } - } - } - } - else if (frame_type == FLB_SNAPPY_FRAME_TYPE_COMPRESSED_DATA) { - chunk = (struct flb_snappy_data_chunk * ) \ - flb_calloc(1, sizeof(struct flb_snappy_data_chunk)); - - if (chunk != NULL) { - /* We add the chunk to the list now because that way - * even if the process fails we can clean up in a single - * place. - */ - compressed_chunk_count++; - - chunk->dynamically_allocated_buffer = FLB_TRUE; - - cfl_list_add(&chunk->_head, &chunks); - - frame_checksum = *((uint32_t *) &frame_body[0]); - frame_body = &frame_body[4]; - - result = flb_snappy_uncompress( - frame_body, - frame_length - sizeof(uint32_t), - &chunk->buffer, - &chunk->length); - - /* decompressed data */ - if (result == 0) { - decompressed_data_checksum = calculate_checksum( - chunk->buffer, - chunk->length); - - if (decompressed_data_checksum != frame_checksum) { - result = -3; - } - else { - aggregated_data_length += chunk->length; - } - } - else { - result = -4; - } - } - } - else if (frame_type == FLB_SNAPPY_FRAME_TYPE_UNCOMPRESSED_DATA) { - chunk = (struct flb_snappy_data_chunk *) \ - flb_calloc(1, sizeof(struct flb_snappy_data_chunk)); - - if (chunk != NULL) { - /* We add the chunk to the list now because that way - * even if the process fails we can clean up in a single - * place. - */ - uncompressed_chunk_count++; - - chunk->dynamically_allocated_buffer = FLB_FALSE; - - cfl_list_add(&chunk->_head, &chunks); - - frame_checksum = *((uint32_t *) &frame_body[0]); - frame_body = &frame_body[4]; - - chunk->buffer = frame_body; - chunk->length = frame_length - sizeof(uint32_t); - - decompressed_data_checksum = calculate_checksum( - chunk->buffer, - chunk->length); - - if (decompressed_data_checksum != frame_checksum) { - result = -3; - } - else { - aggregated_data_length += chunk->length; - } - } - } - else if (frame_type == FLB_SNAPPY_FRAME_TYPE_PADDING) { - /* We just need to skip these frames */ - } - else if (frame_type >= FLB_SNAPPY_FRAME_TYPE_RESERVED_UNSKIPPABLE_BASE && - frame_type <= FLB_SNAPPY_FRAME_TYPE_RESERVED_UNSKIPPABLE_TOP) { - result = -5; - } - else if (frame_type >= FLB_SNAPPY_FRAME_TYPE_RESERVED_SKIPPABLE_BASE && - frame_type <= FLB_SNAPPY_FRAME_TYPE_RESERVED_SKIPPABLE_TOP) { - /* We just need to skip these frames */ - } - - offset += frame_length + 4; - } - - aggregated_data_buffer = NULL; - aggregated_data_length = 0; - - if (compressed_chunk_count == 1 && - uncompressed_chunk_count == 0 && - result == 0) { - /* This is a "past path" to avoid unnecessarily copying - * data whene the input is only comprised of a single - * compressed chunk. - */ - - chunk = cfl_list_entry_first(&chunks, - struct flb_snappy_data_chunk, _head); - - aggregated_data_buffer = chunk->buffer; - aggregated_data_length = chunk->length; - aggregated_data_offset = aggregated_data_length; - - flb_free(chunk); - } - else { - if (aggregated_data_length > 0) { - aggregated_data_buffer = flb_calloc(aggregated_data_length, - sizeof(char)); - - if (aggregated_data_buffer == NULL) { - result = -6; - } - } - - aggregated_data_offset = 0; - cfl_list_foreach_safe(iterator, iterator_backup, &chunks) { - chunk = cfl_list_entry(iterator, - struct flb_snappy_data_chunk, _head); - - if (chunk->buffer != NULL) { - if (aggregated_data_buffer != NULL && - result == 0) { - memcpy(&aggregated_data_buffer[aggregated_data_offset], - chunk->buffer, - chunk->length); - - aggregated_data_offset += chunk->length; - } - - if (chunk->dynamically_allocated_buffer) { - flb_free(chunk->buffer); - } - } - - cfl_list_del(&chunk->_head); - - flb_free(chunk); - } - } - - *out_data = (char *) aggregated_data_buffer; - *out_len = aggregated_data_offset; - - return result; -} diff --git a/fluent-bit/src/flb_socket.c b/fluent-bit/src/flb_socket.c deleted file mode 100644 index daf995e37..000000000 --- a/fluent-bit/src/flb_socket.c +++ /dev/null @@ -1,46 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <fluent-bit/flb_info.h> -#include <fluent-bit/flb_log.h> -#include <fluent-bit/flb_socket.h> - -#ifndef _WIN32 - -int flb_socket_error(int fd) -{ - int ret; - int error = 0; - socklen_t slen = sizeof(error); - - ret = getsockopt(fd, SOL_SOCKET, SO_ERROR, &error, &slen); - if (ret == -1) { - flb_debug("[socket] could not validate socket status for #%i (don't worry)", - fd); - return -1; - } - - if (error != 0) { - return error; - } - - return 0; -} - -#endif diff --git a/fluent-bit/src/flb_sosreport.c b/fluent-bit/src/flb_sosreport.c deleted file mode 100644 index 7bbd79aef..000000000 --- a/fluent-bit/src/flb_sosreport.c +++ /dev/null @@ -1,322 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <fluent-bit/flb_compat.h> -#include <fluent-bit/flb_info.h> -#include <fluent-bit/flb_config.h> -#include <fluent-bit/flb_input.h> -#include <fluent-bit/flb_output.h> -#include <fluent-bit/flb_filter.h> -#include <fluent-bit/flb_router.h> -#include <fluent-bit/flb_version.h> -#include <fluent-bit/flb_utils.h> -#include <fluent-bit/flb_kv.h> - -#ifndef _MSC_VER -#include <sys/utsname.h> -#endif - -static void print_key(char *key) -{ - printf(" %-20s", key); -} - -static void print_kv(char *key, char *val) -{ - print_key(key); - printf("%s\n", val); -} - -static char *get_str(char *p) -{ - if (p) { - return p; - } - - return "(not set)"; -} - - - -static char *log_level(int x) -{ - switch (x) { - case 0: return "Off"; - case 1: return "Error"; - case 2: return "Warn"; - case 3: return "Info"; - case 4: return "Debug"; - case 5: return "Trace"; - default: return "Unknown"; - } -} - -static void input_flags(int flags) -{ - if (flags & FLB_INPUT_NET) { - printf("NET "); - } - - if (flags & FLB_INPUT_CORO) { - printf("CORO "); - } - - printf("\n"); -} - -static void print_host(struct flb_net_host *host) -{ - if (host->address) { - printf(" Host.Address\t%s\n", host->address); - } - if (host->port > 0) { - printf(" Host.TCP_Port\t%i\n", host->port); - } - if (host->name) { - printf(" Host.Name\t\t%s\n", host->name); - } - if (host->listen) { - printf(" Host.Listen\t\t%s\n", host->listen); - } -} - -static void print_properties(struct mk_list *props) -{ - struct mk_list *head; - struct flb_kv *kv; - - mk_list_foreach(head, props) { - kv = mk_list_entry(head, struct flb_kv, _head); - print_kv(kv->key, kv->val); - } -} - -#ifdef _MSC_VER -/* A definition table of SYSTEM_INFO.wProcessorArchitecture. - * - * This is a streight-forward translation of the official manual. - * https://docs.microsoft.com/en-us/windows/desktop/api/sysinfoapi/ - */ -static const char* win32_arch(int archid) -{ - switch(archid) - { - case PROCESSOR_ARCHITECTURE_AMD64: - return "x64 (AMD or Intel)"; - case PROCESSOR_ARCHITECTURE_ARM: - return "ARM"; - case PROCESSOR_ARCHITECTURE_ARM64: - return "ARM64"; - case PROCESSOR_ARCHITECTURE_IA64: - return "Intel Itanium-based"; - case PROCESSOR_ARCHITECTURE_INTEL: - return "x86"; - case PROCESSOR_ARCHITECTURE_UNKNOWN: - default: - return "unknown"; - } -} - -static void win32_operating_system_info() -{ - OSVERSIONINFOA win32os; - - /* TODO Support "Application Manifest". Windows 10 reports a wrong - * version info if we do not manifest the supported OS. - * https://blogs.msdn.microsoft.com/chuckw/2013/09/10/manifest-madness/ - */ - win32os.dwOSVersionInfoSize = sizeof(OSVERSIONINFOA); - GetVersionExA(&win32os); - - printf("[Operating System]\n"); - printf(" Name\t\tWindows\n"); - printf(" Version\t\t%i.%i\n", win32os.dwMajorVersion, win32os.dwMinorVersion); - printf(" Build\t\t%i\n", win32os.dwBuildNumber); - printf("\n"); -} - -static void win32_hardware_info() -{ - SYSTEM_INFO win32info; - - GetNativeSystemInfo(&win32info); - printf("[Hardware]\n"); - printf(" Architecture\t%s\n", win32_arch(win32info.wProcessorArchitecture)); - printf(" Processors\t\t%i\n", win32info.dwNumberOfProcessors); - printf("\n"); -} -#endif - -int flb_sosreport(struct flb_config *config) -{ - char tmp[32]; - struct mk_list *head; - struct mk_list *head_r; - struct flb_input_plugin *in; - struct flb_filter_plugin *filter; - struct flb_output_plugin *out; - struct flb_input_instance *ins_in; - struct flb_filter_instance *ins_filter; - struct flb_output_instance *ins_out; - struct flb_router_path *route; - - printf("\n"); - printf("Fluent Bit Enterprise - SOS Report\n"); - printf("==================================\n"); - printf("The following report aims to be used by Fluent Bit and Fluentd " - "community users.\n\n"); - - /* Fluent Bit */ - printf("\n[Fluent Bit]\n"); - printf(" Version\t\t%s\n", FLB_VERSION_STR); - printf(" Built Flags\t\t%s\n", FLB_INFO_FLAGS); - printf("\n"); - -#ifndef _MSC_VER - struct utsname uts; - uname(&uts); - - /* Operating System */ - printf("[Operating System]\n"); - printf(" Name\t\t%s\n", uts.sysname); - printf(" Release\t\t%s\n", uts.release); - printf(" Version\t\t%s\n", uts.version); - printf("\n"); - - /* Basic hardware info */ - printf("[Hardware]\n"); - printf(" Architecture\t%s\n", uts.machine); - printf(" Processors\t\t%i\n", (int) sysconf(_SC_NPROCESSORS_ONLN)); - printf("\n"); -#else - win32_operating_system_info(); - win32_hardware_info(); -#endif - - /* Fluent Bit */ - printf("[Built Plugins]\n"); - print_key("Inputs"); - mk_list_foreach(head, &config->in_plugins) { - in = mk_list_entry(head, struct flb_input_plugin, _head); - printf("%s ", in->name); - } - printf("\n"); - - print_key("Filters"); - mk_list_foreach(head, &config->filter_plugins) { - filter = mk_list_entry(head, struct flb_filter_plugin, _head); - printf("%s ", filter->name); - } - printf("\n"); - - print_key("Outputs"); - mk_list_foreach(head, &config->out_plugins) { - out = mk_list_entry(head, struct flb_output_plugin, _head); - printf("%s ", out->name); - } - printf("\n"); - - /* Runtime configuration, what do the Engine have before to start */ - printf("\n"); - - /* Config: [SERVER] */ - printf("[SERVER] Runtime configuration\n"); - printf(" Flush\t\t%f\n", config->flush); - printf(" Daemon\t\t%s\n", config->daemon ? "On": "Off"); - printf(" Log_Level\t\t%s\n", log_level(config->verbose)); - printf("\n"); - - /* Config: [INPUT] */ - mk_list_foreach(head, &config->inputs) { - ins_in = mk_list_entry(head, struct flb_input_instance, _head); - printf("[INPUT] Instance\n"); - printf(" Name\t\t%s (%s, id=%i)\n", ins_in->name, ins_in->p->name, - ins_in->id); - printf(" Flags\t\t"); input_flags(ins_in->flags); - printf(" Coroutines\t\t%s\n", ins_in->runs_in_coroutine ? "Yes": "No"); - if (ins_in->tag) { - printf(" Tag\t\t\t%s\n", ins_in->tag); - } - if (ins_in->flags & FLB_INPUT_NET) { - print_host(&ins_in->host); - } - - if (ins_in->mem_buf_limit > 0) { - flb_utils_bytes_to_human_readable_size(ins_in->mem_buf_limit, - tmp, sizeof(tmp) - 1); - printf(" Mem_Buf_Limit\t%s\n", tmp); - } - - print_properties(&ins_in->properties); - - /* Fixed Routes */ - if (mk_list_is_empty(&ins_in->routes) != 0) { - printf(" Routes\t\t"); - mk_list_foreach(head_r, &ins_in->routes) { - route = mk_list_entry(head_r, struct flb_router_path, _head); - printf("%s ", route->ins->name); - } - printf("\n"); - } - printf("\n"); - } - - /* Config: [FILTER] */ - mk_list_foreach(head, &config->filters) { - ins_filter = mk_list_entry(head, struct flb_filter_instance, _head); - printf("[FILTER] Instance\n"); - printf(" Name\t\t%s (%s, id=%i)\n", ins_filter->name, ins_filter->p->name, - ins_filter->id); - printf(" Match\t\t%s\n", ins_filter->match); - print_properties(&ins_filter->properties); - } - printf("\n"); - - /* Config: [OUTPUT] */ - mk_list_foreach(head, &config->outputs) { - ins_out = mk_list_entry(head, struct flb_output_instance, _head); - printf("[OUTPUT] Instance\n"); - printf(" Name\t\t%s (%s, id=%" PRIu64 ")\n", ins_out->name, ins_out->p->name, - (uint64_t) ins_out->id); - printf(" Match\t\t%s\n", ins_out->match); - -#ifdef FLB_HAVE_TLS - printf(" TLS Active\t\t%s\n", ins_out->use_tls ? "Yes" : "No"); - if (ins_out->use_tls == FLB_TRUE) { - printf(" TLS.Verify\t\t%s\n", ins_out->tls_verify ? "On": "Off"); - printf(" TLS.Ca_File\t\t%s\n", get_str(ins_out->tls_ca_file)); - printf(" TLS.Crt_File\t%s\n", get_str(ins_out->tls_crt_file)); - printf(" TLS.Key_File\t%s\n", get_str(ins_out->tls_key_file)); - printf(" TLS.Key_Passwd\t%s\n", - ins_out->tls_key_passwd ? "*****" : "(not set)"); - } -#endif - if (ins_out->retry_limit == FLB_OUT_RETRY_UNLIMITED) { - printf(" Retry Limit\t\tno limit\n"); - } - else { - printf(" Retry Limit\t\t%i\n", ins_out->retry_limit); - } - print_host(&ins_out->host); - print_properties(&ins_out->properties); - printf("\n"); - } - - return 0; -} diff --git a/fluent-bit/src/flb_sqldb.c b/fluent-bit/src/flb_sqldb.c deleted file mode 100644 index 6633501e0..000000000 --- a/fluent-bit/src/flb_sqldb.c +++ /dev/null @@ -1,132 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <fluent-bit/flb_mem.h> -#include <fluent-bit/flb_str.h> -#include <fluent-bit/flb_log.h> -#include <fluent-bit/flb_sqldb.h> - -/* - * Open or create a new database. Note that this function will always try to - * use an open database and share it handler in as a new context. - */ -struct flb_sqldb *flb_sqldb_open(const char *path, const char *desc, - struct flb_config *config) -{ - int ret; - struct mk_list *head; - struct flb_sqldb *db_temp = NULL; - struct flb_sqldb *db; - sqlite3 *sdb = NULL; - - db = flb_malloc(sizeof(struct flb_sqldb)); - if (!db) { - flb_errno(); - return NULL; - } - db->parent = NULL; - db->shared = FLB_FALSE; - db->users = 0; - - /* - * The database handler can be shared across different instances of - * Fluent Bit. Before to open a new one, try to find a database that - * is already open. - */ - mk_list_foreach(head, &config->sqldb_list) { - db_temp = mk_list_entry(head, struct flb_sqldb, _head); - - /* Only lookup for original database, not contexts already shared */ - if (db_temp->shared == FLB_TRUE) { - continue; - } - - if (strcmp(db_temp->path, path) == 0) { - break; - } - db_temp = NULL; - } - - /* Found a database that can be shared */ - if (db_temp) { - /* Increase users counter */ - db_temp->users++; - - /* Setup the new context */ - db->handler = db_temp->handler; - db->shared = FLB_TRUE; - db->parent = db_temp; - } - else { - ret = sqlite3_open(path, &sdb); - if (ret) { - flb_error("[sqldb] cannot open database %s", path); - flb_free(db); - return NULL; - } - db->handler = sdb; - } - - db->path = flb_strdup(path); - db->desc = flb_strdup(desc); - mk_list_add(&db->_head, &config->sqldb_list); - - return db; -} - -int flb_sqldb_close(struct flb_sqldb *db) -{ - struct flb_sqldb *parent; - - if (db->shared == FLB_TRUE) { - parent = db->parent; - parent->users--; - } - else { - sqlite3_close(db->handler); - } - mk_list_del(&db->_head); - flb_free(db->path); - flb_free(db->desc); - flb_free(db); - - return 0; -} - -int flb_sqldb_query(struct flb_sqldb *db, const char *sql, - int (*callback) (void *, int, char **, char **), - void *data) -{ - int ret; - char *err_msg = NULL; - - ret = sqlite3_exec(db->handler, sql, callback, data, &err_msg); - if (ret != SQLITE_OK) { - flb_error("[sqldb] error=%s", err_msg); - sqlite3_free(err_msg); - return FLB_ERROR; - } - - return FLB_OK; -} - -int64_t flb_sqldb_last_id(struct flb_sqldb *db) -{ - return sqlite3_last_insert_rowid(db->handler); -} diff --git a/fluent-bit/src/flb_storage.c b/fluent-bit/src/flb_storage.c deleted file mode 100644 index a89a4c4b5..000000000 --- a/fluent-bit/src/flb_storage.c +++ /dev/null @@ -1,718 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <fluent-bit/flb_info.h> -#include <fluent-bit/flb_input.h> -#include <fluent-bit/flb_log.h> -#include <fluent-bit/flb_storage.h> -#include <fluent-bit/flb_scheduler.h> -#include <fluent-bit/flb_utils.h> -#include <fluent-bit/flb_http_server.h> - -static struct cmt *metrics_context_create(struct flb_storage_metrics *sm) -{ - struct cmt *cmt; - - cmt = cmt_create(); - if (!cmt) { - return NULL; - } - - sm->cmt_chunks = cmt_gauge_create(cmt, - "fluentbit", "storage", "chunks", - "Total number of chunks in the storage layer.", - 0, (char *[]) { NULL }); - - sm->cmt_mem_chunks = cmt_gauge_create(cmt, - "fluentbit", "storage", "mem_chunks", - "Total number of memory chunks.", - 0, (char *[]) { NULL }); - - sm->cmt_fs_chunks = cmt_gauge_create(cmt, - "fluentbit", "storage", "fs_chunks", - "Total number of filesystem chunks.", - 0, (char *[]) { NULL }); - - sm->cmt_fs_chunks_up = cmt_gauge_create(cmt, - "fluentbit", "storage", "fs_chunks_up", - "Total number of filesystem chunks up in memory.", - 0, (char *[]) { NULL }); - - sm->cmt_fs_chunks_down = cmt_gauge_create(cmt, - "fluentbit", "storage", "fs_chunks_down", - "Total number of filesystem chunks down.", - 0, (char *[]) { NULL }); - - return cmt; -} - - -/* This function collect the 'global' metrics of the storage layer (cmetrics) */ -int flb_storage_metrics_update(struct flb_config *ctx, struct flb_storage_metrics *sm) -{ - uint64_t ts; - struct cio_stats st; - - /* Retrieve general stats from the storage layer */ - cio_stats_get(ctx->cio, &st); - - ts = cfl_time_now(); - - cmt_gauge_set(sm->cmt_chunks, ts, st.chunks_total, 0, NULL); - cmt_gauge_set(sm->cmt_mem_chunks, ts, st.chunks_mem, 0, NULL); - cmt_gauge_set(sm->cmt_fs_chunks, ts, st.chunks_fs, 0, NULL); - cmt_gauge_set(sm->cmt_fs_chunks_up, ts, st.chunks_fs_up, 0, NULL); - cmt_gauge_set(sm->cmt_fs_chunks_down, ts, st.chunks_fs_down, 0, NULL); - - return 0; -} - -static void metrics_append_general(msgpack_packer *mp_pck, - struct flb_config *ctx, - struct flb_storage_metrics *sm) -{ - struct cio_stats storage_st; - - /* Retrieve general stats from the storage layer */ - cio_stats_get(ctx->cio, &storage_st); - - msgpack_pack_str(mp_pck, 13); - msgpack_pack_str_body(mp_pck, "storage_layer", 13); - msgpack_pack_map(mp_pck, 1); - - /* Chunks */ - msgpack_pack_str(mp_pck, 6); - msgpack_pack_str_body(mp_pck, "chunks", 6); - msgpack_pack_map(mp_pck, 5); - - /* chunks['total_chunks'] */ - msgpack_pack_str(mp_pck, 12); - msgpack_pack_str_body(mp_pck, "total_chunks", 12); - msgpack_pack_uint64(mp_pck, storage_st.chunks_total); - - /* chunks['mem_chunks'] */ - msgpack_pack_str(mp_pck, 10); - msgpack_pack_str_body(mp_pck, "mem_chunks", 10); - msgpack_pack_uint64(mp_pck, storage_st.chunks_mem); - - /* chunks['fs_chunks'] */ - msgpack_pack_str(mp_pck, 9); - msgpack_pack_str_body(mp_pck, "fs_chunks", 9); - msgpack_pack_uint64(mp_pck, storage_st.chunks_fs); - - /* chunks['fs_up_chunks'] */ - msgpack_pack_str(mp_pck, 12); - msgpack_pack_str_body(mp_pck, "fs_chunks_up", 12); - msgpack_pack_uint64(mp_pck, storage_st.chunks_fs_up); - - /* chunks['fs_down_chunks'] */ - msgpack_pack_str(mp_pck, 14); - msgpack_pack_str_body(mp_pck, "fs_chunks_down", 14); - msgpack_pack_uint64(mp_pck, storage_st.chunks_fs_down); -} - -static void metrics_append_input(msgpack_packer *mp_pck, - struct flb_config *ctx, - struct flb_storage_metrics *sm) -{ - int len; - int ret; - uint64_t ts; - const char *tmp; - char buf[32]; - ssize_t size; - size_t total_chunks; - - /* chunks */ - int up; - int down; - int busy; - char *name; - ssize_t busy_size; - struct mk_list *head; - struct mk_list *h_chunks; - struct flb_input_instance *i; - struct flb_input_chunk *ic; - - /* - * DISCLAIMER: This interface will be deprecated once we extend Chunk I/O - * stats per stream. - * - * For now and to avoid duplication of iterating chunks we are adding the - * metrics counting for CMetrics inside the same logic for the old code. - */ - - msgpack_pack_str(mp_pck, 12); - msgpack_pack_str_body(mp_pck, "input_chunks", 12); - msgpack_pack_map(mp_pck, mk_list_size(&ctx->inputs)); - - /* current time */ - ts = cfl_time_now(); - - /* Input Plugins Ingestion */ - mk_list_foreach(head, &ctx->inputs) { - i = mk_list_entry(head, struct flb_input_instance, _head); - - name = (char *) flb_input_name(i); - total_chunks = mk_list_size(&i->chunks); - - tmp = flb_input_name(i); - len = strlen(tmp); - - msgpack_pack_str(mp_pck, len); - msgpack_pack_str_body(mp_pck, tmp, len); - - /* Map for 'status' and 'chunks' */ - msgpack_pack_map(mp_pck, 2); - - /* - * Status - * ====== - */ - msgpack_pack_str(mp_pck, 6); - msgpack_pack_str_body(mp_pck, "status", 6); - - /* 'status' map has 2 keys: overlimit and chunks */ - msgpack_pack_map(mp_pck, 3); - - /* status['overlimit'] */ - msgpack_pack_str(mp_pck, 9); - msgpack_pack_str_body(mp_pck, "overlimit", 9); - - - /* CMetrics */ - ret = FLB_FALSE; - if (i->mem_buf_limit > 0) { - if (i->mem_chunks_size >= i->mem_buf_limit) { - ret = FLB_TRUE; - } - } - if (ret == FLB_TRUE) { - /* cmetrics */ - cmt_gauge_set(i->cmt_storage_overlimit, ts, 1, - 1, (char *[]) {name}); - - /* old code */ - msgpack_pack_true(mp_pck); - } - else { - /* cmetrics */ - cmt_gauge_set(i->cmt_storage_overlimit, ts, 0, - 1, (char *[]) {name}); - - /* old code */ - msgpack_pack_false(mp_pck); - } - - /* fluentbit_storage_memory_bytes */ - cmt_gauge_set(i->cmt_storage_memory_bytes, ts, i->mem_chunks_size, - 1, (char *[]) {name}); - - /* status['mem_size'] */ - msgpack_pack_str(mp_pck, 8); - msgpack_pack_str_body(mp_pck, "mem_size", 8); - - /* Current memory size used based on last ingestion */ - flb_utils_bytes_to_human_readable_size(i->mem_chunks_size, - buf, sizeof(buf) - 1); - len = strlen(buf); - msgpack_pack_str(mp_pck, len); - msgpack_pack_str_body(mp_pck, buf, len); - - /* status['mem_limit'] */ - msgpack_pack_str(mp_pck, 9); - msgpack_pack_str_body(mp_pck, "mem_limit", 9); - - flb_utils_bytes_to_human_readable_size(i->mem_buf_limit, - buf, sizeof(buf) - 1); - len = strlen(buf); - msgpack_pack_str(mp_pck, len); - msgpack_pack_str_body(mp_pck, buf, len); - - /* - * Chunks - * ====== - */ - - /* cmetrics */ - cmt_gauge_set(i->cmt_storage_chunks, ts, total_chunks, - 1, (char *[]) {name}); - - - /* old code */ - msgpack_pack_str(mp_pck, 6); - msgpack_pack_str_body(mp_pck, "chunks", 6); - - /* 'chunks' has 3 keys: total, up, down, busy and busy_size */ - msgpack_pack_map(mp_pck, 5); - - /* chunks['total_chunks'] */ - msgpack_pack_str(mp_pck, 5); - msgpack_pack_str_body(mp_pck, "total", 5); - msgpack_pack_uint64(mp_pck, total_chunks); - - /* - * chunks Details: chunks marked as 'busy' are 'locked' since they are in - * a 'flush' state. No more data can be appended to a busy chunk. - */ - busy = 0; - busy_size = 0; - - /* up/down */ - up = 0; - down = 0; - - /* Iterate chunks for the input instance in question */ - mk_list_foreach(h_chunks, &i->chunks) { - ic = mk_list_entry(h_chunks, struct flb_input_chunk, _head); - if (ic->busy == FLB_TRUE) { - busy++; - size = cio_chunk_get_content_size(ic->chunk); - if (size >= 0) { - busy_size += size; - } - } - - if (cio_chunk_is_up(ic->chunk) == CIO_TRUE) { - up++; - } - else { - down++; - } - - } - - /* fluentbit_storage_chunks_up */ - cmt_gauge_set(i->cmt_storage_chunks_up, ts, up, - 1, (char *[]) {name}); - - /* chunks['up'] */ - msgpack_pack_str(mp_pck, 2); - msgpack_pack_str_body(mp_pck, "up", 2); - msgpack_pack_uint64(mp_pck, up); - - /* fluentbit_storage_chunks_down */ - cmt_gauge_set(i->cmt_storage_chunks_down, ts, down, - 1, (char *[]) {name}); - - /* chunks['down'] */ - msgpack_pack_str(mp_pck, 4); - msgpack_pack_str_body(mp_pck, "down", 4); - msgpack_pack_uint64(mp_pck, down); - - /* fluentbit_storage_chunks_busy */ - cmt_gauge_set(i->cmt_storage_chunks_busy, ts, busy, - 1, (char *[]) {name}); - - /* chunks['busy'] */ - msgpack_pack_str(mp_pck, 4); - msgpack_pack_str_body(mp_pck, "busy", 4); - msgpack_pack_uint64(mp_pck, busy); - - /* fluentbit_storage_chunks_busy_size */ - cmt_gauge_set(i->cmt_storage_chunks_busy_bytes, ts, busy_size, - 1, (char *[]) {name}); - - /* chunks['busy_size'] */ - msgpack_pack_str(mp_pck, 9); - msgpack_pack_str_body(mp_pck, "busy_size", 9); - - flb_utils_bytes_to_human_readable_size(busy_size, buf, sizeof(buf) - 1); - len = strlen(buf); - msgpack_pack_str(mp_pck, len); - msgpack_pack_str_body(mp_pck, buf, len); - } -} - -static void cb_storage_metrics_collect(struct flb_config *ctx, void *data) -{ - msgpack_sbuffer mp_sbuf; - msgpack_packer mp_pck; - - /* Prepare new outgoing buffer */ - msgpack_sbuffer_init(&mp_sbuf); - msgpack_packer_init(&mp_pck, &mp_sbuf, msgpack_sbuffer_write); - - /* Pack main map and append relevant data */ - msgpack_pack_map(&mp_pck, 2); - metrics_append_general(&mp_pck, ctx, data); - metrics_append_input(&mp_pck, ctx, data); - -#ifdef FLB_HAVE_HTTP_SERVER - if (ctx->http_server == FLB_TRUE && ctx->storage_metrics == FLB_TRUE) { - flb_hs_push_storage_metrics(ctx->http_ctx, mp_sbuf.data, mp_sbuf.size); - } -#endif - msgpack_sbuffer_destroy(&mp_sbuf); -} - -struct flb_storage_metrics *flb_storage_metrics_create(struct flb_config *ctx) -{ - int ret; - struct flb_storage_metrics *sm; - - sm = flb_calloc(1, sizeof(struct flb_storage_metrics)); - if (!sm) { - flb_errno(); - return NULL; - } - sm->cmt = metrics_context_create(sm); - if(!sm->cmt) { - flb_free(sm); - return NULL; - } - - ret = flb_sched_timer_cb_create(ctx->sched, FLB_SCHED_TIMER_CB_PERM, 5000, - cb_storage_metrics_collect, - ctx->storage_metrics_ctx, NULL); - if (ret == -1) { - flb_error("[storage metrics] cannot create timer to collect metrics"); - flb_free(sm); - return NULL; - } - - return sm; -} - -static int sort_chunk_cmp(const void *a_arg, const void *b_arg) -{ - char *p; - struct cio_chunk *chunk_a = *(struct cio_chunk **) a_arg; - struct cio_chunk *chunk_b = *(struct cio_chunk **) b_arg; - struct timespec tm_a; - struct timespec tm_b; - - /* Scan Chunk A */ - p = strchr(chunk_a->name, '-'); - if (!p) { - return -1; - } - p++; - - sscanf(p, "%lu.%lu.flb", &tm_a.tv_sec, &tm_a.tv_nsec); - - /* Scan Chunk B */ - p = strchr(chunk_b->name, '-'); - if (!p) { - return -1; - } - p++; - sscanf(p, "%lu.%lu.flb", &tm_b.tv_sec, &tm_b.tv_nsec); - - /* Compare */ - if (tm_a.tv_sec != tm_b.tv_sec) { - if (tm_a.tv_sec > tm_b.tv_sec) { - return 1; - } - else { - return -1; - } - } - else { - if (tm_a.tv_nsec > tm_b.tv_nsec) { - return 1; - } - else if (tm_a.tv_nsec < tm_b.tv_nsec) { - return -1; - } - } - - return 0; -} - -static void print_storage_info(struct flb_config *ctx, struct cio_ctx *cio) -{ - char *type; - char *sync; - char *checksum; - struct flb_input_instance *in; - - if (cio->options.root_path) { - type = "memory+filesystem"; - } - else { - type = "memory"; - } - - if (cio->options.flags & CIO_FULL_SYNC) { - sync = "full"; - } - else { - sync = "normal"; - } - - if (cio->options.flags & CIO_CHECKSUM) { - checksum = "on"; - } - else { - checksum = "off"; - } - - flb_info("[storage] ver=%s, type=%s, sync=%s, checksum=%s, max_chunks_up=%i", - cio_version(), type, sync, checksum, ctx->storage_max_chunks_up); - - /* Storage input plugin */ - if (ctx->storage_input_plugin) { - in = (struct flb_input_instance *) ctx->storage_input_plugin; - flb_info("[storage] backlog input plugin: %s", in->name); - } -} - -static int log_cb(struct cio_ctx *ctx, int level, const char *file, int line, - char *str) -{ - if (level == CIO_LOG_ERROR) { - flb_error("[storage] %s", str); - } - else if (level == CIO_LOG_WARN) { - flb_warn("[storage] %s", str); - } - else if (level == CIO_LOG_INFO) { - flb_info("[storage] %s", str); - } - else if (level == CIO_LOG_DEBUG) { - flb_debug("[storage] %s", str); - } - - return 0; -} - -int flb_storage_input_create(struct cio_ctx *cio, - struct flb_input_instance *in) -{ - int cio_storage_type; - struct flb_storage_input *si; - struct cio_stream *stream; - - /* storage config: get stream type */ - if (in->storage_type == -1) { - in->storage_type = FLB_STORAGE_MEM; - } - - if (in->storage_type == FLB_STORAGE_FS && cio->options.root_path == NULL) { - flb_error("[storage] instance '%s' requested filesystem storage " - "but no filesystem path was defined.", - flb_input_name(in)); - return -1; - } - - /* - * The input instance can define it owns storage type which is based on some - * specific Chunk I/O storage type. We handle the proper initialization here. - */ - cio_storage_type = in->storage_type; - if (in->storage_type == FLB_STORAGE_MEMRB) { - cio_storage_type = FLB_STORAGE_MEM; - } - - /* Check for duplicates */ - stream = cio_stream_get(cio, in->name); - if (!stream) { - /* create stream for input instance */ - stream = cio_stream_create(cio, in->name, cio_storage_type); - if (!stream) { - flb_error("[storage] cannot create stream for instance %s", - in->name); - return -1; - } - } - - /* allocate storage context for the input instance */ - si = flb_malloc(sizeof(struct flb_storage_input)); - if (!si) { - flb_errno(); - return -1; - } - - si->stream = stream; - si->cio = cio; - si->type = in->storage_type; - in->storage = si; - - return 0; -} - -void flb_storage_input_destroy(struct flb_input_instance *in) -{ - struct mk_list *tmp; - struct mk_list *head; - struct flb_input_chunk *ic; - - /* Save current temporary data and destroy chunk references */ - mk_list_foreach_safe(head, tmp, &in->chunks) { - ic = mk_list_entry(head, struct flb_input_chunk, _head); - flb_input_chunk_destroy(ic, FLB_FALSE); - } - - flb_free(in->storage); - in->storage = NULL; -} - -static int storage_contexts_create(struct flb_config *config) -{ - int c = 0; - int ret; - struct mk_list *head; - struct flb_input_instance *in; - - /* Iterate each input instance and create a stream for it */ - mk_list_foreach(head, &config->inputs) { - in = mk_list_entry(head, struct flb_input_instance, _head); - ret = flb_storage_input_create(config->cio, in); - if (ret == -1) { - flb_error("[storage] could not create storage for instance: %s", - in->name); - return -1; - } - c++; - } - - return c; -} - -int flb_storage_create(struct flb_config *ctx) -{ - int ret; - int flags; - struct flb_input_instance *in = NULL; - struct cio_ctx *cio; - struct cio_options opts = {0}; - - /* always use read/write mode */ - flags = CIO_OPEN; - - /* if explicitly stated any irrecoverably corrupted - * chunks will be deleted */ - if (ctx->storage_del_bad_chunks) { - flags |= CIO_DELETE_IRRECOVERABLE; - } - - /* synchronization mode */ - if (ctx->storage_sync) { - if (strcasecmp(ctx->storage_sync, "normal") == 0) { - /* do nothing, keep the default */ - } - else if (strcasecmp(ctx->storage_sync, "full") == 0) { - flags |= CIO_FULL_SYNC; - } - else { - flb_error("[storage] invalid synchronization mode"); - return -1; - } - } - - /* checksum */ - if (ctx->storage_checksum == FLB_TRUE) { - flags |= CIO_CHECKSUM; - } - - /* file trimming */ - if (ctx->storage_trim_files == FLB_TRUE) { - flags |= CIO_TRIM_FILES; - } - - /* chunkio options */ - cio_options_init(&opts); - - opts.root_path = ctx->storage_path; - opts.flags = flags; - opts.log_cb = log_cb; - opts.log_level = CIO_LOG_INFO; - - /* Create chunkio context */ - cio = cio_create(&opts); - if (!cio) { - flb_error("[storage] error initializing storage engine"); - return -1; - } - ctx->cio = cio; - - /* Set Chunk I/O maximum number of chunks up */ - if (ctx->storage_max_chunks_up == 0) { - ctx->storage_max_chunks_up = FLB_STORAGE_MAX_CHUNKS_UP; - } - cio_set_max_chunks_up(ctx->cio, ctx->storage_max_chunks_up); - - /* Load content from the file system if any */ - ret = cio_load(ctx->cio, NULL); - if (ret == -1) { - flb_error("[storage] error scanning root path content: %s", - ctx->storage_path); - cio_destroy(ctx->cio); - return -1; - } - - /* Sort chunks */ - cio_qsort(ctx->cio, sort_chunk_cmp); - - /* - * If we have a filesystem storage path, create an instance of the - * storage_backlog input plugin to consume any possible pending - * chunks. - */ - if (ctx->storage_path) { - in = flb_input_new(ctx, "storage_backlog", cio, FLB_FALSE); - if (!in) { - flb_error("[storage] cannot init storage backlog input plugin"); - cio_destroy(cio); - ctx->cio = NULL; - return -1; - } - ctx->storage_input_plugin = in; - - /* Set a queue memory limit */ - if (!ctx->storage_bl_mem_limit) { - ctx->storage_bl_mem_limit = flb_strdup(FLB_STORAGE_BL_MEM_LIMIT); - } - } - - /* Create streams for input instances */ - ret = storage_contexts_create(ctx); - if (ret == -1) { - return -1; - } - - /* print storage info */ - print_storage_info(ctx, cio); - - return 0; -} - -void flb_storage_destroy(struct flb_config *ctx) -{ - struct cio_ctx *cio; - struct flb_storage_metrics *sm; - - /* Destroy Chunk I/O context */ - cio = (struct cio_ctx *) ctx->cio; - - if (!cio) { - return; - } - - sm = ctx->storage_metrics_ctx; - if (ctx->storage_metrics == FLB_TRUE && sm != NULL) { - cmt_destroy(sm->cmt); - flb_free(sm); - ctx->storage_metrics_ctx = NULL; - } - - cio_destroy(cio); - ctx->cio = NULL; -} diff --git a/fluent-bit/src/flb_strptime.c b/fluent-bit/src/flb_strptime.c deleted file mode 100644 index 492a6ee74..000000000 --- a/fluent-bit/src/flb_strptime.c +++ /dev/null @@ -1,750 +0,0 @@ -/* $OpenBSD: strptime.c,v 1.30 2019/05/12 12:49:52 schwarze Exp $ */ -/* $NetBSD: strptime.c,v 1.12 1998/01/20 21:39:40 mycroft Exp $ */ -/*- - * Copyright (c) 1997, 1998, 2005, 2008 The NetBSD Foundation, Inc. - * All rights reserved. - * - * This code was contributed to The NetBSD Foundation by Klaus Klein. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS - * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED - * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS - * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ - -/* - * This file provides a portable implementation of strptime(2), based - * on the work of OpenSBD project. Since various platforms implement - * strptime differently, this one should work as a fallback. - */ - -#include <ctype.h> -#include <locale.h> -#include <stdint.h> -#include <string.h> - -#include <fluent-bit/flb_compat.h> -#include <fluent-bit/flb_langinfo.h> -#include <fluent-bit/flb_time.h> - -#define _ctloc(x) (nl_langinfo(x)) - -/* - * We do not implement alternate representations. However, we always - * check whether a given modifier is allowed for a certain conversion. - */ -#define _ALT_E 0x01 -#define _ALT_O 0x02 -#define _LEGAL_ALT(x) { if (alt_format & ~(x)) return (0); } - -/* - * Copied from libc/time/private.h and libc/time/tzfile.h - */ -#define TM_YEAR_BASE 1900 -#define DAYSPERNYEAR 365 -#define DAYSPERLYEAR 366 -#define DAYSPERWEEK 7 -#define MONSPERYEAR 12 -#define EPOCH_YEAR 1970 -#define EPOCH_WDAY 4 /* Thursday */ -#define SECSPERHOUR 3600 -#define SECSPERMIN 60 - -#define isleap(y) (((y) % 4) == 0 && (((y) % 100) != 0 || ((y) % 400) == 0)) - -/* - * We keep track of some of the fields we set in order to compute missing ones. - */ -#define FIELD_TM_MON (1 << 0) -#define FIELD_TM_MDAY (1 << 1) -#define FIELD_TM_WDAY (1 << 2) -#define FIELD_TM_YDAY (1 << 3) -#define FIELD_TM_YEAR (1 << 4) - -static char gmt[] = { "GMT" }; -static char utc[] = { "UTC" }; -/* RFC-822/RFC-2822 */ -static const char * const nast[5] = { - "EST", "CST", "MST", "PST", "\0\0\0" -}; -static const char * const nadt[5] = { - "EDT", "CDT", "MDT", "PDT", "\0\0\0" -}; - -static const int mon_lengths[2][MONSPERYEAR] = { - { 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31 }, - { 31, 29, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31 } -}; - -static nl_item day[] = { - DAY_1, DAY_2, DAY_3, DAY_4, DAY_5, DAY_6, DAY_7 -}; - -static nl_item mon[] = { - MON_1, MON_2, MON_3, MON_4, MON_5, MON_6, MON_7, MON_8, MON_9, - MON_10, MON_11, MON_12 -}; - -static nl_item abday[] = { - ABDAY_1, ABDAY_2, ABDAY_3, ABDAY_4, ABDAY_5, ABDAY_6, ABDAY_7 -}; - -static nl_item abmon[] = { - ABMON_1, ABMON_2, ABMON_3, ABMON_4, ABMON_5, ABMON_6, ABMON_7, - ABMON_8, ABMON_9, ABMON_10, ABMON_11, ABMON_12 -}; - -static int _conv_num64(const unsigned char **, int64_t *, int64_t, int64_t); -static int _conv_num(const unsigned char **, int *, int, int); -static int leaps_thru_end_of(const int y); -static char *_flb_strptime(const char *, const char *, struct flb_tm *, int); -static const u_char *_find_string(const u_char *, int *, const char * const *, - const char * const *, int); - -/* - * FreeBSD does not support `timezone` in time.h. - * https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=24590 - */ -#ifdef __FreeBSD__ -int flb_timezone(void) -{ - struct tm tm; - time_t t = 0; - tzset(); - localtime_r(&t, &tm); - return -(tm.tm_gmtoff); -} -#define timezone (flb_timezone()) -#endif - -char * -flb_strptime(const char *buf, const char *fmt, struct flb_tm *tm) -{ - return(_flb_strptime(buf, fmt, tm, 1)); -} - -static char * -_flb_strptime(const char *buf, const char *fmt, struct flb_tm *tm, int initialize) -{ - unsigned char c; - const unsigned char *bp, *ep; - size_t len = 0; - int alt_format, i, offs; - int neg = 0; - static int century, relyear, fields; - - if (initialize) { - century = TM_YEAR_BASE; - relyear = -1; - fields = 0; - } - - bp = (const unsigned char *)buf; - while ((c = *fmt) != '\0') { - /* Clear `alternate' modifier prior to new conversion. */ - alt_format = 0; - - /* Eat up white-space. */ - if (isspace(c)) { - while (isspace(*bp)) - bp++; - - fmt++; - continue; - } - - /* - * Having increased bp we need to ensure we are not - * moving beyond bounds. - */ - if (*bp == '\0') - return (NULL); - - if ((c = *fmt++) != '%') - goto literal; - - -again: switch (c = *fmt++) { - case '%': /* "%%" is converted to "%". */ -literal: - if (c != *bp++) - return (NULL); - - break; - - /* - * "Alternative" modifiers. Just set the appropriate flag - * and start over again. - */ - case 'E': /* "%E?" alternative conversion modifier. */ - _LEGAL_ALT(0); - alt_format |= _ALT_E; - goto again; - - case 'O': /* "%O?" alternative conversion modifier. */ - _LEGAL_ALT(0); - alt_format |= _ALT_O; - goto again; - - /* - * "Complex" conversion rules, implemented through recursion. - */ - case 'c': /* Date and time, using the locale's format. */ - _LEGAL_ALT(_ALT_E); - if (!(bp = (const unsigned char *)_flb_strptime((const char *)bp, _ctloc(D_T_FMT), tm, 0))) - return (NULL); - break; - - case 'D': /* The date as "%m/%d/%y". */ - _LEGAL_ALT(0); - if (!(bp = (const unsigned char *)_flb_strptime((const char *)bp, "%m/%d/%y", tm, 0))) - return (NULL); - break; - - case 'F': /* The date as "%Y-%m-%d". */ - _LEGAL_ALT(0); - if (!(bp = (const unsigned char *)_flb_strptime((const char *)bp, "%Y-%m-%d", tm, 0))) - return (NULL); - continue; - - case 'R': /* The time as "%H:%M". */ - _LEGAL_ALT(0); - if (!(bp = (const unsigned char *)_flb_strptime((const char *)bp, "%H:%M", tm, 0))) - return (NULL); - break; - - case 'r': /* The time as "%I:%M:%S %p". */ - _LEGAL_ALT(0); - if (!(bp = (const unsigned char *)_flb_strptime((const char *)bp, "%I:%M:%S %p", tm, 0))) - return (NULL); - break; - - case 'T': /* The time as "%H:%M:%S". */ - _LEGAL_ALT(0); - if (!(bp = (const unsigned char *)_flb_strptime((const char *)bp, "%H:%M:%S", tm, 0))) - return (NULL); - break; - - case 'X': /* The time, using the locale's format. */ - _LEGAL_ALT(_ALT_E); - if (!(bp = (const unsigned char *)_flb_strptime((const char *)bp, _ctloc(T_FMT), tm, 0))) - return (NULL); - break; - - case 'x': /* The date, using the locale's format. */ - _LEGAL_ALT(_ALT_E); - if (!(bp = (const unsigned char *)_flb_strptime((const char *)bp, _ctloc(D_FMT), tm, 0))) - return (NULL); - break; - - /* - * "Elementary" conversion rules. - */ - case 'A': /* The day of week, using the locale's form. */ - case 'a': - _LEGAL_ALT(0); - for (i = 0; i < 7; i++) { - /* Full name. */ - len = strlen(_ctloc(day[i])); - if (strncasecmp(_ctloc(day[i]), (const char *)bp, len) == 0) - break; - - /* Abbreviated name. */ - len = strlen(_ctloc(abday[i])); - if (strncasecmp(_ctloc(abday[i]), (const char *)bp, len) == 0) - break; - } - - /* Nothing matched. */ - if (i == 7) - return (NULL); - - tm->tm.tm_wday = i; - bp += len; - fields |= FIELD_TM_WDAY; - break; - - case 'B': /* The month, using the locale's form. */ - case 'b': - case 'h': - _LEGAL_ALT(0); - for (i = 0; i < 12; i++) { - /* Full name. */ - len = strlen(_ctloc(mon[i])); - if (strncasecmp(_ctloc(mon[i]), (const char *)bp, len) == 0) - break; - - /* Abbreviated name. */ - len = strlen(_ctloc(abmon[i])); - if (strncasecmp(_ctloc(abmon[i]), (const char *)bp, len) == 0) - break; - } - - /* Nothing matched. */ - if (i == 12) - return (NULL); - - tm->tm.tm_mon = i; - bp += len; - fields |= FIELD_TM_MON; - break; - - case 'C': /* The century number. */ - _LEGAL_ALT(_ALT_E); - if (!(_conv_num(&bp, &i, 0, 99))) - return (NULL); - - century = i * 100; - break; - - case 'e': /* The day of month. */ - if (isspace(*bp)) - bp++; - /* FALLTHROUGH */ - case 'd': - _LEGAL_ALT(_ALT_O); - if (!(_conv_num(&bp, &tm->tm.tm_mday, 1, 31))) - return (NULL); - fields |= FIELD_TM_MDAY; - break; - - case 'k': /* The hour (24-hour clock representation). */ - _LEGAL_ALT(0); - /* FALLTHROUGH */ - case 'H': - _LEGAL_ALT(_ALT_O); - if (!(_conv_num(&bp, &tm->tm.tm_hour, 0, 23))) - return (NULL); - break; - - case 'l': /* The hour (12-hour clock representation). */ - _LEGAL_ALT(0); - /* FALLTHROUGH */ - case 'I': - _LEGAL_ALT(_ALT_O); - if (!(_conv_num(&bp, &tm->tm.tm_hour, 1, 12))) - return (NULL); - break; - - case 'j': /* The day of year. */ - _LEGAL_ALT(0); - if (!(_conv_num(&bp, &tm->tm.tm_yday, 1, 366))) - return (NULL); - tm->tm.tm_yday--; - fields |= FIELD_TM_YDAY; - break; - - case 'M': /* The minute. */ - _LEGAL_ALT(_ALT_O); - if (!(_conv_num(&bp, &tm->tm.tm_min, 0, 59))) - return (NULL); - break; - - case 'm': /* The month. */ - _LEGAL_ALT(_ALT_O); - if (!(_conv_num(&bp, &tm->tm.tm_mon, 1, 12))) - return (NULL); - tm->tm.tm_mon--; - fields |= FIELD_TM_MON; - break; - - case 'p': /* The locale's equivalent of AM/PM. */ - _LEGAL_ALT(0); - /* AM? */ - len = strlen(_ctloc(AM_STR)); - if (strncasecmp(_ctloc(AM_STR), (const char *)bp, len) == 0) { - if (tm->tm.tm_hour > 12) /* i.e., 13:00 AM ?! */ - return (NULL); - else if (tm->tm.tm_hour == 12) - tm->tm.tm_hour = 0; - - bp += len; - break; - } - /* PM? */ - len = strlen(_ctloc(PM_STR)); - if (strncasecmp(_ctloc(PM_STR), (const char *)bp, len) == 0) { - if (tm->tm.tm_hour > 12) /* i.e., 13:00 PM ?! */ - return (NULL); - else if (tm->tm.tm_hour < 12) - tm->tm.tm_hour += 12; - - bp += len; - break; - } - - /* Nothing matched. */ - return (NULL); - - case 'S': /* The seconds. */ - _LEGAL_ALT(_ALT_O); - if (!(_conv_num(&bp, &tm->tm.tm_sec, 0, 60))) - return (NULL); - break; - case 's': /* Seconds since epoch */ - { - int64_t i64; - if (!(_conv_num64(&bp, &i64, 0, INT64_MAX))) - return (NULL); - if (!gmtime_r((const time_t *) &i64, &tm->tm)) - return (NULL); - fields = 0xffff; /* everything */ - } - break; - case 'U': /* The week of year, beginning on sunday. */ - case 'W': /* The week of year, beginning on monday. */ - _LEGAL_ALT(_ALT_O); - /* - * XXX This is bogus, as we can not assume any valid - * information present in the tm structure at this - * point to calculate a real value, so just check the - * range for now. - */ - if (!(_conv_num(&bp, &i, 0, 53))) - return (NULL); - break; - - case 'w': /* The day of week, beginning on sunday. */ - _LEGAL_ALT(_ALT_O); - if (!(_conv_num(&bp, &tm->tm.tm_wday, 0, 6))) - return (NULL); - fields |= FIELD_TM_WDAY; - break; - - case 'u': /* The day of week, monday = 1. */ - _LEGAL_ALT(_ALT_O); - if (!(_conv_num(&bp, &i, 1, 7))) - return (NULL); - tm->tm.tm_wday = i % 7; - fields |= FIELD_TM_WDAY; - continue; - - case 'g': /* The year corresponding to the ISO week - * number but without the century. - */ - if (!(_conv_num(&bp, &i, 0, 99))) - return (NULL); - continue; - - case 'G': /* The year corresponding to the ISO week - * number with century. - */ - do - bp++; - while (isdigit(*bp)); - continue; - - case 'V': /* The ISO 8601:1988 week number as decimal */ - if (!(_conv_num(&bp, &i, 0, 53))) - return (NULL); - continue; - - case 'Y': /* The year. */ - _LEGAL_ALT(_ALT_E); - if (!(_conv_num(&bp, &i, 0, 9999))) - return (NULL); - - relyear = -1; - tm->tm.tm_year = i - TM_YEAR_BASE; - fields |= FIELD_TM_YEAR; - break; - - case 'y': /* The year within the century (2 digits). */ - _LEGAL_ALT(_ALT_E | _ALT_O); - if (!(_conv_num(&bp, &relyear, 0, 99))) - return (NULL); - break; - - case 'Z': - tzset(); - if (strncmp((const char *)bp, gmt, 3) == 0) { - tm->tm.tm_isdst = 0; - flb_tm_gmtoff(tm) = 0; -#ifdef FLB_HAVE_ZONE - tm->tm.tm_zone = gmt; -#endif - bp += 3; - } else if (strncmp((const char *)bp, utc, 3) == 0) { - tm->tm.tm_isdst = 0; - flb_tm_gmtoff(tm) = 0; -#ifdef FLB_HAVE_ZONE - tm->tm.tm_zone = utc; -#endif - bp += 3; - } else { - ep = _find_string(bp, &i, - (const char * const *)tzname, - NULL, 2); - if (ep == NULL) - return (NULL); - - tm->tm.tm_isdst = i; - flb_tm_gmtoff(tm) = -(timezone); -#ifdef FLB_HAVE_ZONE - tm->tm.tm_zone = tzname[i]; -#endif - bp = ep; - } - continue; - - case 'z': - /* - * We recognize all ISO 8601 formats: - * Z = Zulu time/UTC - * [+-]hhmm - * [+-]hh:mm - * [+-]hh - * We recognize all RFC-822/RFC-2822 formats: - * UT|GMT - * North American : UTC offsets - * E[DS]T = Eastern : -4 | -5 - * C[DS]T = Central : -5 | -6 - * M[DS]T = Mountain: -6 | -7 - * P[DS]T = Pacific : -7 | -8 - */ - while (isspace(*bp)) - bp++; - - switch (*bp++) { - case 'G': - if (*bp++ != 'M') - return NULL; - /*FALLTHROUGH*/ - case 'U': - if (*bp++ != 'T') - return NULL; - /*FALLTHROUGH*/ - case 'Z': - tm->tm.tm_isdst = 0; - flb_tm_gmtoff(tm) = 0; -#ifdef FLB_HAVE_ZONE - tm->tm.tm_zone = utc; -#endif - continue; - case '+': - neg = 0; - break; - case '-': - neg = 1; - break; - default: - --bp; - ep = _find_string(bp, &i, nast, NULL, 4); - if (ep != NULL) { - flb_tm_gmtoff(tm) = (-5 - i) * SECSPERHOUR; -#ifdef FLB_HAVE_ZONE - tm->tm.tm_zone = (char *)nast[i]; -#endif - bp = ep; - continue; - } - ep = _find_string(bp, &i, nadt, NULL, 4); - if (ep != NULL) { - tm->tm.tm_isdst = 1; - flb_tm_gmtoff(tm) = (-4 - i) * SECSPERHOUR; -#ifdef FLB_HAVE_ZONE - tm->tm.tm_zone = (char *)nadt[i]; -#endif - bp = ep; - continue; - } - return NULL; - } - if (!isdigit(bp[0]) || !isdigit(bp[1])) - return NULL; - offs = ((bp[0]-'0') * 10 + (bp[1]-'0')) * SECSPERHOUR; - bp += 2; - if (*bp == ':') - bp++; - if (isdigit(*bp)) { - offs += (*bp++ - '0') * 10 * SECSPERMIN; - if (!isdigit(*bp)) - return NULL; - offs += (*bp++ - '0') * SECSPERMIN; - } - if (neg) - offs = -offs; - tm->tm.tm_isdst = 0; /* XXX */ - flb_tm_gmtoff(tm) = offs; -#ifdef FLB_HAVE_ZONE - tm->tm.tm_zone = NULL; /* XXX */ -#endif - continue; - - /* - * Miscellaneous conversions. - */ - case 'n': /* Any kind of white-space. */ - case 't': - _LEGAL_ALT(0); - while (isspace(*bp)) - bp++; - break; - - - default: /* Unknown/unsupported conversion. */ - return (NULL); - } - - - } - - /* - * We need to evaluate the two digit year spec (%y) - * last as we can get a century spec (%C) at any time. - */ - if (relyear != -1) { - if (century == TM_YEAR_BASE) { - if (relyear <= 68) - tm->tm.tm_year = relyear + 2000 - TM_YEAR_BASE; - else - tm->tm.tm_year = relyear + 1900 - TM_YEAR_BASE; - } else { - tm->tm.tm_year = relyear + century - TM_YEAR_BASE; - } - fields |= FIELD_TM_YEAR; - } - - /* Compute some missing values when possible. */ - if (fields & FIELD_TM_YEAR) { - const int year = (unsigned int)tm->tm.tm_year + (unsigned int)TM_YEAR_BASE; - const int *mon_lens = mon_lengths[isleap(year)]; - if (!(fields & FIELD_TM_YDAY) && - (fields & FIELD_TM_MON) && (fields & FIELD_TM_MDAY)) { - tm->tm.tm_yday = tm->tm.tm_mday - 1; - for (i = 0; i < tm->tm.tm_mon; i++) - tm->tm.tm_yday += mon_lens[i]; - fields |= FIELD_TM_YDAY; - } - if (fields & FIELD_TM_YDAY) { - int days = tm->tm.tm_yday; - if (!(fields & FIELD_TM_WDAY)) { - tm->tm.tm_wday = EPOCH_WDAY + - ((year - EPOCH_YEAR) % DAYSPERWEEK) * - (DAYSPERNYEAR % DAYSPERWEEK) + - leaps_thru_end_of(year - 1) - - leaps_thru_end_of(EPOCH_YEAR - 1) + - tm->tm.tm_yday; - tm->tm.tm_wday %= DAYSPERWEEK; - if (tm->tm.tm_wday < 0) - tm->tm.tm_wday += DAYSPERWEEK; - } - if (!(fields & FIELD_TM_MON)) { - tm->tm.tm_mon = 0; - while (tm->tm.tm_mon < MONSPERYEAR && days >= mon_lens[tm->tm.tm_mon]) - days -= mon_lens[tm->tm.tm_mon++]; - } - if (!(fields & FIELD_TM_MDAY)) - tm->tm.tm_mday = days + 1; - } - } - - return ((char *)bp); -} - - -static int -_conv_num(const unsigned char **buf, int *dest, int llim, int ulim) -{ - int result = 0; - int rulim = ulim; - - if (**buf < '0' || **buf > '9') - return (0); - - /* we use rulim to break out of the loop when we run out of digits */ - do { - result *= 10; - result += *(*buf)++ - '0'; - rulim /= 10; - } while ((result * 10 <= ulim) && rulim && **buf >= '0' && **buf <= '9'); - - if (result < llim || result > ulim) - return (0); - - *dest = result; - return (1); -} - -static int -_conv_num64(const unsigned char **buf, int64_t *dest, int64_t llim, int64_t ulim) -{ - int64_t result = 0; - int64_t rulim = ulim; - - if (**buf < '0' || **buf > '9') - return (0); - - /* we use rulim to break out of the loop when we run out of digits */ - do { - /* Avoid overflow: result > ((2**64)/2.0) / 10.0 */ - if (result > 922337203685477580) { - return (0); - } - result *= 10; - - /* Avoid overflow: result > ((2**64)/2.0) - 48 */ - if (result > 9223372036854775760) { - return (0); - } - result += *(*buf)++ - '0'; - rulim /= 10; - /* watch out for overflows. If value gets above - * ((2**64)/2.0)/10.0 then we will overflow. So instead - * we return 0 */ - if (result >= 922337203685477580) { - return (0); - } - } while ((result * 10 <= ulim) && rulim && **buf >= '0' && **buf <= '9'); - - if (result < llim || result > ulim) - return (0); - - *dest = result; - return (1); -} - -static const u_char * -_find_string(const u_char *bp, int *tgt, const char * const *n1, - const char * const *n2, int c) -{ - int i; - unsigned int len; - - /* check full name - then abbreviated ones */ - for (; n1 != NULL; n1 = n2, n2 = NULL) { - for (i = 0; i < c; i++, n1++) { - len = strlen(*n1); - if (strncasecmp(*n1, (const char *)bp, len) == 0) { - *tgt = i; - return bp + len; - } - } - } - - /* Nothing matched */ - return NULL; -} - -static int -leaps_thru_end_of(const int y) -{ - return (y >= 0) ? (y / 4 - y / 100 + y / 400) : - -(leaps_thru_end_of(-(y + 1)) + 1); -} diff --git a/fluent-bit/src/flb_task.c b/fluent-bit/src/flb_task.c deleted file mode 100644 index bc9ddc634..000000000 --- a/fluent-bit/src/flb_task.c +++ /dev/null @@ -1,520 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <stdio.h> -#include <stdlib.h> - -#include <fluent-bit/flb_info.h> -#include <fluent-bit/flb_config.h> -#include <fluent-bit/flb_input.h> -#include <fluent-bit/flb_input_chunk.h> -#include <fluent-bit/flb_output.h> -#include <fluent-bit/flb_router.h> -#include <fluent-bit/flb_task.h> -#include <fluent-bit/flb_mem.h> -#include <fluent-bit/flb_str.h> -#include <fluent-bit/flb_scheduler.h> - -/* - * Every task created must have an unique ID, this function lookup the - * lowest number available in the tasks_map. - * - * This 'id' is used by the task interface to communicate with the engine event - * loop about some action. - */ - -static inline int map_get_task_id(struct flb_config *config) -{ - int i; - int map_size = (sizeof(config->tasks_map) / sizeof(struct flb_task_map)); - - for (i = 0; i < map_size; i++) { - if (config->tasks_map[i].task == NULL) { - return i; - } - } - - return -1; -} - -static inline void map_set_task_id(int id, struct flb_task *task, - struct flb_config *config) -{ - config->tasks_map[id].task = task; - -} - -static inline void map_free_task_id(int id, struct flb_config *config) -{ - config->tasks_map[id].task = NULL; -} - -void flb_task_retry_destroy(struct flb_task_retry *retry) -{ - int ret; - - /* Make sure to invalidate any request from the scheduler */ - ret = flb_sched_request_invalidate(retry->parent->config, retry); - if (ret == 0) { - flb_debug("[retry] task retry=%p, invalidated from the scheduler", - retry); - } - - mk_list_del(&retry->_head); - flb_free(retry); -} - -/* - * For an existing task 'retry', re-schedule it. One of the use case of this function - * is when the engine dispatcher fails to bring the chunk up due to Chunk I/O - * configuration restrictions, the task needs to be re-scheduled. - */ -int flb_task_retry_reschedule(struct flb_task_retry *retry, struct flb_config *config) -{ - int seconds; - struct flb_task *task; - - task = retry->parent; - seconds = flb_sched_request_create(config, retry, retry->attempts); - if (seconds == -1) { - /* - * This is the worse case scenario: 'cannot re-schedule a retry'. If the Chunk - * resides only in memory, it will be lost. */ - flb_warn("[task] retry for task %i could not be re-scheduled", task->id); - flb_task_retry_destroy(retry); - if (task->users == 0 && mk_list_size(&task->retries) == 0) { - flb_task_destroy(task, FLB_TRUE); - } - return -1; - } - else { - flb_info("[task] re-schedule retry=%p %i in the next %i seconds", - retry, task->id, seconds); - } - - return 0; -} - -struct flb_task_retry *flb_task_retry_create(struct flb_task *task, - struct flb_output_instance *ins) -{ - struct mk_list *head; - struct mk_list *tmp; - struct flb_task_retry *retry = NULL; - - /* First discover if is there any previous retry context in the task */ - mk_list_foreach_safe(head, tmp, &task->retries) { - retry = mk_list_entry(head, struct flb_task_retry, _head); - if (retry->o_ins == ins) { - if (retry->attempts >= ins->retry_limit && ins->retry_limit >= 0) { - flb_debug("[task] task_id=%i reached retry-attempts limit %i/%i", - task->id, retry->attempts, ins->retry_limit); - flb_task_retry_destroy(retry); - return NULL; - } - break; - } - retry = NULL; - } - - if (!retry) { - /* Create a new re-try instance */ - retry = flb_malloc(sizeof(struct flb_task_retry)); - if (!retry) { - flb_errno(); - return NULL; - } - - retry->attempts = 1; - retry->o_ins = ins; - retry->parent = task; - mk_list_add(&retry->_head, &task->retries); - - flb_debug("[retry] new retry created for task_id=%i attempts=%i", - task->id, retry->attempts); - } - else { - retry->attempts++; - flb_debug("[retry] re-using retry for task_id=%i attempts=%i", - task->id, retry->attempts); - } - - /* - * This 'retry' was issued by an output plugin, from an Engine perspective - * we need to determinate if the source input plugin have some memory - * restrictions and if the Storage type is 'filesystem' we need to put - * the file content down. - */ - flb_input_chunk_set_up_down(task->ic); - - /* - * Besides limits adjusted above, a retry that's going to only one place - * must be down. - */ - if (mk_list_size(&task->routes) == 1) { - flb_input_chunk_down(task->ic); - } - - return retry; -} - -/* - * Return FLB_TRUE or FLB_FALSE if the chunk pointed by the task was - * created on this running instance or it comes from a chunk in the - * filesystem from a previous run. - */ -int flb_task_from_fs_storage(struct flb_task *task) -{ - struct flb_input_chunk *ic; - - ic = (struct flb_input_chunk *) task->ic; - return ic->fs_backlog; -} - -int flb_task_retry_count(struct flb_task *task, void *data) -{ - struct mk_list *head; - struct flb_task_retry *retry; - struct flb_output_instance *o_ins; - - o_ins = (struct flb_output_instance *) data; - - mk_list_foreach(head, &task->retries) { - retry = mk_list_entry(head, struct flb_task_retry, _head); - - if (retry->o_ins == o_ins) { - return retry->attempts; - } - } - - return -1; -} - -/* Check if a 'retry' context exists for a specific task, if so, cleanup */ -int flb_task_retry_clean(struct flb_task *task, struct flb_output_instance *ins) -{ - struct mk_list *tmp; - struct mk_list *head; - struct flb_task_retry *retry; - - /* Delete 'retries' only associated with the output instance */ - mk_list_foreach_safe(head, tmp, &task->retries) { - retry = mk_list_entry(head, struct flb_task_retry, _head); - if (retry->o_ins == ins) { - flb_task_retry_destroy(retry); - return 0; - } - } - - return -1; -} - -/* Allocate an initialize a basic Task structure */ -static struct flb_task *task_alloc(struct flb_config *config) -{ - int task_id; - struct flb_task *task; - - /* Allocate the new task */ - task = (struct flb_task *) flb_calloc(1, sizeof(struct flb_task)); - if (!task) { - flb_errno(); - return NULL; - } - - /* Get ID and set back 'task' reference */ - task_id = map_get_task_id(config); - if (task_id == -1) { - flb_free(task); - return NULL; - } - map_set_task_id(task_id, task, config); - - flb_trace("[task %p] created (id=%i)", task, task_id); - - /* Initialize minimum variables */ - task->id = task_id; - task->config = config; - task->status = FLB_TASK_NEW; - task->users = 0; - mk_list_init(&task->routes); - mk_list_init(&task->retries); - - pthread_mutex_init(&task->lock, NULL); - - return task; -} - -/* Return the number of tasks with 'running status' or tasks with retries */ -int flb_task_running_count(struct flb_config *config) -{ - int count = 0; - struct mk_list *head; - struct mk_list *t_head; - struct flb_task *task; - struct flb_input_instance *ins; - - mk_list_foreach(head, &config->inputs) { - ins = mk_list_entry(head, struct flb_input_instance, _head); - mk_list_foreach(t_head, &ins->tasks) { - task = mk_list_entry(t_head, struct flb_task, _head); - if (task->users > 0 || mk_list_size(&task->retries) > 0) { - count++; - } - } - } - - return count; -} - -int flb_task_running_print(struct flb_config *config) -{ - int count = 0; - flb_sds_t tmp; - flb_sds_t routes; - struct mk_list *head; - struct mk_list *t_head; - struct mk_list *r_head; - struct flb_task *task; - struct flb_task_route *route; - struct flb_input_instance *ins; - - routes = flb_sds_create_size(256); - if (!routes) { - flb_error("[task] cannot allocate space to report pending tasks"); - return -1; - } - - mk_list_foreach(head, &config->inputs) { - ins = mk_list_entry(head, struct flb_input_instance, _head); - count = mk_list_size(&ins->tasks); - flb_info("[task] %s/%s has %i pending task(s):", - ins->p->name, flb_input_name(ins), count); - mk_list_foreach(t_head, &ins->tasks) { - task = mk_list_entry(t_head, struct flb_task, _head); - - mk_list_foreach(r_head, &task->routes) { - route = mk_list_entry(r_head, struct flb_task_route, _head); - tmp = flb_sds_printf(&routes, "%s/%s ", - route->out->p->name, - flb_output_name(route->out)); - if (!tmp) { - flb_sds_destroy(routes); - flb_error("[task] cannot print report for pending tasks"); - return -1; - } - routes = tmp; - } - - flb_info("[task] task_id=%i still running on route(s): %s", - task->id, routes); - flb_sds_len_set(routes, 0); - } - } - flb_sds_destroy(routes); - return 0; -} - -/* Create an engine task to handle the output plugin flushing work */ -struct flb_task *flb_task_create(uint64_t ref_id, - const char *buf, - size_t size, - struct flb_input_instance *i_ins, - struct flb_input_chunk *ic, - const char *tag_buf, int tag_len, - struct flb_config *config, - int *err) -{ - int count = 0; - int total_events = 0; - struct flb_task *task; - struct flb_event_chunk *evc; - struct flb_task_route *route; - struct flb_router_path *route_path; - struct flb_output_instance *o_ins; - struct flb_input_chunk *task_ic; - struct mk_list *i_head; - struct mk_list *o_head; - - /* No error status */ - *err = FLB_FALSE; - - /* allocate task */ - task = task_alloc(config); - if (!task) { - *err = FLB_TRUE; - return NULL; - } - -#ifdef FLB_HAVE_METRICS - total_events = ((struct flb_input_chunk *) ic)->total_records; -#endif - - /* event chunk */ - evc = flb_event_chunk_create(ic->event_type, - total_events, - (char *) tag_buf, tag_len, - (char *) buf, size); - if (!evc) { - flb_free(task); - *err = FLB_TRUE; - return NULL; - } - task->event_chunk = evc; - task_ic = (struct flb_input_chunk *) ic; - task_ic->task = task; - - /* Keep track of origins */ - task->ref_id = ref_id; - task->i_ins = i_ins; - task->ic = ic; - mk_list_add(&task->_head, &i_ins->tasks); - -#ifdef FLB_HAVE_METRICS - task->records = ((struct flb_input_chunk *) ic)->total_records; -#endif - - /* Direct connects betweek input <> outputs (API based) */ - if (mk_list_size(&i_ins->routes_direct) > 0) { - mk_list_foreach(i_head, &i_ins->routes_direct) { - route_path = mk_list_entry(i_head, struct flb_router_path, _head); - o_ins = route_path->ins; - - route = flb_malloc(sizeof(struct flb_task_route)); - if (!route) { - flb_errno(); - task->event_chunk->data = NULL; - flb_task_destroy(task, FLB_TRUE); - return NULL; - } - - route->out = o_ins; - mk_list_add(&route->_head, &task->routes); - } - flb_debug("[task] created direct task=%p id=%i OK", task, task->id); - return task; - } - - /* Find matching routes for the incoming task */ - mk_list_foreach(o_head, &config->outputs) { - o_ins = mk_list_entry(o_head, - struct flb_output_instance, _head); - - /* skip output plugins that don't handle proper event types */ - if (!flb_router_match_type(ic->event_type, o_ins)) { - continue; - } - - if (flb_routes_mask_get_bit(task_ic->routes_mask, o_ins->id) != 0) { - route = flb_calloc(1, sizeof(struct flb_task_route)); - if (!route) { - flb_errno(); - continue; - } - - route->status = FLB_TASK_ROUTE_INACTIVE; - route->out = o_ins; - mk_list_add(&route->_head, &task->routes); - count++; - } - } - - /* no destinations ?, useless task. */ - if (count == 0) { - flb_debug("[task] created task=%p id=%i without routes, dropping.", - task, task->id); - task->event_chunk->data = NULL; - flb_task_destroy(task, FLB_TRUE); - return NULL; - } - - flb_debug("[task] created task=%p id=%i OK", task, task->id); - return task; -} - -void flb_task_destroy(struct flb_task *task, int del) -{ - struct mk_list *tmp; - struct mk_list *head; - struct flb_task_route *route; - struct flb_task_retry *retry; - - flb_debug("[task] destroy task=%p (task_id=%i)", task, task->id); - - /* Release task_id */ - map_free_task_id(task->id, task->config); - - /* Remove routes */ - mk_list_foreach_safe(head, tmp, &task->routes) { - route = mk_list_entry(head, struct flb_task_route, _head); - mk_list_del(&route->_head); - flb_free(route); - } - - /* Unlink and release task */ - mk_list_del(&task->_head); - - /* destroy chunk */ - flb_input_chunk_destroy(task->ic, del); - - /* Remove 'retries' */ - mk_list_foreach_safe(head, tmp, &task->retries) { - retry = mk_list_entry(head, struct flb_task_retry, _head); - flb_task_retry_destroy(retry); - } - - flb_input_chunk_set_limits(task->i_ins); - - if (task->event_chunk) { - flb_event_chunk_destroy(task->event_chunk); - } - flb_free(task); -} - -struct flb_task_queue* flb_task_queue_create() { - struct flb_task_queue *tq; - tq = flb_malloc(sizeof(struct flb_task_queue)); - if (!tq) { - flb_errno(); - return NULL; - } - mk_list_init(&tq->pending); - mk_list_init(&tq->in_progress); - return tq; -} - -void flb_task_queue_destroy(struct flb_task_queue *queue) { - struct flb_task_enqueued *queued_task; - struct mk_list *tmp; - struct mk_list *head; - - mk_list_foreach_safe(head, tmp, &queue->pending) { - queued_task = mk_list_entry(head, struct flb_task_enqueued, _head); - mk_list_del(&queued_task->_head); - flb_free(queued_task); - } - - mk_list_foreach_safe(head, tmp, &queue->in_progress) { - queued_task = mk_list_entry(head, struct flb_task_enqueued, _head); - mk_list_del(&queued_task->_head); - flb_free(queued_task); - } - - flb_free(queue); -} diff --git a/fluent-bit/src/flb_thread_pool.c b/fluent-bit/src/flb_thread_pool.c deleted file mode 100644 index e3b854717..000000000 --- a/fluent-bit/src/flb_thread_pool.c +++ /dev/null @@ -1,209 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <fluent-bit/flb_info.h> -#include <fluent-bit/flb_mem.h> -#include <fluent-bit/flb_log.h> -#include <fluent-bit/flb_worker.h> -#include <fluent-bit/flb_thread_pool.h> - -/* Return the next thread id. We use the list size to set an id */ -static int flb_tp_thread_get_id(struct flb_tp *tp) -{ - return mk_list_size(&tp->list_threads); -} - -/* Create a thread manager context */ -struct flb_tp *flb_tp_create(struct flb_config *config) -{ - struct flb_tp *tp; - - tp = flb_calloc(1, sizeof(struct flb_tp)); - if (!tp) { - flb_errno(); - return NULL; - } - tp->config = config; - mk_list_init(&tp->list_threads); - - return tp; -} - -void flb_tp_destroy(struct flb_tp *tp) -{ - struct mk_list *tmp; - struct mk_list *head; - struct flb_tp_thread *th; - - mk_list_foreach_safe(head, tmp, &tp->list_threads) { - th = mk_list_entry(head, struct flb_tp_thread, _head); - mk_list_del(&th->_head); - flb_free(th); - } - - flb_free(tp); -} - -struct flb_tp_thread *flb_tp_thread_create(struct flb_tp *tp, - void (*func)(void *), void *arg, - struct flb_config *config) - -{ - struct flb_tp_thread *th; - - /* Create thread context */ - th = flb_calloc(1, sizeof(struct flb_tp_thread)); - if (!th) { - flb_errno(); - return NULL; - } - th->config = config; - - /* - * To spawn a thread, we use the 'worker' interface. Since the worker will - * start the thread as soon as is invoked, we keep a reference to the worker - * parameters in our context and we only use them when the thread is really - * started through the call flb_tp_thread_start(). - */ - th->params.func = func; - th->params.data = arg; - - /* Status */ - th->status = FLB_THREAD_POOL_NONE; - - /* Set the thread id */ - th->id = flb_tp_thread_get_id(tp); - - /* Link this thread context to the parent context list */ - mk_list_add(&th->_head, &tp->list_threads); - - return th; -} - - -/* Get a candidate thread using round-robin */ -struct flb_tp_thread *flb_tp_thread_get_rr(struct flb_tp *tp) -{ - struct flb_tp_thread *th; - - if (!tp->thread_cur) { - th = mk_list_entry_first(&tp->list_threads, - struct flb_tp_thread, _head); - } - else { - th = mk_list_entry_next(tp->thread_cur, - struct flb_tp_thread, _head, - &tp->list_threads); - } - tp->thread_cur = &th->_head; - - return th; -} - -int flb_tp_thread_start(struct flb_tp *tp, struct flb_tp_thread *th) -{ - int ret; - - ret = flb_worker_create(th->params.func, th->params.data, &th->tid, - th->config); - if (ret == -1) { - th->status = FLB_THREAD_POOL_ERROR; - return -1; - } - - /* - * Retrieve the Worker context. The worker API don't return the - * id or the context, so we use the created pthread_t (task id) - * to obtain the reference. - */ - th->worker = flb_worker_lookup(th->tid, tp->config); - th->status = FLB_THREAD_POOL_RUNNING; - - return 0; -} - -int flb_tp_thread_start_id(struct flb_tp *tp, int id) -{ - int i = 0; - struct mk_list *head; - struct flb_tp_thread *th = NULL; - - mk_list_foreach(head, &tp->list_threads) { - if (i == id) { - th = mk_list_entry(head, struct flb_tp_thread, _head); - break; - } - th = NULL; - i++; - } - - if (!th) { - return -1; - } - - return flb_tp_thread_start(tp, th); -} - -int flb_tp_thread_start_all(struct flb_tp *tp) -{ - struct mk_list *head; - struct flb_tp_thread *th; - - mk_list_foreach(head, &tp->list_threads) { - th = mk_list_entry(head, struct flb_tp_thread, _head); - flb_tp_thread_start(tp, th); - } - - return 0; -} - -int flb_tp_thread_stop(struct flb_tp *tp, struct flb_tp_thread *th) -{ - return 0; -} - -int flb_tp_thread_stop_all(struct flb_tp *tp) -{ - int ret; - struct mk_list *head; - struct flb_tp_thread *th; - - /* - * Iterate each worker thread, signal them to stop working - * and wait a proper exit. - */ - mk_list_foreach(head, &tp->list_threads) { - th = mk_list_entry(head, struct flb_tp_thread, _head); - if (th->status != FLB_THREAD_POOL_RUNNING) { - continue; - } - - ret = flb_tp_thread_stop(tp, th); - if (ret == -1) { - - } - } - - return 0; -} - -int flb_tp_thread_destroy() -{ - return 0; -} diff --git a/fluent-bit/src/flb_time.c b/fluent-bit/src/flb_time.c deleted file mode 100644 index 624b70d02..000000000 --- a/fluent-bit/src/flb_time.c +++ /dev/null @@ -1,444 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "cmetrics/lib/mpack/src/mpack/mpack.h" -#include <msgpack.h> -#include <mpack/mpack.h> -#include <fluent-bit/flb_compat.h> -#include <fluent-bit/flb_macros.h> -#include <fluent-bit/flb_log.h> -#include <fluent-bit/flb_time.h> -#include <stdint.h> -#ifdef FLB_HAVE_CLOCK_GET_TIME -# include <mach/clock.h> -# include <mach/mach.h> -#endif - -#include <string.h> -#include <inttypes.h> -#include <time.h> - -#define ONESEC_IN_NSEC 1000000000 - -static int is_valid_format(int fmt) -{ - return (FLB_TIME_ETFMT_INT <= fmt) && (fmt < FLB_TIME_ETFMT_OTHER) ? - FLB_TRUE : FLB_FALSE; -} - -static int _flb_time_get(struct flb_time *tm) -{ - if (tm == NULL) { - return -1; - } -#if defined FLB_TIME_FORCE_FMT_INT - tm->tm.tv_sec = time(NULL); - tm->tm.tv_nsec = 0; - return 0; -#elif defined FLB_HAVE_TIMESPEC_GET - /* C11 supported! */ - return timespec_get(&tm->tm, TIME_UTC); -#elif defined FLB_CLOCK_GET_TIME - clock_serv_t cclock; - mach_timespec_t mts; - host_get_clock_service(mach_host_self(), CALENDAR_CLOCK, &cclock); - clock_get_time(cclock, &mts); - tm->tv_sec = mts.tv_sec; - tm->tv_nsec = mts.tv_nsec; - return mach_port_deallocate(mach_task_self(), cclock); -#else /* __STDC_VERSION__ */ - return clock_gettime(CLOCK_REALTIME, &tm->tm); -#endif -} - -int flb_time_get(struct flb_time *tm) -{ - return _flb_time_get(tm); -} - -/* A portable function to sleep N msec */ -int flb_time_msleep(uint32_t ms) -{ -#ifdef _MSC_VER - Sleep((DWORD) ms); - return 0; -#else - struct timespec ts; - ts.tv_sec = ms / 1000; - ts.tv_nsec = (ms % 1000) * 1000000; - return nanosleep(&ts, NULL); -#endif -} - -double flb_time_to_double(struct flb_time *tm) -{ - return (double)(tm->tm.tv_sec) + ((double)tm->tm.tv_nsec/(double)ONESEC_IN_NSEC); -} - -uint64_t flb_time_to_nanosec(struct flb_time *tm) -{ - return (((uint64_t)tm->tm.tv_sec * 1000000000L) + tm->tm.tv_nsec); -} - -uint64_t flb_time_to_millisec(struct flb_time *tm) -{ - return (((uint64_t)tm->tm.tv_sec * 1000L) + tm->tm.tv_nsec / 1000000L); -} - -int flb_time_add(struct flb_time *base, struct flb_time *duration, struct flb_time *result) -{ - if (base == NULL || duration == NULL|| result == NULL) { - return -1; - } - result->tm.tv_sec = base->tm.tv_sec + duration->tm.tv_sec; - result->tm.tv_nsec = base->tm.tv_nsec + duration->tm.tv_nsec; - - if (result->tm.tv_nsec > ONESEC_IN_NSEC) { - result->tm.tv_nsec -= ONESEC_IN_NSEC; - result->tm.tv_sec++; - } else if (result->tm.tv_nsec < 0) { - result->tm.tv_nsec += ONESEC_IN_NSEC; - result->tm.tv_sec--; - } - - return 0; -} - -int flb_time_diff(struct flb_time *time1, - struct flb_time *time0,struct flb_time *result) -{ - if (time1 == NULL || time0 == NULL || result == NULL) { - return -1; - } - - if (time1->tm.tv_sec >= time0->tm.tv_sec) { - result->tm.tv_sec = time1->tm.tv_sec - time0->tm.tv_sec; - if (time1->tm.tv_nsec >= time0->tm.tv_nsec) { - result->tm.tv_nsec = time1->tm.tv_nsec - time0->tm.tv_nsec; - } - else if(result->tm.tv_sec == 0){ - /* underflow */ - return -2; - } - else{ - result->tm.tv_nsec = ONESEC_IN_NSEC - + time1->tm.tv_nsec - time0->tm.tv_nsec; - result->tm.tv_sec--; - } - } - else { - /* underflow */ - return -3; - } - return 0; -} - -int flb_time_append_to_mpack(mpack_writer_t *writer, struct flb_time *tm, int fmt) -{ - int ret = 0; - struct flb_time l_time; - char ext_data[8]; - uint32_t tmp; - - if (!is_valid_format(fmt)) { -#ifdef FLB_TIME_FORCE_FMT_INT - fmt = FLB_TIME_ETFMT_INT; -#else - fmt = FLB_TIME_ETFMT_V1_FIXEXT; -#endif - } - - if (tm == NULL) { - if (fmt == FLB_TIME_ETFMT_INT) { - l_time.tm.tv_sec = time(NULL); - } - else { - _flb_time_get(&l_time); - } - tm = &l_time; - } - - switch(fmt) { - case FLB_TIME_ETFMT_INT: - mpack_write_uint(writer, tm->tm.tv_sec); - break; - - case FLB_TIME_ETFMT_V0: - case FLB_TIME_ETFMT_V1_EXT: - /* We can't set with msgpack-c !! */ - /* see pack_template.h and msgpack_pack_inline_func(_ext) */ - case FLB_TIME_ETFMT_V1_FIXEXT: - tmp = htonl((uint32_t)tm->tm.tv_sec); /* second from epoch */ - memcpy(&ext_data, &tmp, 4); - tmp = htonl((uint32_t)tm->tm.tv_nsec);/* nanosecond */ - memcpy(&ext_data[4], &tmp, 4); - - /* https://github.com/fluent/fluentd/wiki/Forward-Protocol-Specification-v1#eventtime-ext-format */ - mpack_write_ext(writer, 0 /*ext type=0 */, ext_data, sizeof(ext_data)); - break; - - default: - ret = -1; - } - - return ret; -} - -int flb_time_append_to_msgpack(struct flb_time *tm, msgpack_packer *pk, int fmt) -{ - int ret = 0; - struct flb_time l_time; - char ext_data[8]; - uint32_t tmp; - - if (!is_valid_format(fmt)) { -#ifdef FLB_TIME_FORCE_FMT_INT - fmt = FLB_TIME_ETFMT_INT; -#else - fmt = FLB_TIME_ETFMT_V1_FIXEXT; -#endif - } - - if (tm == NULL) { - if (fmt == FLB_TIME_ETFMT_INT) { - l_time.tm.tv_sec = time(NULL); - } - else { - _flb_time_get(&l_time); - } - tm = &l_time; - } - - switch(fmt) { - case FLB_TIME_ETFMT_INT: - msgpack_pack_uint64(pk, tm->tm.tv_sec); - break; - - case FLB_TIME_ETFMT_V0: - case FLB_TIME_ETFMT_V1_EXT: - /* We can't set with msgpack-c !! */ - /* see pack_template.h and msgpack_pack_inline_func(_ext) */ - case FLB_TIME_ETFMT_V1_FIXEXT: - tmp = htonl((uint32_t)tm->tm.tv_sec); /* second from epoch */ - memcpy(&ext_data, &tmp, 4); - tmp = htonl((uint32_t)tm->tm.tv_nsec);/* nanosecond */ - memcpy(&ext_data[4], &tmp, 4); - - msgpack_pack_ext(pk, 8/*fixext8*/, 0); - msgpack_pack_ext_body(pk, ext_data, sizeof(ext_data)); - - break; - - default: - ret = -1; - } - - return ret; -} - -static inline int is_eventtime(msgpack_object *obj) -{ - if (obj->via.ext.type != 0 || obj->via.ext.size != 8) { - return FLB_FALSE; - } - return FLB_TRUE; -} - -int flb_time_msgpack_to_time(struct flb_time *time, msgpack_object *obj) -{ - uint32_t tmp; - - switch(obj->type) { - case MSGPACK_OBJECT_POSITIVE_INTEGER: - time->tm.tv_sec = obj->via.u64; - time->tm.tv_nsec = 0; - break; - case MSGPACK_OBJECT_FLOAT: - time->tm.tv_sec = obj->via.f64; - time->tm.tv_nsec = ((obj->via.f64 - time->tm.tv_sec) * ONESEC_IN_NSEC); - break; - case MSGPACK_OBJECT_EXT: - if (is_eventtime(obj) != FLB_TRUE) { - flb_warn("[time] unknown ext type. type=%d size=%d", - obj->via.ext.type, obj->via.ext.size); - return -1; - } - memcpy(&tmp, &obj->via.ext.ptr[0], 4); - time->tm.tv_sec = (uint32_t) ntohl(tmp); - memcpy(&tmp, &obj->via.ext.ptr[4], 4); - time->tm.tv_nsec = (uint32_t) ntohl(tmp); - break; - default: - flb_warn("unknown time format %x", obj->type); - return -1; - } - - return 0; -} - -int flb_time_pop_from_mpack(struct flb_time *time, mpack_reader_t *reader) -{ - mpack_tag_t tag; - double d; - float f; - int64_t i; - uint32_t tmp; - char extbuf[8]; - size_t ext_len; - int header_detected; - - if (time == NULL) { - return -1; - } - - header_detected = FLB_FALSE; - - /* consume the record array */ - tag = mpack_read_tag(reader); - - if (mpack_reader_error(reader) != mpack_ok || - mpack_tag_type(&tag) != mpack_type_array || - mpack_tag_array_count(&tag) == 0) { - return -1; - } - - /* consume the header array or the timestamp - * depending on the chunk encoding - */ - tag = mpack_read_tag(reader); - - if (mpack_reader_error(reader) != mpack_ok) { - return -1; - } - - if (mpack_tag_type(&tag) == mpack_type_array) { - if(mpack_tag_array_count(&tag) != 2) { - return -1; - } - - /* consume the timestamp element */ - tag = mpack_read_tag(reader); - - if (mpack_reader_error(reader) != mpack_ok) { - return -1; - } - - header_detected = FLB_TRUE; - } - - switch (mpack_tag_type(&tag)) { - case mpack_type_int: - i = mpack_tag_int_value(&tag); - if (i < 0) { - flb_warn("expecting positive integer, got %" PRId64, i); - return -1; - } - time->tm.tv_sec = i; - time->tm.tv_nsec = 0; - break; - case mpack_type_uint: - time->tm.tv_sec = mpack_tag_uint_value(&tag); - time->tm.tv_nsec = 0; - break; - case mpack_type_float: - f = mpack_tag_float_value(&tag); - time->tm.tv_sec = f; - time->tm.tv_nsec = ((f - time->tm.tv_sec) * ONESEC_IN_NSEC); - case mpack_type_double: - d = mpack_tag_double_value(&tag); - time->tm.tv_sec = d; - time->tm.tv_nsec = ((d - time->tm.tv_sec) * ONESEC_IN_NSEC); - break; - case mpack_type_ext: - ext_len = mpack_tag_ext_length(&tag); - if (ext_len != 8) { - flb_warn("expecting ext_len is 8, got %ld", ext_len); - return -1; - } - mpack_read_bytes(reader, extbuf, ext_len); - memcpy(&tmp, extbuf, 4); - time->tm.tv_sec = (uint32_t) ntohl(tmp); - memcpy(&tmp, extbuf + 4, 4); - time->tm.tv_nsec = (uint32_t) ntohl(tmp); - break; - default: - flb_warn("unknown time format %d", tag.type); - return -1; - } - - /* discard the metadata map if present */ - - if (header_detected) { - mpack_discard(reader); - } - - return 0; -} - -int flb_time_pop_from_msgpack(struct flb_time *time, msgpack_unpacked *upk, - msgpack_object **map) -{ - int ret; - msgpack_object obj; - - if (time == NULL || upk == NULL) { - return -1; - } - - if (upk->data.type != MSGPACK_OBJECT_ARRAY) { - return -1; - } - - obj = upk->data.via.array.ptr[0]; - - if (obj.type == MSGPACK_OBJECT_ARRAY) { - if (obj.via.array.size != 2) { - return -1; - } - - obj = obj.via.array.ptr[0]; - } - - *map = &upk->data.via.array.ptr[1]; - - ret = flb_time_msgpack_to_time(time, &obj); - return ret; -} - -long flb_time_tz_offset_to_second() -{ - time_t t = time(NULL); - struct tm local = *localtime(&t); - struct tm utc = *gmtime(&t); - - long diff = ((local.tm_hour - utc.tm_hour) \ - * 60 + (local.tm_min - utc.tm_min)) \ - * 60L + (local.tm_sec - utc.tm_sec); - - int delta_day = local.tm_mday - utc.tm_mday; - - if ((delta_day == 1) || (delta_day < -1)) { - diff += 24L * 60 * 60; - } - else if ((delta_day == -1) || (delta_day > 1)) { - diff -= 24L * 60 * 60; - } - - return diff; -} diff --git a/fluent-bit/src/flb_typecast.c b/fluent-bit/src/flb_typecast.c deleted file mode 100644 index 4c2d10536..000000000 --- a/fluent-bit/src/flb_typecast.c +++ /dev/null @@ -1,525 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <fluent-bit/flb_mem.h> -#include <fluent-bit/flb_utils.h> -#include <fluent-bit/flb_typecast.h> -#include <string.h> -#include <inttypes.h> -#include <msgpack.h> - -flb_typecast_type_t flb_typecast_str_to_type_t(char *type_str, int type_len) -{ - if (!strncasecmp(type_str, "int", type_len)) { - return FLB_TYPECAST_TYPE_INT; - } - else if (!strncasecmp(type_str, "uint", type_len)) { - return FLB_TYPECAST_TYPE_UINT; - } - else if (!strncasecmp(type_str, "float", type_len)) { - return FLB_TYPECAST_TYPE_FLOAT; - } - else if (!strncasecmp(type_str, "hex", type_len)) { - return FLB_TYPECAST_TYPE_HEX; - } - else if (!strncasecmp(type_str, "string", type_len)) { - return FLB_TYPECAST_TYPE_STR; - } - else if(!strncasecmp(type_str, "bool", type_len)) { - return FLB_TYPECAST_TYPE_BOOL; - } - - return FLB_TYPECAST_TYPE_ERROR; -} - -const char * flb_typecast_type_t_to_str(flb_typecast_type_t type) -{ - switch(type) { - case FLB_TYPECAST_TYPE_INT: - return "int"; - case FLB_TYPECAST_TYPE_UINT: - return "uint"; - case FLB_TYPECAST_TYPE_FLOAT: - return "float"; - case FLB_TYPECAST_TYPE_HEX: - return "hex"; - case FLB_TYPECAST_TYPE_STR: - return "string"; - case FLB_TYPECAST_TYPE_BOOL: - return "bool"; - default: - return "unknown type"; - } - -} - -static int flb_typecast_conv_str(const char *input, int input_len, - struct flb_typecast_rule *rule, - msgpack_packer *pck, - struct flb_typecast_value *output) -{ - flb_sds_t tmp_str; - int ret = 0; - - if(input == NULL || rule == NULL || output == NULL) { - return -1; - } - else if (rule->from_type != FLB_TYPECAST_TYPE_STR) { - flb_error("%s: Type is not string.",__FUNCTION__); - return -1; - } - - /* - * msgpack char is not null terminated. - * So make a temporary copy. - */ - tmp_str = flb_sds_create_len(input, input_len); - if (tmp_str == NULL) { - flb_errno(); - return -1; - } - - switch(rule->to_type) { - case FLB_TYPECAST_TYPE_INT: - output->val.i_num = strtoimax(tmp_str, NULL, 10); - if (output->val.i_num == 0) { - flb_error("%s: convert error. input=%s", __FUNCTION__, tmp_str); - ret = -1; - goto typecast_conv_str_end; - } - if (pck != NULL) { - msgpack_pack_int64(pck, output->val.i_num); - } - break; - case FLB_TYPECAST_TYPE_UINT: - output->val.ui_num = strtoumax(tmp_str, NULL, 10); - if (output->val.ui_num == 0) { - flb_error("%s: convert error. input=%s", __FUNCTION__, tmp_str); - ret = -1; - goto typecast_conv_str_end; - } - if (pck != NULL) { - msgpack_pack_uint64(pck, output->val.ui_num); - } - break; - case FLB_TYPECAST_TYPE_HEX: - output->val.ui_num = strtoumax(tmp_str, NULL, 16); - if (output->val.ui_num == 0) { - flb_error("%s: convert error. input=%s", __FUNCTION__, tmp_str); - ret = -1; - goto typecast_conv_str_end; - } - if (pck != NULL) { - msgpack_pack_uint64(pck, output->val.ui_num); - } - break; - case FLB_TYPECAST_TYPE_FLOAT: - output->val.d_num = atof(tmp_str); - if (pck != NULL) { - msgpack_pack_double(pck, output->val.d_num); - } - break; - case FLB_TYPECAST_TYPE_BOOL: - if (input_len >= 4 && !strncasecmp(tmp_str, "true", 4)) { - output->val.boolean = FLB_TRUE; - } - else if (input_len >= 5 && !strncasecmp(tmp_str, "false", 5)) { - output->val.boolean = FLB_FALSE; - } - else { - flb_error("%s: convert error. input=%s", __FUNCTION__, tmp_str); - ret = -1; - goto typecast_conv_str_end; - } - - if (pck != NULL) { - if (output->val.boolean) { - msgpack_pack_true(pck); - } - else { - msgpack_pack_false(pck); - } - } - - break; - case FLB_TYPECAST_TYPE_STR: - flb_error("%s: str to str. nothing to do.", __FUNCTION__); - return -1; - break; - default: - flb_error("%s: unknown type %d", __FUNCTION__, rule->to_type); - ret = -1; - } - typecast_conv_str_end: - flb_sds_destroy(tmp_str); - return ret; -} - -static int flb_typecast_conv_bool(int input_bool, - struct flb_typecast_rule *rule, - msgpack_packer *pck, - struct flb_typecast_value *output) -{ - if(rule == NULL || output == NULL) { - return -1; - } - - if (rule->to_type != FLB_TYPECAST_TYPE_STR) { - flb_error("%s: type %s is not supported",__FUNCTION__, - flb_typecast_type_t_to_str(rule->to_type)); - return -1; - } - - if (input_bool == FLB_TRUE) { - output->val.str = flb_sds_create_len("true", 4); - if (pck != NULL) { - msgpack_pack_str(pck, 4); - msgpack_pack_str_body(pck, "true", 4); - } - return 0; - } - else if (input_bool == FLB_FALSE) { - output->val.str = flb_sds_create_len("false", 5); - if (pck != NULL) { - msgpack_pack_str(pck, 5); - msgpack_pack_str_body(pck, "false", 5); - } - return 0; - } - flb_error("%s: unsupported input %d",__FUNCTION__, - input_bool); - return -1; -} - -static int flb_typecast_conv_int(int64_t input, - struct flb_typecast_rule *rule, - msgpack_packer *pck, - struct flb_typecast_value *output) -{ - char temp[32] = {0}; - int i; - - if(rule == NULL || output == NULL) { - return -1; - } - - switch(rule->to_type) { - case FLB_TYPECAST_TYPE_STR: - i = snprintf(temp, sizeof(temp) -1, "%"PRId64, input); - output->val.str = flb_sds_create_len(temp, i); - if(pck != NULL) { - msgpack_pack_str(pck, i); - msgpack_pack_str_body(pck, output->val.str, i); - } - break; - - case FLB_TYPECAST_TYPE_FLOAT: - output->val.d_num = (double)input; - if (pck != NULL) { - msgpack_pack_double(pck, output->val.d_num); - } - break; - case FLB_TYPECAST_TYPE_UINT: - output->val.ui_num = (uint64_t)input; - if (pck != NULL) { - msgpack_pack_uint64(pck, output->val.ui_num); - } - break; - - default: - flb_error("%s: type %s is not supported",__FUNCTION__, - flb_typecast_type_t_to_str(rule->to_type)); - return -1; - } - return 0; -} - -static int flb_typecast_conv_uint(uint64_t input, - struct flb_typecast_rule *rule, - msgpack_packer *pck, - struct flb_typecast_value *output) -{ - char temp[32] = {0}; - int i; - - if(rule == NULL || output == NULL) { - return -1; - } - - switch(rule->to_type) { - case FLB_TYPECAST_TYPE_STR: - i = snprintf(temp, sizeof(temp) -1, "%"PRIu64, input); - output->val.str = flb_sds_create_len(temp, i); - if(pck != NULL) { - msgpack_pack_str(pck, i); - msgpack_pack_str_body(pck, output->val.str, i); - } - break; - - case FLB_TYPECAST_TYPE_FLOAT: - output->val.d_num = (double)input; - if (pck != NULL) { - msgpack_pack_double(pck, output->val.d_num); - } - break; - case FLB_TYPECAST_TYPE_INT: - output->val.i_num = (int64_t)input; - if (pck != NULL) { - msgpack_pack_int64(pck, output->val.ui_num); - } - break; - - default: - flb_error("%s: type %s is not supported",__FUNCTION__, - flb_typecast_type_t_to_str(rule->to_type)); - return -1; - } - return 0; -} - -static int flb_typecast_conv_float(double input, - struct flb_typecast_rule *rule, - msgpack_packer *pck, - struct flb_typecast_value *output) -{ - char temp[512] = {0}; - int i; - - if(rule == NULL || output == NULL) { - return -1; - } - - switch(rule->to_type) { - case FLB_TYPECAST_TYPE_STR: - if (input == (double)(long long int)input) { - i = snprintf(temp, sizeof(temp)-1, "%.1f", input); - } - else { - i = snprintf(temp, sizeof(temp)-1, "%.16g", input); - } - output->val.str = flb_sds_create_len(temp, i); - if(pck != NULL) { - msgpack_pack_str(pck, i); - msgpack_pack_str_body(pck, output->val.str, i); - } - break; - case FLB_TYPECAST_TYPE_INT: - output->val.i_num = (int64_t)input; - if (pck != NULL) { - msgpack_pack_int64(pck, output->val.ui_num); - } - break; - case FLB_TYPECAST_TYPE_UINT: - output->val.ui_num = (uint64_t)input; - if (pck != NULL) { - msgpack_pack_uint64(pck, output->val.ui_num); - } - break; - - default: - flb_error("%s: type %s is not supported",__FUNCTION__, - flb_typecast_type_t_to_str(rule->to_type)); - return -1; - } - return 0; -} - -int flb_typecast_rule_destroy(struct flb_typecast_rule *rule) -{ - if(rule == NULL) { - return 0; - } - flb_free(rule); - - return 0; -} - -struct flb_typecast_rule *flb_typecast_rule_create(char *from_type, int from_len, - char *to_type, int to_len) -{ - struct flb_typecast_rule *rule = NULL; - - if (from_type == NULL || to_type == NULL) { - return NULL; - } - rule = flb_malloc(sizeof(struct flb_typecast_rule)); - if (rule == NULL) { - flb_errno(); - return NULL; - } - - rule->from_type = flb_typecast_str_to_type_t(from_type, from_len); - if (rule->from_type == FLB_TYPECAST_TYPE_ERROR) { - flb_error("%s: unknown from str %s", __FUNCTION__, from_type); - flb_typecast_rule_destroy(rule); - return NULL; - } - - rule->to_type = flb_typecast_str_to_type_t(to_type, to_len); - if (rule->to_type == FLB_TYPECAST_TYPE_ERROR) { - flb_error("%s: unknown to str %s", __FUNCTION__, to_type); - flb_typecast_rule_destroy(rule); - return NULL; - } - - return rule; -} - - -/** - * Convert msgpack object according to a rule. - * - * @param input msgpack object to be converted - * @param rule conversion rule - * @param pck msgpack packer to write converted object. If NULL, not to write. - * @param output converted value. User must call flb_typecast_value_destroy after using. - * If NULL, not to be filled. - * - * @return 0 : success, !0: fail - */ -static int flb_typecast_value_conv(msgpack_object input, - struct flb_typecast_rule *rule, - msgpack_packer *pck, - struct flb_typecast_value *output) -{ - int ret = -1; - - if (rule == NULL || output == NULL) { - return -1; - } - - switch(rule->from_type) { - case FLB_TYPECAST_TYPE_STR: - if (input.type != MSGPACK_OBJECT_STR) { - flb_error("%s: src type is not str", __FUNCTION__); - return -1; - } - ret = flb_typecast_conv_str(input.via.str.ptr, - input.via.str.size, - rule , pck, output); - break; - case FLB_TYPECAST_TYPE_BOOL: - if (input.type != MSGPACK_OBJECT_BOOLEAN) { - flb_error("%s: src type is not boolean", __FUNCTION__); - return -1; - } - ret = flb_typecast_conv_bool(input.via.boolean ? FLB_TRUE:FLB_FALSE, - rule, pck, output); - break; - case FLB_TYPECAST_TYPE_INT: - if (input.type != MSGPACK_OBJECT_POSITIVE_INTEGER && - input.type != MSGPACK_OBJECT_NEGATIVE_INTEGER) { - flb_error("%s: src type is not int", __FUNCTION__); - return -1; - } - ret = flb_typecast_conv_int(input.via.i64, rule, pck, output); - - break; - case FLB_TYPECAST_TYPE_UINT: - if (input.type != MSGPACK_OBJECT_POSITIVE_INTEGER && - input.type != MSGPACK_OBJECT_NEGATIVE_INTEGER) { - flb_error("%s: src type is not uint", __FUNCTION__); - return -1; - } - ret = flb_typecast_conv_uint(input.via.u64, rule, pck, output); - - break; - case FLB_TYPECAST_TYPE_FLOAT: - if (input.type != MSGPACK_OBJECT_FLOAT32 && - input.type != MSGPACK_OBJECT_FLOAT64) { - flb_error("%s: src type is not float", __FUNCTION__); - return -1; - } - ret = flb_typecast_conv_float(input.via.f64, rule, pck, output); - - break; - - default: - flb_error("%s: unknown type %d", __FUNCTION__, rule->from_type); - } - return ret; -} - -int flb_typecast_value_destroy(struct flb_typecast_value* val) -{ - if (val == NULL) { - return 0; - } - if (val->type == FLB_TYPECAST_TYPE_STR) { - flb_sds_destroy(val->val.str); - } - flb_free(val); - return 0; -} - - -struct flb_typecast_value *flb_typecast_value_create(msgpack_object input, - struct flb_typecast_rule *rule) -{ - int ret = -1; - struct flb_typecast_value *val; - - if (rule == NULL) { - return NULL; - } - val = flb_malloc(sizeof(struct flb_typecast_value)); - if (val == NULL) { - flb_errno(); - return NULL; - } - val->type = FLB_TYPECAST_TYPE_ERROR; - ret = flb_typecast_value_conv(input, rule, NULL, val); - if (ret < 0) { - flb_free(val); - return NULL; - } - val->type = rule->to_type; - - return val; -} - -/** - * Convert msgpack object according to a rule. - * - * @param input msgpack object to be converted - * @param rule conversion rule - * @param pck msgpack packer to write converted object - * - * @return 0 : success, !0: fail - */ -int flb_typecast_pack(msgpack_object input, - struct flb_typecast_rule *rule, - msgpack_packer *pck) -{ - int ret = -1; - struct flb_typecast_value val; - - if (rule == NULL || pck == NULL) { - flb_error("%s: input is null", __FUNCTION__); - return -1; - } - - ret = flb_typecast_value_conv(input, rule, pck, &val); - - if (ret == 0 && rule->to_type == FLB_TYPECAST_TYPE_STR) { - flb_sds_destroy(val.val.str); - } - - return ret; -} diff --git a/fluent-bit/src/flb_unescape.c b/fluent-bit/src/flb_unescape.c deleted file mode 100644 index 44f575b41..000000000 --- a/fluent-bit/src/flb_unescape.c +++ /dev/null @@ -1,328 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <fluent-bit/flb_compat.h> -#include <fluent-bit/flb_info.h> -#include <fluent-bit/flb_log.h> - -#include <stdlib.h> -#include <string.h> -#include <inttypes.h> - -static int octal_digit(char c) -{ - return (c >= '0' && c <= '7'); -} - -static int hex_digit(char c) -{ - return ((c >= '0' && c <= '9') || - (c >= 'A' && c <= 'F') || - (c >= 'a' && c <= 'f')); -} - -static int u8_wc_toutf8(char *dest, uint32_t ch) -{ - if (ch < 0x80) { - dest[0] = (char)ch; - return 1; - } - if (ch < 0x800) { - dest[0] = (ch>>6) | 0xC0; - dest[1] = (ch & 0x3F) | 0x80; - return 2; - } - if (ch < 0x10000) { - dest[0] = (ch>>12) | 0xE0; - dest[1] = ((ch>>6) & 0x3F) | 0x80; - dest[2] = (ch & 0x3F) | 0x80; - return 3; - } - if (ch < 0x110000) { - dest[0] = (ch>>18) | 0xF0; - dest[1] = ((ch>>12) & 0x3F) | 0x80; - dest[2] = ((ch>>6) & 0x3F) | 0x80; - dest[3] = (ch & 0x3F) | 0x80; - return 4; - } - return 0; -} - -/* assumes that src points to the character after a backslash - returns number of input characters processed */ -static int u8_read_escape_sequence(const char *str, int size, uint32_t *dest) -{ - uint32_t ch; - char digs[9]="\0\0\0\0\0\0\0\0"; - int dno=0, i=1; - - ch = (uint32_t)str[0]; /* take literal character */ - - if (str[0] == 'n') - ch = L'\n'; - else if (str[0] == 't') - ch = L'\t'; - else if (str[0] == 'r') - ch = L'\r'; - else if (str[0] == 'b') - ch = L'\b'; - else if (str[0] == 'f') - ch = L'\f'; - else if (str[0] == 'v') - ch = L'\v'; - else if (str[0] == 'a') - ch = L'\a'; - else if (octal_digit(str[0])) { - i = 0; - do { - digs[dno++] = str[i++]; - } while (i < size && octal_digit(str[i]) && dno < 3); - ch = strtol(digs, NULL, 8); - } - else if (str[0] == 'x') { - while (i < size && hex_digit(str[i]) && dno < 2) { - digs[dno++] = str[i++]; - } - if (dno > 0) { - ch = strtol(digs, NULL, 16); - } - } - else if (str[0] == 'u') { - while (i < size && hex_digit(str[i]) && dno < 4) { - digs[dno++] = str[i++]; - } - if (dno > 0) { - ch = strtol(digs, NULL, 16); - } - } - else if (str[0] == 'U') { - while (i < size && hex_digit(str[i]) && dno < 8) { - digs[dno++] = str[i++]; - } - if (dno > 0) { - ch = strtol(digs, NULL, 16); - } - } - *dest = ch; - - return i; -} - -int flb_unescape_string_utf8(const char *in_buf, int sz, char *out_buf) -{ - uint32_t ch; - char temp[4]; - const char *end; - const char *next; - int size; - - - int count_out = 0; - int count_in = 0; - int esc_in = 0; - int esc_out = 0; - - end = in_buf + sz; - while (in_buf < end && *in_buf && count_in < sz) { - next = in_buf + 1; - if (next < end && *in_buf == '\\') { - esc_in = 2; - switch (*next) { - case '"': - ch = '"'; - break; - case '\'': - ch = '\''; - break; - case '\\': - ch = '\\'; - break; - case '/': - ch = '/'; - break; - case 'n': - ch = '\n'; - break; - case 'b': - ch = '\b'; - break; - case 't': - ch = '\t'; - break; - case 'f': - ch = '\f'; - break; - case 'r': - ch = '\r'; - break; - default: - size = end - next; - if (size > 0) { - esc_in = u8_read_escape_sequence(next, size, &ch) + 1; - } - else { - /* because char is unsigned char by default on arm, so we need to do a explicit conversion */ - ch = (uint32_t) (signed char) *in_buf; - esc_in = 1; - } - } - } - else { - /* explicit convert char to signed char */ - ch = (uint32_t) (signed char) *in_buf; - esc_in = 1; - } - - in_buf += esc_in; - count_in += esc_in; - - esc_out = u8_wc_toutf8(temp, ch); - if (esc_out > sz-count_out) { - flb_error("Crossing over string boundary"); - break; - } - - if (esc_out == 0) { - out_buf[count_out] = ch; - esc_out = 1; - } - else if (esc_out == 1) { - out_buf[count_out] = (char) temp[0]; - } - else { - memcpy(&out_buf[count_out], temp, esc_out); - } - count_out += esc_out; - } - if (count_in < sz) { - flb_error("Not at boundary but still NULL terminating : %d - '%s'", sz, in_buf); - } - out_buf[count_out] = '\0'; - return count_out; -} - -int flb_unescape_string(const char *buf, int buf_len, char **unesc_buf) -{ - int i = 0; - int j = 0; - char *p; - char n; - - p = *unesc_buf; - while (i < buf_len) { - if (buf[i] == '\\') { - if (i + 1 < buf_len) { - n = buf[i + 1]; - if (n == 'n') { - p[j++] = '\n'; - i++; - } - else if (n == 'a') { - p[j++] = '\a'; - i++; - } - else if (n == 'b') { - p[j++] = '\b'; - i++; - } - else if (n == 't') { - p[j++] = '\t'; - i++; - } - else if (n == 'v') { - p[j++] = '\v'; - i++; - } - else if (n == 'f') { - p[j++] = '\f'; - i++; - } - else if (n == 'r') { - p[j++] = '\r'; - i++; - } - else if (n == '\\') { - p[j++] = '\\'; - i++; - } - i++; - continue; - } - else { - i++; - } - } - p[j++] = buf[i++]; - } - p[j] = '\0'; - return j; -} - - -/* mysql unquote */ -int flb_mysql_unquote_string(char *buf, int buf_len, char **unesc_buf) -{ - int i = 0; - int j = 0; - char *p; - char n; - - p = *unesc_buf; - while (i < buf_len) { - if ((n = buf[i++]) != '\\') { - p[j++] = n; - } else if(i >= buf_len) { - p[j++] = n; - } else { - n = buf[i++]; - switch(n) { - case 'n': - p[j++] = '\n'; - break; - case 'r': - p[j++] = '\r'; - break; - case 't': - p[j++] = '\t'; - break; - case '\\': - p[j++] = '\\'; - break; - case '\'': - p[j++] = '\''; - break; - case '\"': - p[j++] = '\"'; - break; - case '0': - p[j++] = 0; - break; - case 'Z': - p[j++] = 0x1a; - break; - default: - p[j++] = '\\'; - p[j++] = n; - break; - } - } - } - p[j] = '\0'; - return j; -} diff --git a/fluent-bit/src/flb_upstream.c b/fluent-bit/src/flb_upstream.c deleted file mode 100644 index 9ec39b84c..000000000 --- a/fluent-bit/src/flb_upstream.c +++ /dev/null @@ -1,1202 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <monkey/mk_core.h> -#include <fluent-bit/flb_info.h> -#include <fluent-bit/flb_mem.h> -#include <fluent-bit/flb_kv.h> -#include <fluent-bit/flb_slist.h> -#include <fluent-bit/flb_str.h> -#include <fluent-bit/flb_upstream.h> -#include <fluent-bit/flb_io.h> -#include <fluent-bit/tls/flb_tls.h> -#include <fluent-bit/flb_utils.h> -#include <fluent-bit/flb_engine.h> -#include <fluent-bit/flb_config_map.h> -#include <fluent-bit/flb_thread_storage.h> - -FLB_TLS_DEFINE(struct mk_list, flb_upstream_list_key); - -/* Config map for Upstream networking setup */ -struct flb_config_map upstream_net[] = { - { - FLB_CONFIG_MAP_STR, "net.dns.mode", NULL, - 0, FLB_TRUE, offsetof(struct flb_net_setup, dns_mode), - "Select the primary DNS connection type (TCP or UDP)" - }, - - { - FLB_CONFIG_MAP_STR, "net.dns.resolver", NULL, - 0, FLB_TRUE, offsetof(struct flb_net_setup, dns_resolver), - "Select the primary DNS resolver type (LEGACY or ASYNC)" - }, - - { - FLB_CONFIG_MAP_BOOL, "net.dns.prefer_ipv4", "false", - 0, FLB_TRUE, offsetof(struct flb_net_setup, dns_prefer_ipv4), - "Prioritize IPv4 DNS results when trying to establish a connection" - }, - - { - FLB_CONFIG_MAP_BOOL, "net.keepalive", "true", - 0, FLB_TRUE, offsetof(struct flb_net_setup, keepalive), - "Enable or disable Keepalive support" - }, - - { - FLB_CONFIG_MAP_TIME, "net.keepalive_idle_timeout", "30s", - 0, FLB_TRUE, offsetof(struct flb_net_setup, keepalive_idle_timeout), - "Set maximum time allowed for an idle Keepalive connection" - }, - - { - FLB_CONFIG_MAP_TIME, "net.io_timeout", "0s", - 0, FLB_TRUE, offsetof(struct flb_net_setup, io_timeout), - "Set maximum time a connection can stay idle while assigned" - }, - - { - FLB_CONFIG_MAP_TIME, "net.connect_timeout", "10s", - 0, FLB_TRUE, offsetof(struct flb_net_setup, connect_timeout), - "Set maximum time allowed to establish a connection, this time " - "includes the TLS handshake" - }, - - { - FLB_CONFIG_MAP_BOOL, "net.connect_timeout_log_error", "true", - 0, FLB_TRUE, offsetof(struct flb_net_setup, connect_timeout_log_error), - "On connection timeout, specify if it should log an error. When disabled, " - "the timeout is logged as a debug message" - }, - - { - FLB_CONFIG_MAP_STR, "net.source_address", NULL, - 0, FLB_TRUE, offsetof(struct flb_net_setup, source_address), - "Specify network address to bind for data traffic" - }, - - { - FLB_CONFIG_MAP_INT, "net.keepalive_max_recycle", "2000", - 0, FLB_TRUE, offsetof(struct flb_net_setup, keepalive_max_recycle), - "Set maximum number of times a keepalive connection can be used " - "before it is retried." - }, - - { - FLB_CONFIG_MAP_INT, "net.max_worker_connections", "0", - 0, FLB_TRUE, offsetof(struct flb_net_setup, max_worker_connections), - "Set the maximum number of active TCP connections that can be used per worker thread." - }, - - /* EOF */ - {0} -}; - -int flb_upstream_needs_proxy(const char *host, const char *proxy, - const char *no_proxy); - -static void flb_upstream_increment_busy_connections_count( - struct flb_upstream *stream); - -static void flb_upstream_decrement_busy_connections_count( - struct flb_upstream *stream); - -static void flb_upstream_increment_total_connections_count( - struct flb_upstream *stream); - -static void flb_upstream_decrement_total_connections_count( - struct flb_upstream *stream); - -/* Enable thread-safe mode for upstream connection */ -void flb_upstream_thread_safe(struct flb_upstream *u) -{ - /* - * Upon upstream creation, automatically the upstream is linked into - * the main Fluent Bit context (struct flb_config *)->upstreams. We - * have to avoid any access to this context outside of the worker - * thread. - */ - - flb_stream_enable_thread_safety(&u->base); -} - -struct mk_list *flb_upstream_get_config_map(struct flb_config *config) -{ - size_t config_index; - struct mk_list *config_map; - - /* If a global dns mode was provided in the SERVICE category then we set it as - * the default value for net.dns_mode, that way the user can set a global value and - * override it on a per plugin basis, however, it's not because of this flexibility - * that it was done but because in order to be able to save the value in the - * flb_net_setup structure (and not lose it when flb_output_upstream_set overwrites - * the structure) we need to do it this way (or at least that's what I think) - */ - for (config_index = 0 ; upstream_net[config_index].name != NULL ; config_index++) { - if (config->dns_mode != NULL) { - if (strcmp(upstream_net[config_index].name, "net.dns.mode") == 0) { - upstream_net[config_index].def_value = config->dns_mode; - } - } - if (config->dns_resolver != NULL) { - if (strcmp(upstream_net[config_index].name, "net.dns.resolver") == 0) { - upstream_net[config_index].def_value = config->dns_resolver; - } - } - if (config->dns_prefer_ipv4) { - if (strcmp(upstream_net[config_index].name, - "net.dns.prefer_ipv4") == 0) { - upstream_net[config_index].def_value = "true"; - } - } - } - - config_map = flb_config_map_create(config, upstream_net); - - return config_map; -} - -void flb_upstream_queue_init(struct flb_upstream_queue *uq) -{ - mk_list_init(&uq->av_queue); - mk_list_init(&uq->busy_queue); - mk_list_init(&uq->destroy_queue); -} - -struct flb_upstream_queue *flb_upstream_queue_get(struct flb_upstream *u) -{ - struct mk_list *head; - struct mk_list *list; - struct flb_upstream *th_u; - struct flb_upstream_queue *uq; - - /* - * Get the upstream queue, this might be different if the upstream is running - * in single-thread or multi-thread mode. - */ - if (flb_stream_is_thread_safe(&u->base) == FLB_TRUE) { - list = flb_upstream_list_get(); - if (!list) { - /* - * Here is the issue: a plugin enabled in multiworker mode in the - * initialization callback might wanted to use an upstream - * connection, but the init callback does not run in threaded mode - * so we hit this problem. - * - * As a fallback mechanism: just cross our fingers and return the - * principal upstream queue. - */ - return (struct flb_upstream_queue *) &u->queue; - } - - mk_list_foreach(head, list) { - th_u = mk_list_entry(head, struct flb_upstream, base._head); - if (th_u->parent_upstream == u) { - break; - } - th_u = NULL; - } - - if (!th_u) { - return NULL; - } - uq = &th_u->queue; - } - else { - uq = &u->queue; - } - - return uq; -} - -void flb_upstream_list_set(struct mk_list *list) -{ - FLB_TLS_SET(flb_upstream_list_key, list); -} - -struct mk_list *flb_upstream_list_get() -{ - return FLB_TLS_GET(flb_upstream_list_key); -} - -/* Initialize any upstream environment context */ -void flb_upstream_init() -{ - /* Initialize the upstream queue thread local storage */ - FLB_TLS_INIT(flb_upstream_list_key); -} - -/* Creates a new upstream context */ -struct flb_upstream *flb_upstream_create(struct flb_config *config, - const char *host, int port, int flags, - struct flb_tls *tls) -{ - int ret; - char *proxy_protocol = NULL; - char *proxy_host = NULL; - char *proxy_port = NULL; - char *proxy_username = NULL; - char *proxy_password = NULL; - struct flb_upstream *u; - - u = flb_calloc(1, sizeof(struct flb_upstream)); - if (!u) { - flb_errno(); - return NULL; - } - - u->base.dynamically_allocated = FLB_TRUE; - - flb_stream_setup(&u->base, - FLB_UPSTREAM, - FLB_TRANSPORT_TCP, - flags, - tls, - config, - NULL); - - /* Set upstream to the http_proxy if it is specified. */ - if (flb_upstream_needs_proxy(host, config->http_proxy, config->no_proxy) == FLB_TRUE) { - flb_debug("[upstream] config->http_proxy: %s", config->http_proxy); - ret = flb_utils_proxy_url_split(config->http_proxy, &proxy_protocol, - &proxy_username, &proxy_password, - &proxy_host, &proxy_port); - if (ret == -1) { - flb_errno(); - flb_free(u); - return NULL; - } - - u->tcp_host = flb_strdup(proxy_host); - u->tcp_port = atoi(proxy_port); - u->proxied_host = flb_strdup(host); - u->proxied_port = port; - - if (proxy_username && proxy_password) { - u->proxy_username = flb_strdup(proxy_username); - u->proxy_password = flb_strdup(proxy_password); - } - - flb_free(proxy_protocol); - flb_free(proxy_host); - flb_free(proxy_port); - flb_free(proxy_username); - flb_free(proxy_password); - } - else { - u->tcp_host = flb_strdup(host); - u->tcp_port = port; - } - - if (!u->tcp_host) { - flb_free(u); - return NULL; - } - - flb_stream_enable_flags(&u->base, FLB_IO_ASYNC); - - /* Initialize queues */ - flb_upstream_queue_init(&u->queue); - - mk_list_add(&u->base._head, &config->upstreams); - - return u; -} - -/* - * Checks whehter a destinate URL should be proxied. - */ -int flb_upstream_needs_proxy(const char *host, const char *proxy, - const char *no_proxy) -{ - int ret; - struct mk_list no_proxy_list; - struct mk_list *head; - struct flb_slist_entry *e = NULL; - - /* No HTTP_PROXY, should not set up proxy for the upstream `host`. */ - if (proxy == NULL) { - return FLB_FALSE; - } - - /* No NO_PROXY with HTTP_PROXY set, should set up proxy for the upstream `host`. */ - if (no_proxy == NULL) { - return FLB_TRUE; - } - - /* NO_PROXY=`*`, it matches all hosts. */ - if (strcmp(no_proxy, "*") == 0) { - return FLB_FALSE; - } - - /* check the URL list in the NO_PROXY */ - ret = flb_slist_create(&no_proxy_list); - if (ret != 0) { - return FLB_TRUE; - } - ret = flb_slist_split_string(&no_proxy_list, no_proxy, ',', -1); - if (ret <= 0) { - return FLB_TRUE; - } - ret = FLB_TRUE; - mk_list_foreach(head, &no_proxy_list) { - e = mk_list_entry(head, struct flb_slist_entry, _head); - if (strcmp(host, e->str) == 0) { - ret = FLB_FALSE; - break; - } - } - - /* clean up the resources. */ - flb_slist_destroy(&no_proxy_list); - - return ret; -} - -/* Create an upstream context using a valid URL (protocol, host and port) */ -struct flb_upstream *flb_upstream_create_url(struct flb_config *config, - const char *url, int flags, - struct flb_tls *tls) -{ - int ret; - int tmp_port = 0; - char *prot = NULL; - char *host = NULL; - char *port = NULL; - char *uri = NULL; - struct flb_upstream *u = NULL; - - /* Parse and split URL */ - ret = flb_utils_url_split(url, &prot, &host, &port, &uri); - if (ret == -1) { - flb_error("[upstream] invalid URL: %s", url); - return NULL; - } - - if (!prot) { - flb_error("[upstream] unknown protocol type from URL: %s", url); - goto out; - } - - /* Manage some default ports */ - if (!port) { - if (strcasecmp(prot, "http") == 0) { - tmp_port = 80; - } - else if (strcasecmp(prot, "https") == 0) { - tmp_port = 443; - if ((flags & FLB_IO_TLS) == 0) { - flags |= FLB_IO_TLS; - } - } - } - else { - tmp_port = atoi(port); - } - - if (tmp_port <= 0) { - flb_error("[upstream] unknown TCP port in URL: %s", url); - goto out; - } - - u = flb_upstream_create(config, host, tmp_port, flags, tls); - if (!u) { - flb_error("[upstream] error creating context from URL: %s", url); - } - - out: - if (prot) { - flb_free(prot); - } - if (host) { - flb_free(host); - } - if (port) { - flb_free(port); - } - if (uri) { - flb_free(uri); - } - - return u; -} - -/* This function shuts the connection down in order to cause - * any client code trying to read or write from it to fail. - */ -static void shutdown_connection(struct flb_connection *u_conn) -{ - if (u_conn->fd > 0 && - !u_conn->shutdown_flag) { - shutdown(u_conn->fd, SHUT_RDWR); - - u_conn->shutdown_flag = FLB_TRUE; - } -} - -/* - * This function moves the 'upstream connection' into the queue to be - * destroyed. Note that the caller is responsible to validate and check - * required mutex if this is being used in multi-worker mode. - */ -static int prepare_destroy_conn(struct flb_connection *u_conn) -{ - struct flb_upstream *u; - struct flb_upstream_queue *uq; - - u = u_conn->upstream; - - uq = flb_upstream_queue_get(u); - - flb_trace("[upstream] destroy connection #%i to %s:%i", - u_conn->fd, u->tcp_host, u->tcp_port); - - if (MK_EVENT_IS_REGISTERED((&u_conn->event))) { - mk_event_del(u_conn->evl, &u_conn->event); - } - - if (u_conn->fd > 0) { -#ifdef FLB_HAVE_TLS - if (u_conn->tls_session != NULL) { - flb_tls_session_destroy(u_conn->tls_session); - - u_conn->tls_session = NULL; - } -#endif - shutdown_connection(u_conn); - - flb_socket_close(u_conn->fd); - - u_conn->fd = -1; - u_conn->event.fd = -1; - } - - /* remove connection from the queue */ - mk_list_del(&u_conn->_head); - - /* Add node to destroy queue */ - mk_list_add(&u_conn->_head, &uq->destroy_queue); - - flb_upstream_decrement_total_connections_count(u); - - /* - * note: the connection context is destroyed by the engine once all events - * have been processed. - */ - return 0; -} - -/* 'safe' version of prepare_destroy_conn. It set locks if necessary */ -static inline int prepare_destroy_conn_safe(struct flb_connection *u_conn) -{ - int ret; - - flb_stream_acquire_lock(u_conn->stream, FLB_TRUE); - - ret = prepare_destroy_conn(u_conn); - - flb_stream_release_lock(u_conn->stream); - - return ret; -} - -static int destroy_conn(struct flb_connection *u_conn) -{ - /* Delay the destruction of busy connections */ - if (u_conn->busy_flag) { - return 0; - } - - mk_list_del(&u_conn->_head); - - flb_connection_destroy(u_conn); - - return 0; -} - -static struct flb_connection *create_conn(struct flb_upstream *u) -{ - struct flb_coro *coro; - struct flb_connection *conn; - int ret; - struct flb_upstream_queue *uq; - - coro = flb_coro_get(); - - conn = flb_connection_create(FLB_INVALID_SOCKET, - FLB_UPSTREAM_CONNECTION, - (void *) u, - flb_engine_evl_get(), - flb_coro_get()); - if (conn == NULL) { - return NULL; - } - - conn->busy_flag = FLB_TRUE; - - if (flb_stream_is_keepalive(&u->base)) { - flb_upstream_conn_recycle(conn, FLB_TRUE); - } - else { - flb_upstream_conn_recycle(conn, FLB_FALSE); - } - - flb_stream_acquire_lock(&u->base, FLB_TRUE); - - /* Link new connection to the busy queue */ - uq = flb_upstream_queue_get(u); - mk_list_add(&conn->_head, &uq->busy_queue); - - flb_upstream_increment_total_connections_count(u); - - flb_stream_release_lock(&u->base); - - flb_connection_reset_connection_timeout(conn); - - /* Start connection */ - ret = flb_io_net_connect(conn, coro); - if (ret == -1) { - flb_connection_unset_connection_timeout(conn); - - flb_debug("[upstream] connection #%i failed to %s:%i", - conn->fd, u->tcp_host, u->tcp_port); - - prepare_destroy_conn_safe(conn); - conn->busy_flag = FLB_FALSE; - - return NULL; - } - - flb_connection_unset_connection_timeout(conn); - - if (flb_stream_is_keepalive(&u->base)) { - flb_debug("[upstream] KA connection #%i to %s:%i is connected", - conn->fd, u->tcp_host, u->tcp_port); - } - - /* Invalidate timeout for connection */ - conn->busy_flag = FLB_FALSE; - - return conn; -} - -int flb_upstream_destroy(struct flb_upstream *u) -{ - struct mk_list *tmp; - struct mk_list *head; - struct flb_connection *u_conn; - struct flb_upstream_queue *uq; - - uq = flb_upstream_queue_get(u); - if (!uq) { - uq = &u->queue; - } - - mk_list_foreach_safe(head, tmp, &uq->av_queue) { - u_conn = mk_list_entry(head, struct flb_connection, _head); - prepare_destroy_conn(u_conn); - } - - mk_list_foreach_safe(head, tmp, &uq->busy_queue) { - u_conn = mk_list_entry(head, struct flb_connection, _head); - prepare_destroy_conn(u_conn); - } - - mk_list_foreach_safe(head, tmp, &uq->destroy_queue) { - u_conn = mk_list_entry(head, struct flb_connection, _head); - destroy_conn(u_conn); - } - - flb_free(u->tcp_host); - flb_free(u->proxied_host); - flb_free(u->proxy_username); - flb_free(u->proxy_password); - mk_list_del(&u->base._head); - flb_free(u); - - return 0; -} - -/* Enable or disable 'recycle' flag for the connection */ -int flb_upstream_conn_recycle(struct flb_connection *conn, int val) -{ - if (val == FLB_TRUE || val == FLB_FALSE) { - conn->recycle = val; - } - - return -1; -} - -struct flb_connection *flb_upstream_conn_get(struct flb_upstream *u) -{ - int err; - int total_connections = 0; - struct mk_list *tmp; - struct mk_list *head; - struct flb_connection *conn; - struct flb_upstream_queue *uq; - - uq = flb_upstream_queue_get(u); - - flb_trace("[upstream] get new connection for %s:%i, net setup:\n" - "net.connect_timeout = %i seconds\n" - "net.source_address = %s\n" - "net.keepalive = %s\n" - "net.keepalive_idle_timeout = %i seconds\n" - "net.max_worker_connections = %i", - u->tcp_host, u->tcp_port, - u->base.net.connect_timeout, - u->base.net.source_address ? u->base.net.source_address: "any", - u->base.net.keepalive ? "enabled": "disabled", - u->base.net.keepalive_idle_timeout, - u->base.net.max_worker_connections); - - - /* If the upstream is limited by max connections, check current state */ - if (u->base.net.max_worker_connections > 0) { - /* - * Connections are linked to one of these lists: - * - * - av_queue : connections ready to be used (available) - * - busy_queue: connections that are busy (someone is using them) - * - drop_queue: connections in the cleanup phase (to be drop) - * - * Fluent Bit don't create connections ahead of time, just on demand. When - * a connection is created is placed into the busy_queue, when is not longer - * needed one of these things happen: - * - * - if keepalive is enabled (default), the connection is moved to the 'av_queue'. - * - if keepalive is disabled, the connection is moved to 'drop_queue' then is - * closed and destroyed. - * - * Based on the logic described above, to limit the number of total connections - * in the worker, we only need to count the number of connections linked into - * the 'busy_queue' list because if there are connections available 'av_queue' it - * won't create a one. - */ - - /* Count the number of relevant connections */ - flb_stream_acquire_lock(&u->base, FLB_TRUE); - total_connections = mk_list_size(&uq->busy_queue); - flb_stream_release_lock(&u->base); - - if (total_connections >= u->base.net.max_worker_connections) { - flb_debug("[upstream] max worker connections=%i reached to: %s:%i, cannot connect", - u->base.net.max_worker_connections, u->tcp_host, u->tcp_port); - return NULL; - } - } - - conn = NULL; - - /* - * If we are in keepalive mode, iterate list of available connections, - * take a little of time to do some cleanup and assign a connection. If no - * entries exists, just create a new one. - */ - if (u->base.net.keepalive) { - mk_list_foreach_safe(head, tmp, &uq->av_queue) { - conn = mk_list_entry(head, struct flb_connection, _head); - - flb_stream_acquire_lock(&u->base, FLB_TRUE); - - /* This connection works, let's move it to the busy queue */ - mk_list_del(&conn->_head); - mk_list_add(&conn->_head, &uq->busy_queue); - - flb_stream_release_lock(&u->base); - - err = flb_socket_error(conn->fd); - - if (!FLB_EINPROGRESS(err) && err != 0) { - flb_debug("[upstream] KA connection #%i is in a failed state " - "to: %s:%i, cleaning up", - conn->fd, u->tcp_host, u->tcp_port); - prepare_destroy_conn_safe(conn); - conn = NULL; - continue; - } - - /* Reset errno */ - conn->net_error = -1; - - /* Connect timeout */ - conn->ts_assigned = time(NULL); - flb_debug("[upstream] KA connection #%i to %s:%i has been assigned (recycled)", - conn->fd, u->tcp_host, u->tcp_port); - /* - * Note: since we are in a keepalive connection, the socket is already being - * monitored for possible disconnections while idle. Upon re-use by the caller - * when it try to send some data, the I/O interface (flb_io.c) will put the - * proper event mask and reuse, there is no need to remove the socket from - * the event loop and re-add it again. - * - * So just return the connection context. - */ - - break; - } - } - - /* - * There are no keepalive connections available or keepalive is disabled - * so we need to create a new one. - */ - if (conn == NULL) { - conn = create_conn(u); - } - - if (conn != NULL) { - flb_connection_reset_io_timeout(conn); - flb_upstream_increment_busy_connections_count(u); - } - - return conn; -} - -/* - * An 'idle' and keepalive might be disconnected, if so, this callback will perform - * the proper connection cleanup. - */ -static int cb_upstream_conn_ka_dropped(void *data) -{ - struct flb_connection *conn; - - conn = (struct flb_connection *) data; - - flb_debug("[upstream] KA connection #%i to %s:%i has been disconnected " - "by the remote service", - conn->fd, conn->upstream->tcp_host, conn->upstream->tcp_port); - return prepare_destroy_conn_safe(conn); -} - -int flb_upstream_conn_release(struct flb_connection *conn) -{ - int ret; - struct flb_upstream *u = conn->upstream; - struct flb_upstream_queue *uq; - - flb_upstream_decrement_busy_connections_count(u); - - uq = flb_upstream_queue_get(u); - - /* If this is a valid KA connection just recycle */ - if (u->base.net.keepalive == FLB_TRUE && - conn->recycle == FLB_TRUE && - conn->fd > -1 && - conn->net_error == -1) { - /* - * This connection is still useful, move it to the 'available' queue and - * initialize variables. - */ - flb_stream_acquire_lock(&u->base, FLB_TRUE); - - mk_list_del(&conn->_head); - mk_list_add(&conn->_head, &uq->av_queue); - - flb_stream_release_lock(&u->base); - - conn->ts_available = time(NULL); - - /* - * The socket at this point is not longer monitored, so if we want to be - * notified if the 'available keepalive connection' gets disconnected by - * the remote endpoint we need to add it again. - */ - conn->event.handler = cb_upstream_conn_ka_dropped; - - ret = mk_event_add(conn->evl, - conn->fd, - FLB_ENGINE_EV_CUSTOM, - MK_EVENT_CLOSE, - &conn->event); - - conn->event.priority = FLB_ENGINE_PRIORITY_CONNECT; - if (ret == -1) { - /* We failed the registration, for safety just destroy the connection */ - flb_debug("[upstream] KA connection #%i to %s:%i could not be " - "registered, closing.", - conn->fd, u->tcp_host, u->tcp_port); - return prepare_destroy_conn_safe(conn); - } - - flb_debug("[upstream] KA connection #%i to %s:%i is now available", - conn->fd, u->tcp_host, u->tcp_port); - conn->ka_count++; - - /* if we have exceeded our max number of uses of this connection, destroy it */ - if (conn->net->keepalive_max_recycle > 0 && - conn->ka_count > conn->net->keepalive_max_recycle) { - flb_debug("[upstream] KA count %i exceeded configured limit " - "of %i: closing.", - conn->ka_count, - conn->net->keepalive_max_recycle); - - return prepare_destroy_conn_safe(conn); - } - - return 0; - } - - /* No keepalive connections must be destroyed */ - return prepare_destroy_conn_safe(conn); -} - -int flb_upstream_conn_timeouts(struct mk_list *list) -{ - time_t now; - int drop; - const char *reason; - struct mk_list *head; - struct mk_list *u_head; - struct mk_list *tmp; - struct flb_upstream *u; - struct flb_connection *u_conn; - struct flb_upstream_queue *uq; - int elapsed_time; - - now = time(NULL); - - /* Iterate all upstream contexts */ - mk_list_foreach(head, list) { - u = mk_list_entry(head, struct flb_upstream, base._head); - uq = flb_upstream_queue_get(u); - - flb_stream_acquire_lock(&u->base, FLB_TRUE); - - /* Iterate every busy connection */ - mk_list_foreach_safe(u_head, tmp, &uq->busy_queue) { - u_conn = mk_list_entry(u_head, struct flb_connection, _head); - - drop = FLB_FALSE; - - /* Connect timeouts */ - if (u_conn->net->connect_timeout > 0 && - u_conn->ts_connect_timeout > 0 && - u_conn->ts_connect_timeout <= now) { - drop = FLB_TRUE; - reason = "connection timeout"; - elapsed_time = u_conn->net->connect_timeout; - } - else if (u_conn->net->io_timeout > 0 && - u_conn->ts_io_timeout > 0 && - u_conn->ts_io_timeout <= now) { - drop = FLB_TRUE; - reason = "IO timeout"; - elapsed_time = u_conn->net->io_timeout; - } - - if (drop) { - if (!flb_upstream_is_shutting_down(u)) { - if (u->base.net.connect_timeout_log_error) { - flb_error("[upstream] connection #%i to %s timed " - "out after %i seconds (%s)", - u_conn->fd, - flb_connection_get_remote_address(u_conn), - elapsed_time, - reason); - } - else { - flb_debug("[upstream] connection #%i to %s timed " - "out after %i seconds (%s)", - u_conn->fd, - flb_connection_get_remote_address(u_conn), - elapsed_time, - reason); - } - } - - u_conn->net_error = ETIMEDOUT; - - /* We need to shut the connection down - * in order to cause some functions that are not - * aware of the connection error signaling - * mechanism to fail and abort. - * - * These functions do not check the net_error field - * in the connection instance upon being awakened - * so we need to ensure that any read/write - * operations on the socket generate an error. - * - * net_io_write_async - * net_io_read_async - * flb_tls_net_write_async - * flb_tls_net_read_async - * - * This operation could be selectively performed for - * connections that have already been established - * with no side effects because the connection - * establishment code honors `net_error` but - * taking in account that the previous version of - * the code did it unconditionally with no noticeable - * side effects leaving it that way is the safest - * choice at the moment. - */ - - if (MK_EVENT_IS_REGISTERED((&u_conn->event))) { - shutdown_connection(u_conn); - - mk_event_inject(u_conn->evl, - &u_conn->event, - u_conn->event.mask, - FLB_TRUE); - } - else { - /* I can't think of a valid reason for this code path - * to be taken but considering that it was previously - * possible for it to happen (maybe wesley can shed - * some light on it if he remembers) I'll leave this - * for the moment. - * In any case, it's proven not to interfere with the - * coroutine awakening issue this change addresses. - */ - - prepare_destroy_conn(u_conn); - } - - flb_upstream_decrement_busy_connections_count(u); - } - } - - /* Check every available Keepalive connection */ - mk_list_foreach_safe(u_head, tmp, &uq->av_queue) { - u_conn = mk_list_entry(u_head, struct flb_connection, _head); - - if ((now - u_conn->ts_available) >= u->base.net.keepalive_idle_timeout) { - prepare_destroy_conn(u_conn); - flb_debug("[upstream] drop keepalive connection #%i to %s:%i " - "(keepalive idle timeout)", - u_conn->fd, u->tcp_host, u->tcp_port); - } - } - - flb_stream_release_lock(&u->base); - } - - return 0; -} - -int flb_upstream_conn_pending_destroy(struct flb_upstream *u) -{ - struct mk_list *tmp; - struct mk_list *head; - struct flb_connection *u_conn; - struct flb_upstream_queue *uq; - - uq = flb_upstream_queue_get(u); - - flb_stream_acquire_lock(&u->base, FLB_TRUE); - - /* Real destroy of connections context */ - mk_list_foreach_safe(head, tmp, &uq->destroy_queue) { - u_conn = mk_list_entry(head, struct flb_connection, _head); - - destroy_conn(u_conn); - } - - flb_stream_release_lock(&u->base); - - return 0; -} - -int flb_upstream_conn_active_destroy(struct flb_upstream *u) -{ - struct mk_list *tmp; - struct mk_list *head; - struct flb_connection *u_conn; - struct flb_upstream_queue *uq; - - uq = flb_upstream_queue_get(u); - - /* Real destroy of connections context */ - mk_list_foreach_safe(head, tmp, &uq->av_queue) { - u_conn = mk_list_entry(head, struct flb_connection, _head); - - destroy_conn(u_conn); - } - - return 0; -} - -int flb_upstream_conn_active_destroy_list(struct mk_list *list) -{ - struct mk_list *head; - struct flb_upstream *u; - - /* Iterate all upstream contexts */ - mk_list_foreach(head, list) { - u = mk_list_entry(head, struct flb_upstream, base._head); - - flb_upstream_conn_active_destroy(u); - } - - return 0; -} - -int flb_upstream_conn_pending_destroy_list(struct mk_list *list) -{ - struct mk_list *head; - struct flb_upstream *u; - - /* Iterate all upstream contexts */ - mk_list_foreach(head, list) { - u = mk_list_entry(head, struct flb_upstream, base._head); - - flb_upstream_conn_pending_destroy(u); - } - - return 0; -} - -int flb_upstream_is_async(struct flb_upstream *u) -{ - return flb_stream_is_async(&u->base); -} - -void flb_upstream_set_total_connections_label( - struct flb_upstream *stream, - const char *label_value) -{ - stream->cmt_total_connections_label = label_value; -} - -void flb_upstream_set_total_connections_gauge( - struct flb_upstream *stream, - struct cmt_gauge *gauge_instance) -{ - stream->cmt_total_connections = gauge_instance; -} - -static void flb_upstream_increment_total_connections_count( - struct flb_upstream *stream) -{ - if (stream->parent_upstream != NULL) { - stream = (struct flb_upstream *) stream->parent_upstream; - - flb_upstream_increment_total_connections_count(stream); - } - if (stream->cmt_total_connections != NULL) { - if (stream->cmt_total_connections_label != NULL) { - cmt_gauge_inc( - stream->cmt_total_connections, - cfl_time_now(), - 1, - (char *[]) { - (char *) stream->cmt_total_connections_label - }); - } - else { - cmt_gauge_inc(stream->cmt_total_connections, - cfl_time_now(), - 0, NULL); - } - } -} - -static void flb_upstream_decrement_total_connections_count( - struct flb_upstream *stream) -{ - if (stream->parent_upstream != NULL) { - stream = (struct flb_upstream *) stream->parent_upstream; - - flb_upstream_decrement_total_connections_count(stream); - } - else if (stream->cmt_total_connections != NULL) { - if (stream->cmt_total_connections_label != NULL) { - cmt_gauge_dec( - stream->cmt_total_connections, - cfl_time_now(), - 1, - (char *[]) { - (char *) stream->cmt_total_connections_label - }); - } - else { - cmt_gauge_dec(stream->cmt_total_connections, - cfl_time_now(), - 0, NULL); - } - } -} - -void flb_upstream_set_busy_connections_label( - struct flb_upstream *stream, - const char *label_value) -{ - stream->cmt_busy_connections_label = label_value; -} - -void flb_upstream_set_busy_connections_gauge( - struct flb_upstream *stream, - struct cmt_gauge *gauge_instance) -{ - stream->cmt_busy_connections = gauge_instance; -} - -static void flb_upstream_increment_busy_connections_count( - struct flb_upstream *stream) -{ - if (stream->parent_upstream != NULL) { - stream = (struct flb_upstream *) stream->parent_upstream; - - flb_upstream_increment_busy_connections_count(stream); - } - else if (stream->cmt_busy_connections != NULL) { - if (stream->cmt_busy_connections_label != NULL) { - cmt_gauge_inc( - stream->cmt_busy_connections, - cfl_time_now(), - 1, - (char *[]) { - (char *) stream->cmt_busy_connections_label - }); - } - else { - cmt_gauge_inc(stream->cmt_busy_connections, - cfl_time_now(), - 0, NULL); - } - } -} - -static void flb_upstream_decrement_busy_connections_count( - struct flb_upstream *stream) -{ - if (stream->parent_upstream != NULL) { - stream = (struct flb_upstream *) stream->parent_upstream; - - flb_upstream_decrement_busy_connections_count(stream); - } - else if (stream->cmt_busy_connections != NULL) { - if (stream->cmt_busy_connections_label != NULL) { - cmt_gauge_dec( - stream->cmt_busy_connections, - cfl_time_now(), - 1, - (char *[]) { - (char *) stream->cmt_busy_connections_label - }); - } - else { - cmt_gauge_dec(stream->cmt_busy_connections, - cfl_time_now(), - 0, NULL); - } - } -} diff --git a/fluent-bit/src/flb_upstream_ha.c b/fluent-bit/src/flb_upstream_ha.c deleted file mode 100644 index 6b2b68039..000000000 --- a/fluent-bit/src/flb_upstream_ha.c +++ /dev/null @@ -1,357 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <fluent-bit/flb_compat.h> -#include <fluent-bit/flb_info.h> -#include <fluent-bit/flb_log.h> -#include <fluent-bit/flb_mem.h> -#include <fluent-bit/flb_hash_table.h> -#include <fluent-bit/flb_utils.h> -#include <fluent-bit/flb_upstream_ha.h> -#include <fluent-bit/flb_upstream_node.h> -#include <fluent-bit/flb_config_format.h> -#include <fluent-bit/flb_kv.h> - -#include <ctype.h> -#include <sys/types.h> -#include <sys/stat.h> - -/* Creates an Upstream HA Context */ -struct flb_upstream_ha *flb_upstream_ha_create(const char *name) -{ - struct flb_upstream_ha *ctx; - - if (!name) { - return NULL; - } - - ctx = flb_calloc(1, sizeof(struct flb_upstream_ha)); - if (!ctx) { - flb_errno(); - return NULL; - } - - ctx->name = flb_sds_create(name); - if (!ctx->name) { - flb_free(ctx); - return NULL; - } - - mk_list_init(&ctx->nodes); - ctx->last_used_node = NULL; - - return ctx; -} - -void flb_upstream_ha_destroy(struct flb_upstream_ha *ctx) -{ - struct mk_list *tmp; - struct mk_list *head; - struct flb_upstream_node *node; - - /* destroy nodes */ - mk_list_foreach_safe(head, tmp, &ctx->nodes) { - node = mk_list_entry(head, struct flb_upstream_node, _head); - mk_list_del(&node->_head); - flb_upstream_node_destroy(node); - } - - flb_sds_destroy(ctx->name); - flb_free(ctx); -} - -/* Link a new node to the list handled by HA context */ -void flb_upstream_ha_node_add(struct flb_upstream_ha *ctx, - struct flb_upstream_node *node) -{ - mk_list_add(&node->_head, &ctx->nodes); -} - -/* Return a target node to be used for I/O */ -struct flb_upstream_node *flb_upstream_ha_node_get(struct flb_upstream_ha *ctx) -{ - struct flb_upstream_node *cur_node; - struct flb_upstream_node *node; - - if (mk_list_is_empty(&ctx->nodes) == 0) { - return NULL; - } - - if (!ctx->last_used_node) { - node = mk_list_entry_first(&ctx->nodes, struct flb_upstream_node, - _head); - ctx->last_used_node = node; - return node; - } - - cur_node = (struct flb_upstream_node *) ctx->last_used_node; - - node = mk_list_entry_next(&cur_node->_head, struct flb_upstream_node, - _head, &ctx->nodes); - ctx->last_used_node = node; - return node; -} - -static struct flb_upstream_node *create_node(int id, - struct flb_cf *cf, - struct flb_cf_section *s, - struct flb_config *config) -{ - int i; - int ret; - int skip; - int klen; - int vlen; - int tls = FLB_FALSE; - int tls_verify = FLB_TRUE; - int tls_debug = 1; - char key[32]; - char *tmp; - char *name = NULL; - char *host = NULL; - char *port = NULL; - char *tls_vhost = NULL; - char *tls_ca_path = NULL; - char *tls_ca_file = NULL; - char *tls_crt_file = NULL; - char *tls_key_file = NULL; - char *tls_key_passwd = NULL; - struct cfl_list *head; - struct cfl_kvpair *entry; - struct flb_hash_table *ht; - const char *known_keys[] = {"name", "host", "port", - "tls", "tls.vhost", "tls.verify", "tls.debug", - "tls.ca_path", "tls.ca_file", "tls.crt_file", - "tls.key_file", "tls.key_passwd", NULL}; - - struct flb_upstream_node *node; - - /* name */ - name = flb_cf_section_property_get_string(cf, s, "name"); - if (!name) { - flb_error("[upstream_ha] no 'name' has been set on node #%i", - id + 1); - return NULL; - } - - /* host */ - host = flb_cf_section_property_get_string(cf, s, "host"); - if (!host) { - flb_error("[upstream_ha] no 'host' has been set on node #%i", - id + 1); - return NULL; - } - - /* port */ - port = flb_cf_section_property_get_string(cf, s, "port"); - if (!port) { - flb_error("[upstream_ha] no 'port' has been set on node #%i", - id + 1); - return NULL; - } - - /* tls */ - tmp = flb_cf_section_property_get_string(cf, s, "tls"); - if (tmp) { - tls = flb_utils_bool(tmp); - flb_sds_destroy(tmp); - } - - /* tls.verify */ - tmp = flb_cf_section_property_get_string(cf, s, "tls.verify"); - if (tmp) { - tls_verify = flb_utils_bool(tmp); - flb_sds_destroy(tmp); - } - - /* tls.debug */ - tmp = flb_cf_section_property_get_string(cf, s, "tls.debug"); - if (tmp) { - tls_debug = atoi(tmp); - flb_sds_destroy(tmp); - } - - /* tls.vhost */ - tls_vhost = flb_cf_section_property_get_string(cf, s, "tls.vhost"); - - /* tls.ca_path */ - tls_ca_path = flb_cf_section_property_get_string(cf, s, "tls.ca_path"); - - /* tls.ca_file */ - tls_ca_file = flb_cf_section_property_get_string(cf, s, "tls.ca_file"); - - /* tls.crt_file */ - tls_crt_file = flb_cf_section_property_get_string(cf, s, "tls.crt_file"); - - /* tls.key_file */ - tls_key_file = flb_cf_section_property_get_string(cf, s, "tls.key_file"); - - /* tls.key_file */ - tls_key_passwd = flb_cf_section_property_get_string(cf, s, "tls.key_passwd"); - - /* - * Create hash table to store unknown key/values that might be used - * by the caller plugin. - */ - ht = flb_hash_table_create(FLB_HASH_TABLE_EVICT_NONE, 32, 256); - if (!ht) { - flb_error("[upstream_ha] error creating hash table"); - return NULL; - } - - /* - * Iterate mk_rconf section internals, find all unknown keys and add - * them to the hash table associated to the node. - */ - cfl_list_foreach(head, &s->properties->list) { - entry = cfl_list_entry(head, struct cfl_kvpair, _head); - - /* If this is a known entry, just skip it */ - skip = FLB_FALSE; - for (i = 0; known_keys[i] != NULL; i++) { - if (strcasecmp(entry->key, known_keys[i]) == 0) { - skip = FLB_TRUE; - break; - } - } - if (skip == FLB_TRUE) { - continue; - } - - klen = flb_sds_len(entry->key); - vlen = flb_sds_len(entry->val->data.as_string); - - /* Always store keys in lowercase */ - for (i = 0; i < klen; i++) { - key[i] = tolower(entry->key[i]); - } - key[klen] = '\0'; - - /* Add the key and value to the hash table */ - ret = flb_hash_table_add(ht, key, klen, entry->val->data.as_string, vlen); - if (ret == -1) { - flb_error("[upstream_ha] cannot add key %s to hash table", - entry->key); - } - } - - node = flb_upstream_node_create(name, host, port, tls, tls_verify, - tls_debug, tls_vhost, tls_ca_path, tls_ca_file, - tls_crt_file, tls_key_file, - tls_key_passwd, ht, config); - return node; -} - -/* Read an upstream file and generate the context */ -struct flb_upstream_ha *flb_upstream_ha_from_file(const char *file, - struct flb_config *config) -{ - int c = 0; - int ret; - const char *cfg = NULL; - char *tmp; - char path[PATH_MAX + 1]; - struct stat st; - struct mk_list *head; - struct flb_upstream_ha *ups; - struct flb_upstream_node *node; - struct flb_cf *cf = NULL; - struct flb_cf_section *section; - -#ifndef FLB_HAVE_STATIC_CONF - ret = stat(file, &st); - if (ret == -1 && errno == ENOENT) { - /* Try to resolve the real path (if exists) */ - if (file[0] == '/') { - return NULL; - } - - if (config->conf_path) { - snprintf(path, PATH_MAX, "%s%s", config->conf_path, file); - cfg = path; - } - } - else { - cfg = file; - } - flb_debug("[upstream_ha] opening file %s", cfg); - cf = flb_cf_create_from_file(NULL, (char *) cfg); -#else - //DISABLED/FIXME fconf = flb_config_static_open(file); -#endif - - if (!cf) { - return NULL; - } - - /* 'upstream' sections are under enum section_type FLB_CF_OTHER */ - section = flb_cf_section_get_by_name(cf, "upstream"); - if (!section) { - flb_error("[upstream_ha] section name 'upstream' could not be found"); - flb_cf_destroy(cf); - return NULL; - } - - /* upstream name */ - tmp = flb_cf_section_property_get_string(cf, section, "name"); - if (!tmp) { - flb_error("[upstream_ha] missing name for upstream at %s", cfg); - flb_cf_destroy(cf); - return NULL; - } - - ups = flb_upstream_ha_create(tmp); - flb_sds_destroy(tmp); - if (!ups) { - flb_error("[upstream_ha] cannot create context"); - flb_cf_destroy(cf); - return NULL; - } - - /* 'node' sections */ - mk_list_foreach(head, &cf->sections) { - section = mk_list_entry(head, struct flb_cf_section, _head); - if (strcasecmp(section->name, "node") != 0) { - continue; - } - - /* Read section info and create a Node context */ - node = create_node(c, cf, section, config); - if (!node) { - flb_error("[upstream_ha] cannot register node on upstream '%s'", - tmp); - flb_upstream_ha_destroy(ups); - flb_cf_destroy(cf); - return NULL; - } - - flb_upstream_ha_node_add(ups, node); - c++; - } - - if (c == 0) { - flb_error("[upstream_ha] no nodes defined"); - flb_upstream_ha_destroy(ups); - flb_cf_destroy(cf); - return NULL; - } - - flb_cf_destroy(cf); - return ups; -} diff --git a/fluent-bit/src/flb_upstream_node.c b/fluent-bit/src/flb_upstream_node.c deleted file mode 100644 index b88d1e06f..000000000 --- a/fluent-bit/src/flb_upstream_node.c +++ /dev/null @@ -1,213 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <fluent-bit/flb_info.h> -#include <fluent-bit/flb_io.h> -#include <fluent-bit/flb_log.h> -#include <fluent-bit/flb_mem.h> -#include <fluent-bit/flb_sds.h> -#include <fluent-bit/tls/flb_tls.h> -#include <fluent-bit/flb_hash_table.h> -#include <fluent-bit/flb_upstream_node.h> - -/* Create a new Upstream Node context */ -struct flb_upstream_node *flb_upstream_node_create(flb_sds_t name, flb_sds_t host, - flb_sds_t port, - int tls, int tls_verify, - int tls_debug, - const char *tls_vhost, - const char *tls_ca_path, - const char *tls_ca_file, - const char *tls_crt_file, - const char *tls_key_file, - const char *tls_key_passwd, - struct flb_hash_table *ht, - struct flb_config *config) -{ - int i_port; - int io_flags; - char tmp[255]; - struct flb_upstream_node *node; - - if (!host || !port) { - return NULL; - } - - /* port */ - i_port = atoi(port); - - /* Allocate node context */ - node = flb_calloc(1, sizeof(struct flb_upstream_node)); - if (!node) { - flb_errno(); - return NULL; - } - - /* Set node name */ - if (!name) { - /* compose a name using given host and port */ - snprintf(tmp, sizeof(tmp) - 1, "%s:%s", host, port); - node->name = flb_sds_create(tmp); - } - else { - node->name = name; - } - - /* host */ - node->host = host; - if (!node->host) { - flb_upstream_node_destroy(node); - return NULL; - } - - /* port */ - node->port = port; - if (!node->port) { - flb_upstream_node_destroy(node); - return NULL; - } - -#ifdef FLB_HAVE_TLS - - /* tls: ca path */ - node->tls_ca_path = flb_sds_create(tls_ca_path); - if (!node->tls_ca_path) { - flb_upstream_node_destroy(node); - return NULL; - } - - /* tls: ca file */ - node->tls_ca_file = flb_sds_create(tls_ca_file); - if (!node->tls_ca_file) { - flb_upstream_node_destroy(node); - return NULL; - } - - /* tls: crt file */ - node->tls_crt_file = flb_sds_create(tls_crt_file); - if (!node->tls_crt_file) { - flb_upstream_node_destroy(node); - return NULL; - } - - /* tls: key file */ - node->tls_key_file = flb_sds_create(tls_key_file); - if (!node->tls_key_file) { - flb_upstream_node_destroy(node); - return NULL; - } - - /* tls: key passwd */ - node->tls_key_passwd = flb_sds_create(tls_key_passwd); - if (!node->tls_key_passwd) { - flb_upstream_node_destroy(node); - return NULL; - } -#endif - - /* hash table */ - node->ht = ht; - -#ifdef FLB_HAVE_TLS - /* TLS setup */ - if (tls == FLB_TRUE) { - node->tls = flb_tls_create(FLB_TLS_CLIENT_MODE, - tls_verify, - tls_debug, - tls_vhost, - tls_ca_path, - tls_ca_file, - tls_crt_file, - tls_key_file, - tls_key_passwd); - if (!node->tls) { - flb_error("[upstream_node] error initializing TLS context " - "on node '%s'", name); - flb_upstream_node_destroy(node); - return NULL; - } - node->tls_enabled = FLB_TRUE; - } -#endif - - - /* Upstream flags */ - if (tls == FLB_TRUE) { - io_flags = FLB_IO_TLS; - } - else { - io_flags = FLB_IO_TCP; - } - - /* upstream context */ - node->u = flb_upstream_create(config, node->host, i_port, - io_flags, node->tls); - if (!node->u) { - flb_error("[upstream_node] error creating upstream context " - "for node '%s'", name); - flb_upstream_node_destroy(node); - return NULL; - } - - return node; -} - -const char *flb_upstream_node_get_property(const char *prop, - struct flb_upstream_node *node) -{ - int ret; - int len; - void *value; - size_t size; - - len = strlen(prop); - - ret = flb_hash_table_get(node->ht, prop, len, &value, &size); - if (ret == -1) { - return NULL; - } - - return (char *) value; -} - -void flb_upstream_node_destroy(struct flb_upstream_node *node) -{ - flb_sds_destroy(node->name); - flb_sds_destroy(node->host); - flb_sds_destroy(node->port); - - flb_hash_table_destroy(node->ht); - if (node->u) { - flb_upstream_destroy(node->u); - } - -#ifdef FLB_HAVE_TLS - flb_sds_destroy(node->tls_ca_path); - flb_sds_destroy(node->tls_ca_file); - flb_sds_destroy(node->tls_crt_file); - flb_sds_destroy(node->tls_key_file); - flb_sds_destroy(node->tls_key_passwd); - if (node->tls) { - flb_tls_destroy(node->tls); - } -#endif - - /* note: node link must be handled by the caller before this call */ - flb_free(node); -} diff --git a/fluent-bit/src/flb_uri.c b/fluent-bit/src/flb_uri.c deleted file mode 100644 index 48a8b0d88..000000000 --- a/fluent-bit/src/flb_uri.c +++ /dev/null @@ -1,186 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <stdlib.h> -#include <monkey/mk_core.h> - -#include <fluent-bit/flb_info.h> -#include <fluent-bit/flb_mem.h> -#include <fluent-bit/flb_str.h> -#include <fluent-bit/flb_uri.h> -#include <fluent-bit/flb_utils.h> - -/* - * Perform URI encoding for the given string. Always returns a new sds buffer, - * if it fails it returns NULL. - */ -flb_sds_t flb_uri_encode(const char *uri, size_t len) -{ - int i; - flb_sds_t buf = NULL; - flb_sds_t tmp = NULL; - - buf = flb_sds_create_size(len * 2); - if (!buf) { - flb_error("[uri] cannot allocate buffer for URI encoding"); - return NULL; - } - - for (i = 0; i < len; i++) { - if (flb_uri_to_encode(uri[i]) == FLB_TRUE) { - tmp = flb_sds_printf(&buf, "%%%02X", (unsigned char) *(uri + i)); - if (!tmp) { - flb_error("[uri] error formatting special character"); - flb_sds_destroy(buf); - return NULL; - } - continue; - } - - /* Direct assignment, just copy the character */ - if (buf) { - tmp = flb_sds_cat(buf, uri + i, 1); - if (!tmp) { - flb_error("[uri] error composing outgoing buffer"); - flb_sds_destroy(buf); - return NULL; - } - buf = tmp; - } - } - - return buf; -} - -/* Retrieve a given field based on it expected position in the URI */ -struct flb_uri_field *flb_uri_get(struct flb_uri *uri, int pos) -{ - if (pos < 0) { - flb_trace("[uri] negative pos"); - return NULL; - } - - if (pos >= FLB_URI_MAX || pos > uri->count) { - flb_trace("[uri] requested position > FLB_URI_MAX"); - return NULL; - } - - return &uri->map[pos]; -} - -/* - * Given a 'URI' string, split the strings separated by a slash and create a - * context. - */ -struct flb_uri *flb_uri_create(const char *full_uri) -{ - int end; - unsigned int len; - unsigned int val_len; - unsigned int i = 0; - char *val; - size_t uri_size; - void *p; - struct flb_uri_field *field; - struct flb_uri *uri; - - /* Set the required memory space */ - uri_size = sizeof(struct flb_uri); - uri_size += (sizeof(struct flb_uri_field) * FLB_URI_MAX); - - p = flb_calloc(1, uri_size); - if (!p) { - perror("malloc"); - return NULL; - } - - /* Link the 'map' */ - uri = p; - p = ((char *) p) + sizeof(struct flb_uri); - uri->map = p; - - /* Initialize fields list */ - mk_list_init(&uri->list); - uri->count = 0; - - len = strlen(full_uri); - while (i < len && uri->count < FLB_URI_MAX) { - end = mk_string_char_search(full_uri + i, '/', len - i); - - if (end >= 0 && end + i < len) { - end += i; - - if (i == (unsigned int) end) { - i++; - continue; - } - - val = mk_string_copy_substr(full_uri, i, end); - val_len = end - i; - } - else { - val = mk_string_copy_substr(full_uri, i, len); - val_len = len - i; - end = len; - - } - - /* Alloc node */ - field = &uri->map[uri->count]; - field->value = flb_strdup(val); - field->length = val_len; - mk_list_add(&field->_head, &uri->list); - i = end + 1; - uri->count++; - - mk_mem_free(val); - } - - uri->full = flb_strdup(full_uri); - return uri; -} - -/* Destroy an URI context and it resources associated */ -void flb_uri_destroy(struct flb_uri *uri) -{ - struct mk_list *tmp; - struct mk_list *head; - struct flb_uri_field *field; - - mk_list_foreach_safe(head, tmp, &uri->list) { - field = mk_list_entry(head, struct flb_uri_field, _head); - mk_list_del(&field->_head); - flb_free(field->value); - } - - flb_free(uri->full); - flb_free(uri); -} - -void flb_uri_dump(struct flb_uri *uri) -{ - int i; - struct flb_uri_field *f; - - for (i = 0; i < uri->count; i++) { - f = &uri->map[i]; - printf("[%i] length=%lu value='%s'\n", - i, f->length, f->value); - } -} diff --git a/fluent-bit/src/flb_utils.c b/fluent-bit/src/flb_utils.c deleted file mode 100644 index c2b2f58a6..000000000 --- a/fluent-bit/src/flb_utils.c +++ /dev/null @@ -1,1433 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <stdio.h> -#include <stdlib.h> -#include <string.h> -#include <time.h> -#include <ctype.h> - -#include <sys/types.h> -#include <sys/stat.h> -#include <msgpack.h> - -#include <monkey/mk_core.h> -#include <fluent-bit/flb_macros.h> -#include <fluent-bit/flb_config.h> -#include <fluent-bit/flb_error.h> -#include <fluent-bit/flb_input.h> -#include <fluent-bit/flb_output.h> -#include <fluent-bit/flb_utils.h> -#include <fluent-bit/flb_utf8.h> - -#ifdef FLB_HAVE_AWS_ERROR_REPORTER -#include <fluent-bit/aws/flb_aws_error_reporter.h> - -extern struct flb_aws_error_reporter *error_reporter; -#endif - -#ifdef FLB_HAVE_OPENSSL -#include <openssl/rand.h> -#endif - -/* - * The following block descriptor describes the private use unicode character range - * used for denoting invalid utf-8 fragments. Invalid fragment 0xCE would become - * utf-8 codepoint U+E0CE if FLB_UTILS_FRAGMENT_PRIVATE_BLOCK_DESCRIPTOR is set to - * E0 since U+E0CE = U+<FLB_UTILS_FRAGMENT_PRIVATE_BLOCK_DESCRIPTOR><HEX_FRAGMENT> - */ -#define FLB_UTILS_FRAGMENT_PRIVATE_BLOCK_DESCRIPTOR 0xE0 - -void flb_utils_error(int err) -{ - char *msg = NULL; - - switch (err) { - case FLB_ERR_CFG_FILE: - msg = "could not open configuration file"; - break; - case FLB_ERR_CFG_FILE_FORMAT: - msg = "configuration file contains format errors"; - break; - case FLB_ERR_CFG_FILE_STOP: - msg = "configuration file contains errors"; - break; - case FLB_ERR_CFG_FLUSH: - msg = "invalid flush value"; - break; - case FLB_ERR_CFG_FLUSH_CREATE: - msg = "could not create timer for flushing"; - break; - case FLB_ERR_CFG_FLUSH_REGISTER: - msg = "could not register timer for flushing"; - break; - case FLB_ERR_INPUT_INVALID: - msg = "invalid input type"; - break; - case FLB_ERR_INPUT_UNDEF: - msg = "no input(s) have been defined"; - break; - case FLB_ERR_INPUT_UNSUP: - msg = "unsupported Input"; - break; - case FLB_ERR_OUTPUT_UNDEF: - msg = "you must specify an output target"; - break; - case FLB_ERR_OUTPUT_INVALID: - msg = "invalid output target"; - break; - case FLB_ERR_OUTPUT_UNIQ: - msg = "just one output type is supported"; - break; - case FLB_ERR_FILTER_INVALID: - msg = "invalid filter plugin"; - break; - case FLB_ERR_CFG_PARSER_FILE: - msg = "could not open parser configuration file"; - break; - case FLB_ERR_JSON_INVAL: - msg = "invalid JSON string"; - break; - case FLB_ERR_JSON_PART: - msg = "truncated JSON string"; - break; - case FLB_ERR_CORO_STACK_SIZE: - msg = "invalid coroutine stack size"; - break; - case FLB_ERR_CFG_PLUGIN_FILE: - msg = "plugins_file not found"; - break; - case FLB_ERR_RELOADING_IN_PROGRESS: - msg = "reloading in progress"; - break; - default: - flb_error("(error message is not defined. err=%d)", err); - } - - if (!msg) { - fprintf(stderr, - "%sError%s: undefined. Aborting", - ANSI_BOLD ANSI_RED, ANSI_RESET); - #ifdef FLB_HAVE_AWS_ERROR_REPORTER - if (is_error_reporting_enabled()) { - flb_aws_error_reporter_write(error_reporter, "Error: undefined. Aborting\n"); - } - #endif - - } - else { - flb_error("%s, aborting.", msg); - #ifdef FLB_HAVE_AWS_ERROR_REPORTER - if (is_error_reporting_enabled()) { - flb_aws_error_reporter_write(error_reporter, msg); - } - #endif - } - - if (err <= FLB_ERR_FILTER_INVALID) { - exit(EXIT_FAILURE); - } -} - -/* Custom error */ -void flb_utils_error_c(const char *msg) -{ - fprintf(stderr, - "%sError%s: %s. Aborting\n\n", - ANSI_BOLD ANSI_RED, ANSI_RESET, msg); - exit(EXIT_FAILURE); -} - -void flb_utils_warn_c(const char *msg) -{ - fprintf(stderr, - "%sWarning%s: %s", - ANSI_BOLD ANSI_YELLOW, ANSI_RESET, msg); -} - -#ifdef FLB_HAVE_FORK -/* Run current process in background mode */ -int flb_utils_set_daemon(struct flb_config *config) -{ - pid_t pid; - - if ((pid = fork()) < 0){ - flb_error("Failed creating to switch to daemon mode (fork failed)"); - exit(EXIT_FAILURE); - } - - if (pid > 0) { /* parent */ - exit(EXIT_SUCCESS); - } - - /* set files mask */ - umask(0); - - /* Create new session */ - setsid(); - - if (chdir("/") < 0) { /* make sure we can unmount the inherited filesystem */ - flb_error("Unable to unmount the inherited filesystem"); - exit(EXIT_FAILURE); - } - - /* Our last STDOUT messages */ - flb_info("switching to background mode (PID=%ld)", (long) getpid()); - - fclose(stderr); - fclose(stdout); - - return 0; -} -#endif - -void flb_utils_print_setup(struct flb_config *config) -{ - struct mk_list *head; - struct mk_list *head_tmp; - struct flb_input_plugin *plugin; - struct flb_input_collector *collector; - struct flb_input_instance *in; - struct flb_filter_instance *f; - struct flb_output_instance *out; - - flb_info("Configuration:"); - - /* general */ - flb_info(" flush time | %f seconds", config->flush); - flb_info(" grace | %i seconds", config->grace); - flb_info(" daemon | %i", config->daemon); - - /* Inputs */ - flb_info("___________"); - flb_info(" inputs:"); - mk_list_foreach(head, &config->inputs) { - in = mk_list_entry(head, struct flb_input_instance, _head); - flb_info(" %s", in->p->name); - } - - /* Filters */ - flb_info("___________"); - flb_info(" filters:"); - mk_list_foreach(head, &config->filters) { - f = mk_list_entry(head, struct flb_filter_instance, _head); - flb_info(" %s", f->name); - } - - /* Outputs */ - flb_info("___________"); - flb_info(" outputs:"); - mk_list_foreach(head, &config->outputs) { - out = mk_list_entry(head, struct flb_output_instance, _head); - flb_info(" %s", out->name); - } - - /* Collectors */ - flb_info("___________"); - flb_info(" collectors:"); - mk_list_foreach(head, &config->inputs) { - in = mk_list_entry(head, struct flb_input_instance, _head); - mk_list_foreach(head_tmp, &in->collectors) { - collector = mk_list_entry(head_tmp, struct flb_input_collector, _head); - plugin = collector->instance->p; - - if (collector->seconds > 0) { - flb_info("[%s %lus,%luns] ", - plugin->name, - collector->seconds, - collector->nanoseconds); - } - else { - flb_info(" [%s] ", plugin->name); - } - } - } -} - -/* - * quoted_string_len returns the length of a quoted string, not including the quotes. - */ -static int quoted_string_len(const char *str) -{ - int len = 0; - char quote = *str++; /* Consume the quote character. */ - - while (quote != 0) { - char c = *str++; - switch (c) { - case '\0': - /* Error: string ends before end-quote was seen. */ - return -1; - case '\\': - /* Skip escaped quote or \\. */ - if (*str == quote || *str == '\\') { - str++; - } - break; - case '\'': - case '"': - /* End-quote seen: stop iterating. */ - if (c == quote) { - quote = 0; - } - break; - default: - break; - } - len++; - } - - /* Go back one character to ignore end-quote */ - len--; - - return len; -} - -/* - * next_token returns the next token in the string 'str' delimited by 'separator'. - * 'out' is set to the beginning of the token. - * 'out_len' is set to the length of the token. - * 'parse_quotes' is set to FLB_TRUE when quotes shall be considered when tokenizing the 'str'. - * The function returns offset to next token in the string. - */ -static int next_token(const char *str, int separator, char **out, int *out_len, int parse_quotes) { - const char *token_in = str; - char *token_out; - int next_separator = 0; - int quote = 0; /* Parser state: 0 not inside quoted string, or '"' or '\'' when inside quoted string. */ - int len = 0; - int i; - - /* Skip leading separators. */ - while (*token_in == separator) { - token_in++; - } - - /* Should quotes be parsed? Or is token quoted? If not, copy until separator or the end of string. */ - if (parse_quotes == FLB_FALSE || (*token_in != '"' && *token_in != '\'')) { - len = (int)strlen(token_in); - next_separator = mk_string_char_search(token_in, separator, len); - if (next_separator > 0) { - len = next_separator; - } - *out_len = len; - *out = mk_string_copy_substr(token_in, 0, len); - if (*out == NULL) { - return -1; - } - - return (int)(token_in - str) + len; - } - - /* Token is quoted. */ - - len = quoted_string_len(token_in); - if (len < 0) { - return -1; - } - - /* Consume the quote character. */ - quote = *token_in++; - - token_out = flb_malloc(len + 1); - if (!token_out) { - return -1; - } - - /* Copy the token */ - for (i = 0; i < len; i++) { - /* Handle escapes when inside quoted token: - * \" -> " - * \' -> ' - * \\ -> \ - */ - if (*token_in == '\\' && (token_in[1] == quote || token_in[1] == '\\')) { - token_in++; - } - token_out[i] = *token_in++; - } - token_out[i] = '\0'; - - *out = token_out; - *out_len = len; - - return (int)(token_in - str); -} - - -static struct mk_list *split(const char *line, int separator, int max_split, int quoted) -{ - int i = 0; - int count = 0; - int val_len; - int len; - int end; - char *val; - struct mk_list *list; - struct flb_split_entry *new; - - if (!line) { - return NULL; - } - - list = flb_malloc(sizeof(struct mk_list)); - if (!list) { - flb_errno(); - return NULL; - } - mk_list_init(list); - - len = strlen(line); - while (i < len) { - end = next_token(line + i, separator, &val, &val_len, quoted); - if (end == -1) { - flb_error("Parsing failed: %s", line); - flb_utils_split_free(list); - return NULL; - } - - /* Update last position */ - i += end; - - /* Create new entry */ - new = flb_malloc(sizeof(struct flb_split_entry)); - if (!new) { - flb_errno(); - flb_free(val); - flb_utils_split_free(list); - return NULL; - } - new->value = val; - new->len = val_len; - new->last_pos = i; - mk_list_add(&new->_head, list); - count++; - - /* Update index for next loop */ - i++; - - /* - * If the counter exceeded the maximum specified and there - * are still remaining bytes, append those bytes in a new - * and last entry. - */ - if (count >= max_split && max_split > 0 && i < len) { - new = flb_malloc(sizeof(struct flb_split_entry)); - if (!new) { - flb_errno(); - flb_utils_split_free(list); - return NULL; - } - new->value = mk_string_copy_substr(line, i, len); - new->len = len - i; - mk_list_add(&new->_head, list); - break; - } - } - - return list; -} - -struct mk_list *flb_utils_split_quoted(const char *line, int separator, int max_split) -{ - return split(line, separator, max_split, FLB_TRUE); -} - -struct mk_list *flb_utils_split(const char *line, int separator, int max_split) -{ - return split(line, separator, max_split, FLB_FALSE); -} - - -void flb_utils_split_free_entry(struct flb_split_entry *entry) -{ - mk_list_del(&entry->_head); - flb_free(entry->value); - flb_free(entry); -} - -void flb_utils_split_free(struct mk_list *list) -{ - struct mk_list *tmp; - struct mk_list *head; - struct flb_split_entry *entry; - - mk_list_foreach_safe(head, tmp, list) { - entry = mk_list_entry(head, struct flb_split_entry, _head); - flb_utils_split_free_entry(entry); - } - - flb_free(list); -} - -/* When a timer expires, it needs some handling */ -int flb_utils_timer_consume(flb_pipefd_t fd) -{ - int ret; - uint64_t val; - - ret = flb_pipe_r(fd, &val, sizeof(val)); - if (ret == -1) { - flb_errno(); - return -1; - } - -#ifdef __linux__ - /* A timer on linux must return an unisgned 64 bit number */ - if (ret == 0) { - return -1; - } -#endif - - return 0; -} - -int flb_utils_pipe_byte_consume(flb_pipefd_t fd) -{ - int ret; - uint64_t val; - - ret = flb_pipe_r(fd, &val, sizeof(val)); - if (ret == -1) { - flb_errno(); - return -1; - } - - return 0; -} - -int64_t flb_utils_size_to_bytes(const char *size) -{ - int i; - int len; - int plen = 0; - int64_t val; - char c; - char tmp[3] = {0}; - int64_t KB = 1000; - int64_t MB = 1000 * KB; - int64_t GB = 1000 * MB; - - if (!size) { - return -1; - } - - if (strcasecmp(size, "false") == 0) { - return 0; - } - - len = strlen(size); - val = atoll(size); - - if (len == 0) { - return -1; - } - - for (i = len - 1; i > 0; i--) { - if (isdigit(size[i])) { - break; - } - else { - plen++; - } - } - - if (plen == 0) { - return val; - } - else if (plen > 2) { - return -1; - } - - for (i = 0; i < plen; i++) { - c = size[(len - plen) + i]; - tmp[i] = toupper(c); - } - - if (plen == 2) { - if (tmp[1] != 'B') { - return -1; - } - } - - if (tmp[0] == 'K') { - /* set upper bound (2**64/KB)/2 to avoid overflows */ - if (val >= 9223372036854775 || val <= -9223372036854774) - { - return -1; - } - return (val * KB); - } - else if (tmp[0] == 'M') { - /* set upper bound (2**64/MB)/2 to avoid overflows */ - if (val >= 9223372036854 || val <= -9223372036853) { - return -1; - } - return (val * MB); - } - else if (tmp[0] == 'G') { - /* set upper bound (2**64/GB)/2 to avoid overflows */ - if (val >= 9223372036 || val <= -9223372035) { - return -1; - } - return (val * GB); - } - else { - return -1; - } - - return val; -} - -int64_t flb_utils_hex2int(char *hex, int len) -{ - int i = 0; - int64_t res = 0; - char c; - - while ((c = *hex++) && i < len) { - /* Ensure no overflow */ - if (res >= (int64_t)((INT64_MAX/0x10) - 0xff)) { - return -1; - } - - res *= 0x10; - - if (c >= 'a' && c <= 'f') { - res += (c - 0x57); - } - else if (c >= 'A' && c <= 'F') { - res += (c - 0x37); - } - else if (c >= '0' && c <= '9') { - res += (c - 0x30); - } - else { - return -1; - } - i++; - } - - if (res < 0) { - return -1; - } - - return res; -} - -int flb_utils_time_to_seconds(const char *time) -{ - int len; - size_t val; - - len = strlen(time); - if (len == 0) { - return 0; - } - val = atoi(time); - - /* String time to seconds */ - if (time[len - 1] == 'D' || time[len - 1] == 'd') { - val *= 86400; - } - if (time[len - 1] == 'H' || time[len - 1] == 'h') { - val *= 3600; - } - else if (time[len - 1] == 'M' || time[len - 1] == 'm') { - val *= 60; - } - - return val; -} - -int flb_utils_bool(const char *val) -{ - if (strcasecmp(val, "true") == 0 || - strcasecmp(val, "on") == 0 || - strcasecmp(val, "yes") == 0) { - return FLB_TRUE; - } - else if (strcasecmp(val, "false") == 0 || - strcasecmp(val, "off") == 0 || - strcasecmp(val, "no") == 0) { - return FLB_FALSE; - } - - return -1; -} - -/* Convert a 'string' time seconds.nanoseconds to int and long values */ -int flb_utils_time_split(const char *time, int *sec, long *nsec) -{ - char *p; - char *end; - long val = 0; - - errno = 0; - val = strtol(time, &end, 10); - if ((errno == ERANGE && (val == LONG_MAX || val == LONG_MIN)) - || (errno != 0 && val == 0)) { - flb_errno(); - return -1; - } - if (end == time) { - return -1; - } - *sec = (int) val; - - /* Try to find subseconds */ - *nsec = 0; - p = strchr(time, '.'); - if (p) { - p += 1; - val = strtol(p, &end, 10); - if ((errno == ERANGE && (val == LONG_MAX || val == LONG_MIN)) - || (errno != 0 && val == 0)) { - flb_errno(); - return -1; - } - if (end == p) { - return -1; - } - *nsec = val; - } - - return 0; -} - -void flb_utils_bytes_to_human_readable_size(size_t bytes, - char *out_buf, size_t size) -{ - unsigned long i; - unsigned long u = 1024; - static const char *__units[] = { - "b", "K", "M", "G", - "T", "P", "E", "Z", "Y", NULL - }; - - for (i = 0; __units[i] != NULL; i++) { - if ((bytes / u) == 0) { - break; - } - u *= 1024; - } - if (!i) { - snprintf(out_buf, size, "%lu%s", (long unsigned int) bytes, __units[0]); - } - else { - float fsize = (float) ((double) bytes / (u / 1024)); - snprintf(out_buf, size, "%.1f%s", fsize, __units[i]); - } -} - - -static inline void encoded_to_buf(char *out, const char *in, int len) -{ - int i; - char *p = out; - - for (i = 0; i < len; i++) { - *p++ = in[i]; - } -} - -/* - * Write string pointed by 'str' to the destination buffer 'buf'. It's make sure - * to escape sepecial characters and convert utf-8 byte characters to string - * representation. - */ -int flb_utils_write_str(char *buf, int *off, size_t size, - const char *str, size_t str_len) -{ - int i; - int b; - int ret; - int written = 0; - int required; - int len; - int hex_bytes; - int is_valid; - int utf_sequence_number; - int utf_sequence_length; - uint32_t codepoint; - uint32_t state = 0; - char tmp[16]; - size_t available; - uint32_t c; - char *p; - uint8_t *s; - - available = (size - *off); - required = str_len; - if (available <= required) { - return FLB_FALSE; - } - - p = buf + *off; - for (i = 0; i < str_len; i++) { - if ((available - written) < 2) { - return FLB_FALSE; - } - - c = (uint32_t) str[i]; - if (c == '\"') { - *p++ = '\\'; - *p++ = '\"'; - } - else if (c == '\\') { - *p++ = '\\'; - *p++ = '\\'; - } - else if (c == '\n') { - *p++ = '\\'; - *p++ = 'n'; - } - else if (c == '\r') { - *p++ = '\\'; - *p++ = 'r'; - } - else if (c == '\t') { - *p++ = '\\'; - *p++ = 't'; - } - else if (c == '\b') { - *p++ = '\\'; - *p++ = 'b'; - } - else if (c == '\f') { - *p++ = '\\'; - *p++ = 'f'; - } - else if (c < 32 || c == 0x7f) { - if ((available - written) < 6) { - return FLB_FALSE; - } - len = snprintf(tmp, sizeof(tmp) - 1, "\\u%.4hhx", (unsigned char) c); - if ((available - written) < len) { - return FLB_FALSE; - } - encoded_to_buf(p, tmp, len); - p += len; - } - else if (c >= 0x80 && c <= 0xFFFF) { - hex_bytes = flb_utf8_len(str + i); - if (available - written < 6) { - return FLB_FALSE; - } - - if (i + hex_bytes > str_len) { - break; /* skip truncated UTF-8 */ - } - - state = FLB_UTF8_ACCEPT; - codepoint = 0; - - for (b = 0; b < hex_bytes; b++) { - s = (unsigned char *) str + i + b; - ret = flb_utf8_decode(&state, &codepoint, *s); - if (ret == 0) { - break; - } - } - - if (state != FLB_UTF8_ACCEPT) { - /* Invalid UTF-8 hex, just skip utf-8 bytes */ - flb_warn("[pack] invalid UTF-8 bytes found, skipping bytes"); - } - else { - len = snprintf(tmp, sizeof(tmp) - 1, "\\u%.4x", codepoint); - if ((available - written) < len) { - return FLB_FALSE; - } - encoded_to_buf(p, tmp, len); - p += len; - } - i += (hex_bytes - 1); - } - else if (c > 0xFFFF) { - utf_sequence_length = flb_utf8_len(str + i); - - if (i + utf_sequence_length > str_len) { - break; /* skip truncated UTF-8 */ - } - - is_valid = FLB_TRUE; - for (utf_sequence_number = 0; utf_sequence_number < utf_sequence_length; - utf_sequence_number++) { - /* Leading characters must start with bits 11 */ - if (utf_sequence_number == 0 && ((str[i] & 0xC0) != 0xC0)) { - /* Invalid unicode character. replace */ - flb_debug("[pack] unexpected UTF-8 leading byte, " - "substituting character with replacement character"); - tmp[utf_sequence_number] = str[i]; - ++i; /* Consume invalid leading byte */ - utf_sequence_length = utf_sequence_number + 1; - is_valid = FLB_FALSE; - break; - } - /* Trailing characters must start with bits 10 */ - else if (utf_sequence_number > 0 && ((str[i] & 0xC0) != 0x80)) { - /* Invalid unicode character. replace */ - flb_debug("[pack] unexpected UTF-8 continuation byte, " - "substituting character with replacement character"); - /* This byte, i, is the start of the next unicode character */ - utf_sequence_length = utf_sequence_number; - is_valid = FLB_FALSE; - break; - } - - tmp[utf_sequence_number] = str[i]; - ++i; - } - --i; - - if (is_valid) { - if (available - written < utf_sequence_length) { - return FLB_FALSE; - } - - encoded_to_buf(p, tmp, utf_sequence_length); - p += utf_sequence_length; - } - else { - if (available - written < utf_sequence_length * 3) { - return FLB_FALSE; - } - - /* - * Utf-8 sequence is invalid. Map fragments to private use area - * codepoints in range: - * 0x<FLB_UTILS_FRAGMENT_PRIVATE_BLOCK_DESCRIPTOR>00 to - * 0x<FLB_UTILS_FRAGMENT_PRIVATE_BLOCK_DESCRIPTOR>FF - */ - for (b = 0; b < utf_sequence_length; ++b) { - /* - * Utf-8 private block invalid hex mapping. Format unicode charpoint - * in the following format: - * - * +--------+--------+--------+ - * |1110PPPP|10PPPPHH|10HHHHHH| - * +--------+--------+--------+ - * - * Where: - * P is FLB_UTILS_FRAGMENT_PRIVATE_BLOCK_DESCRIPTOR bits (1 byte) - * H is Utf-8 fragment hex bits (1 byte) - * 1 is bit 1 - * 0 is bit 0 - */ - - /* unicode codepoint start */ - *p = 0xE0; - - /* print unicode private block header first 4 bits */ - *p |= FLB_UTILS_FRAGMENT_PRIVATE_BLOCK_DESCRIPTOR >> 4; - ++p; - - /* unicode codepoint middle */ - *p = 0x80; - - /* print end of unicode private block header last 4 bits */ - *p |= ((FLB_UTILS_FRAGMENT_PRIVATE_BLOCK_DESCRIPTOR << 2) & 0x3f); - - /* print hex fragment first 2 bits */ - *p |= (tmp[b] >> 6) & 0x03; - ++p; - - /* unicode codepoint middle */ - *p = 0x80; - - /* print hex fragment last 6 bits */ - *p |= tmp[b] & 0x3f; - ++p; - } - } - } - else { - *p++ = c; - } - written = (p - (buf + *off)); - } - - *off += written; - - return FLB_TRUE; -} - - -int flb_utils_write_str_buf(const char *str, size_t str_len, char **out, size_t *out_size) -{ - int ret; - int off; - char *tmp; - char *buf; - size_t s; - - s = str_len + 1; - buf = flb_malloc(s); - if (!buf) { - flb_errno(); - return -1; - } - - while (1) { - off = 0; - ret = flb_utils_write_str(buf, &off, s, str, str_len); - if (ret == FLB_FALSE) { - s += 256; - tmp = flb_realloc(buf, s); - if (!tmp) { - flb_errno(); - flb_free(buf); - return -1; - } - buf = tmp; - } - else { - /* done */ - break; - } - } - - *out = buf; - *out_size = off; - return 0; -} - -static char *flb_copy_host(const char *string, int pos_init, int pos_end) -{ - if (string[pos_init] == '[') { /* IPv6 */ - if (string[pos_end-1] != ']') - return NULL; - - return mk_string_copy_substr(string, pos_init + 1, pos_end - 1); - } - else - return mk_string_copy_substr(string, pos_init, pos_end); -} - -int flb_utils_url_split(const char *in_url, char **out_protocol, - char **out_host, char **out_port, char **out_uri) -{ - char *protocol = NULL; - char *host = NULL; - char *port = NULL; - char *uri = NULL; - char *p; - char *tmp; - char *sep; - - /* Protocol */ - p = strstr(in_url, "://"); - if (!p) { - return -1; - } - if (p == in_url) { - return -1; - } - - protocol = mk_string_copy_substr(in_url, 0, p - in_url); - if (!protocol) { - flb_errno(); - return -1; - } - - /* Advance position after protocol */ - p += 3; - - /* Check for first '/' */ - sep = strchr(p, '/'); - tmp = strchr(p, ':'); - - /* Validate port separator is found before the first slash */ - if (sep && tmp) { - if (tmp > sep) { - tmp = NULL; - } - } - - if (tmp) { - host = flb_copy_host(p, 0, tmp - p); - if (!host) { - flb_errno(); - goto error; - } - p = tmp + 1; - - /* Look for an optional URI */ - tmp = strchr(p, '/'); - if (tmp) { - port = mk_string_copy_substr(p, 0, tmp - p); - uri = flb_strdup(tmp); - } - else { - port = flb_strdup(p); - uri = flb_strdup("/"); - } - } - else { - tmp = strchr(p, '/'); - if (tmp) { - host = flb_copy_host(p, 0, tmp - p); - uri = flb_strdup(tmp); - } - else { - host = flb_copy_host(p, 0, strlen(p)); - uri = flb_strdup("/"); - } - } - - if (!port) { - if (strcmp(protocol, "http") == 0) { - port = flb_strdup("80"); - } - else if (strcmp(protocol, "https") == 0) { - port = flb_strdup("443"); - } - } - - *out_protocol = protocol; - *out_host = host; - *out_port = port; - *out_uri = uri; - - return 0; - - error: - if (protocol) { - flb_free(protocol); - } - - return -1; -} - - -/* - * flb_utils_proxy_url_split parses a proxy's information from a http_proxy URL. - * The URL is in the form like `http://username:password@myproxy.com:8080`. - * Note: currently only HTTP is supported. - */ -int flb_utils_proxy_url_split(const char *in_url, char **out_protocol, - char **out_username, char **out_password, - char **out_host, char **out_port) -{ - char *protocol = NULL; - char *username = NULL; - char *password = NULL; - char *host = NULL; - char *port = NULL; - char *proto_sep; - char *at_sep; - char *tmp; - - /* Parse protocol */ - proto_sep = strstr(in_url, "://"); - if (!proto_sep) { - return -1; - } - if (proto_sep == in_url) { - return -1; - } - - protocol = mk_string_copy_substr(in_url, 0, proto_sep - in_url); - if (!protocol) { - flb_errno(); - return -1; - } - /* Only HTTP proxy is supported for now. */ - if (strcmp(protocol, "http") != 0) { - flb_free(protocol); - return -1; - } - - /* Advance position after protocol */ - proto_sep += 3; - - /* Seperate `username:password` and `host:port` */ - at_sep = strrchr(proto_sep, '@'); - if (at_sep) { - /* Parse username:passwrod part. */ - tmp = strchr(proto_sep, ':'); - if (!tmp) { - flb_free(protocol); - return -1; - } - username = mk_string_copy_substr(proto_sep, 0, tmp - proto_sep); - tmp += 1; - password = mk_string_copy_substr(tmp, 0, at_sep - tmp); - - /* Parse host:port part. */ - at_sep += 1; - tmp = strchr(at_sep, ':'); - if (tmp) { - host = flb_copy_host(at_sep, 0, tmp - at_sep); - tmp += 1; - port = strdup(tmp); - } - else { - host = flb_copy_host(at_sep, 0, strlen(at_sep)); - port = flb_strdup("80"); - } - } - else { - /* Parse host:port part. */ - tmp = strchr(proto_sep, ':'); - if (tmp) { - host = flb_copy_host(proto_sep, 0, tmp - proto_sep); - tmp += 1; - port = strdup(tmp); - } - else { - host = flb_copy_host(proto_sep, 0, strlen(proto_sep)); - port = flb_strdup("80"); - } - } - - *out_protocol = protocol; - *out_host = host; - *out_port = port; - if (username) { - *out_username = username; - } - if (password) { - *out_password = password; - } - - return 0; -} - - -char *flb_utils_get_os_name() -{ -#ifdef _WIN64 - return "win64"; -#elif _WIN32 - return "win32"; -#elif __APPLE__ || __MACH__ - return "macos"; -#elif __linux__ - return "linux"; -#elif __FreeBSD__ - return "freebsd"; -#elif __unix || __unix__ - return "unix"; -#else - return "other"; -#endif -} - -#ifdef FLB_HAVE_OPENSSL -int flb_utils_uuid_v4_gen(char *buf) -{ - int ret; - union { - struct { - uint32_t time_low; - uint16_t time_mid; - uint16_t time_hi_and_version; - uint8_t clk_seq_hi_res; - uint8_t clk_seq_low; - uint8_t node[6]; - }; - uint8_t __rnd[16]; - } uuid; - - ret = RAND_bytes(uuid.__rnd, sizeof(uuid)); - - uuid.clk_seq_hi_res = (uint8_t) ((uuid.clk_seq_hi_res & 0x3F) | 0x80); - uuid.time_hi_and_version = (uint16_t) ((uuid.time_hi_and_version & 0x0FFF) | 0x4000); - - snprintf(buf, 38, "%08x-%04x-%04x-%02x%02x-%02x%02x%02x%02x%02x%02x", - uuid.time_low, uuid.time_mid, uuid.time_hi_and_version, - uuid.clk_seq_hi_res, uuid.clk_seq_low, - uuid.node[0], uuid.node[1], uuid.node[2], - uuid.node[3], uuid.node[4], uuid.node[5]); - - if (ret == 1) { - return 0; - } - - return -1; -} -#else -int flb_utils_uuid_v4_gen(char *buf) -{ - snprintf(buf, 38, "ddad00f1-3806-46ab-88d1-277a8c863cd6"); - return 0; -} -#endif - -int flb_utils_read_file(char *path, char **out_buf, size_t *out_size) -{ - int fd; - int ret; - size_t bytes; - struct stat st; - flb_sds_t buf; - FILE *fp; - - fp = fopen(path, "rb"); - if (!fp) { - return -1; - } - fd = fileno(fp); - - ret = fstat(fd, &st); - if (ret == -1) { - flb_errno(); - fclose(fp); - return -1; - } - - buf = flb_calloc(1, st.st_size + 1); - if (!buf) { - flb_errno(); - fclose(fp); - return -1; - } - - bytes = fread(buf, st.st_size, 1, fp); - if (bytes < 1) { - if (ferror(fp)) { - flb_errno(); - } - flb_free(buf); - fclose(fp); - return -1; - } - fclose(fp); - - *out_buf = buf; - *out_size = st.st_size; - return 0; -} - -static int machine_id_read_and_sanitize(char *path, - char **out_buf, size_t *out_size) -{ - int ret; - size_t s; - char *p; - char *buf; - size_t bytes; - - ret = flb_utils_read_file(path, &buf, &bytes); - if (ret != 0) { - return -1; - } - - p = buf + bytes - 1; - while (*p == ' ' || *p == '\n') { - p--; - } - - /* set new size */ - s = p - buf + 1; - - buf[s] = '\0'; - *out_size = s; - *out_buf = buf; - - return 0; -} - -int flb_utils_get_machine_id(char **out_id, size_t *out_size) -{ - int ret; - char *id; - size_t bytes; - char *uuid; - -#ifdef __linux__ - char *dbus_var = "/var/lib/dbus/machine-id"; - char *dbus_etc = "/etc/machine-id"; - - /* dbus */ - if (access(dbus_var, F_OK) == 0) { /* check if the file exists first */ - ret = machine_id_read_and_sanitize(dbus_var, &id, &bytes); - if (ret == 0) { - *out_id = id; - *out_size = bytes; - return 0; - } - } - - /* etc */ - if (access(dbus_etc, F_OK) == 0) { /* check if the file exists first */ - ret = machine_id_read_and_sanitize(dbus_etc, &id, &bytes); - if (ret == 0) { - *out_id = id; - *out_size = bytes; - return 0; - } - } -#elif defined(__FreeBSD__) || defined(__NetBSD__) || \ - defined(__OpenBSD__) || defined(__DragonFly__) - - char *hostid = "/etc/hostid"; - - /* hostid */ - ret = machine_id_read_and_sanitize(hostid, &id, &bytes); - if (ret == 0) { - *out_id = id; - *out_size = bytes; - return 0; - } -#endif - - /* generate a random uuid */ - uuid = flb_malloc(38); - if (!uuid) { - flb_errno(); - return -1; - } - ret = flb_utils_uuid_v4_gen(uuid); - if (ret == 0) { - *out_id = uuid; - *out_size = strlen(uuid); - return 0; - } - - return -1; -} - -void flb_utils_set_plugin_string_property(const char *name, - flb_sds_t *field_storage, - flb_sds_t new_value) -{ - if (field_storage == NULL) { - flb_error("[utils] invalid field storage pointer for property '%s'", - name); - - return; - } - - if (*field_storage != NULL) { - flb_warn("[utils] property '%s' is already specified with value '%s'." - " Overwriting with '%s'", - name, - *field_storage, - new_value); - - flb_sds_destroy(*field_storage); - - *field_storage = NULL; - } - - *field_storage = new_value; -} diff --git a/fluent-bit/src/flb_worker.c b/fluent-bit/src/flb_worker.c deleted file mode 100644 index 47154f8f3..000000000 --- a/fluent-bit/src/flb_worker.c +++ /dev/null @@ -1,173 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <monkey/mk_core.h> -#include <fluent-bit/flb_info.h> -#include <fluent-bit/flb_mem.h> -#include <fluent-bit/flb_utils.h> -#include <fluent-bit/flb_config.h> -#include <fluent-bit/flb_worker.h> -#include <fluent-bit/flb_log.h> - -FLB_TLS_DEFINE(struct flb_worker, flb_worker_ctx); - -/* - * The step_callback runs in a POSIX thread context, it have been started - * by flb_worker_create(...). Here we setup specific FLB requirements and - * then we jump into the target/original callback. - */ -static void step_callback(void *data) -{ - struct flb_worker *worker = data; - - /* Set the worker context global */ - FLB_TLS_SET(flb_worker_ctx, worker); - - /* not too scary :) */ - worker->func(worker->data); - - /* FIXME: add a good plan for pthread_exit and 'worker' release */ - pthread_exit(NULL); -} - -struct flb_worker *flb_worker_context_create(void (*func) (void *), void *arg, - struct flb_config *config) -{ - struct flb_worker *worker; - - worker = flb_calloc(1, sizeof(struct flb_worker)); - if (!worker) { - flb_errno(); - return NULL; - } - MK_EVENT_ZERO(&worker->event); - worker->func = func; - worker->data = arg; - worker->config = config; - worker->log_ctx = config->log; - - return worker; -} - -/* - * Creates a worker (POSIX thread). This function creates a worker - * context and also setup the 'step' callback to initialize generic - * Fluent Bit requirements before to invoke the real target callback - * set by the caller. - * - * E.g: We do this intermediary 'step' to initialize the required - * logging context and possible others. - */ -int flb_worker_create(void (*func) (void *), void *arg, pthread_t *tid, - struct flb_config *config) -{ - int ret; - struct flb_worker *worker; - - worker = flb_worker_context_create(func, arg, config); - if (!worker) { - return -1; - } - - /* Initialize log-specific */ - ret = flb_log_worker_init(worker); - if (ret == -1) { - flb_free(worker); - return -1; - } - - /* Spawn the step_callback and the func() */ - ret = mk_utils_worker_spawn(step_callback, worker, &worker->tid); - if (ret != 0) { - flb_free(worker); - return -1; - } - memcpy(tid, &worker->tid, sizeof(pthread_t)); - mk_list_add(&worker->_head, &config->workers); - - return 0; -} - -/* - * The worker interface aims to prepare any context required by Threads when - * running, this function is called just one time. - */ -int flb_worker_init(struct flb_config *config) -{ - FLB_TLS_INIT(flb_worker_ctx); - - return 0; -} - -/* Lookup a worker using it pthread id */ -struct flb_worker *flb_worker_lookup(pthread_t tid, struct flb_config *config) -{ - struct mk_list *head; - struct flb_worker *worker; - - mk_list_foreach(head, &config->workers) { - worker = mk_list_entry(head, struct flb_worker, _head); - if (pthread_equal(worker->tid, tid) != 0) { - return worker; - } - } - - return NULL; -} - -struct flb_worker *flb_worker_get() -{ - return FLB_TLS_GET(flb_worker_ctx); -} - -void flb_worker_destroy(struct flb_worker *worker) -{ - if (!worker) { - return; - } - - if (worker->log_cache) { - flb_log_cache_destroy(worker->log_cache); - } - - mk_list_del(&worker->_head); - flb_free(worker); -} - -int flb_worker_exit(struct flb_config *config) -{ - int c = 0; - struct mk_list *tmp; - struct mk_list *head; - struct flb_worker *worker; - - mk_list_foreach_safe(head, tmp, &config->workers) { - worker = mk_list_entry(head, struct flb_worker, _head); - flb_worker_destroy(worker); - c++; - } - - return c; -} - -int flb_worker_log_level(struct flb_worker *worker) -{ - struct flb_log *log = worker->log_ctx; - return log->level; -}; diff --git a/fluent-bit/src/fluent-bit.c b/fluent-bit/src/fluent-bit.c deleted file mode 100644 index 51b814cfd..000000000 --- a/fluent-bit/src/fluent-bit.c +++ /dev/null @@ -1,1417 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#define _GNU_SOURCE - -#include <stdio.h> -#include <stdint.h> -#include <stdlib.h> -#include <string.h> -#include <signal.h> -#include <ctype.h> - -#include <cfl/cfl.h> -#include <cfl/cfl_array.h> -#include <cfl/cfl_kvlist.h> - -#include <fluent-bit/flb_compat.h> -#include <fluent-bit/flb_info.h> -#include <fluent-bit/flb_dump.h> -#include <fluent-bit/flb_stacktrace.h> -#include <fluent-bit/flb_env.h> -#include <fluent-bit/flb_macros.h> -#include <fluent-bit/flb_utils.h> -#include <fluent-bit/flb_pack.h> -#include <fluent-bit/flb_meta.h> -#include <fluent-bit/flb_config.h> -#include <fluent-bit/flb_version.h> -#include <fluent-bit/flb_error.h> -#include <fluent-bit/flb_custom.h> -#include <fluent-bit/flb_input.h> -#include <fluent-bit/flb_output.h> -#include <fluent-bit/flb_filter.h> -#include <fluent-bit/flb_processor.h> -#include <fluent-bit/flb_engine.h> -#include <fluent-bit/flb_str.h> -#include <fluent-bit/flb_slist.h> -#include <fluent-bit/flb_plugin.h> -#include <fluent-bit/flb_parser.h> -#include <fluent-bit/flb_lib.h> -#include <fluent-bit/flb_help.h> -#include <fluent-bit/flb_record_accessor.h> -#include <fluent-bit/flb_ra_key.h> -#include <fluent-bit/flb_kv.h> -#include <fluent-bit/flb_reload.h> -#include <fluent-bit/flb_config_format.h> - -#ifdef FLB_HAVE_MTRACE -#include <mcheck.h> -#endif - -#ifdef FLB_SYSTEM_WINDOWS -extern int win32_main(int, char**); -extern void win32_started(void); -#endif - -flb_ctx_t *ctx; -struct flb_config *config; -volatile sig_atomic_t exit_signal = 0; -volatile sig_atomic_t flb_bin_restarting = FLB_RELOAD_IDLE; - -#ifdef FLB_HAVE_LIBBACKTRACE -struct flb_stacktrace flb_st; -#endif - -#ifdef FLB_HAVE_CHUNK_TRACE - -#include <fluent-bit/flb_chunk_trace.h> - -#define FLB_LONG_TRACE (1024 + 1) -#define FLB_LONG_TRACE_INPUT (1024 + 2) -#define FLB_LONG_TRACE_OUTPUT (1024 + 3) -#define FLB_LONG_TRACE_OUTPUT_PROPERTY (1024 + 4) - -#endif - -#define FLB_HELP_TEXT 0 -#define FLB_HELP_JSON 1 - - -#define PLUGIN_CUSTOM 0 -#define PLUGIN_INPUT 1 -#define PLUGIN_OUTPUT 2 -#define PLUGIN_FILTER 3 - -#define print_opt(a, b) printf(" %-24s%s\n", a, b) -#define print_opt_i(a, b, c) printf(" %-24s%s (default: %i)\n", a, b, c) -#define print_opt_s(a, b, c) printf(" %-24s%s (default: %s)\n", a, b, c) - -#define get_key(a, b, c) mk_rconf_section_get_key(a, b, c) -#define n_get_key(a, b, c) (intptr_t) get_key(a, b, c) -#define s_get_key(a, b, c) (char *) get_key(a, b, c) - -static char *prog_name; - -static void flb_signal_init(); - -static void flb_help(int rc, struct flb_config *config) -{ - struct mk_list *head; - struct flb_input_plugin *in; - struct flb_output_plugin *out; - struct flb_filter_plugin *filter; - struct flb_processor_plugin *processor; - - printf("Usage: %s [OPTION]\n\n", prog_name); - printf("%sAvailable Options%s\n", ANSI_BOLD, ANSI_RESET); - print_opt("-b --storage_path=PATH", "specify a storage buffering path"); - print_opt("-c --config=FILE", "specify an optional configuration file"); -#ifdef FLB_HAVE_FORK - print_opt("-d, --daemon", "run Fluent Bit in background mode"); -#endif - print_opt("-D, --dry-run", "dry run"); - print_opt_i("-f, --flush=SECONDS", "flush timeout in seconds", - FLB_CONFIG_FLUSH_SECS); - print_opt("-C, --custom=CUSTOM", "enable a custom plugin"); - print_opt("-i, --input=INPUT", "set an input"); - print_opt("-F --filter=FILTER", "set a filter"); - print_opt("-m, --match=MATCH", "set plugin match, same as '-p match=abc'"); - print_opt("-o, --output=OUTPUT", "set an output"); - print_opt("-p, --prop=\"A=B\"", "set plugin configuration property"); -#ifdef FLB_HAVE_PARSER - print_opt("-R, --parser=FILE", "specify a parser configuration file"); -#endif - print_opt("-e, --plugin=FILE", "load an external plugin (shared lib)"); - print_opt("-l, --log_file=FILE", "write log info to a file"); - print_opt("-t, --tag=TAG", "set plugin tag, same as '-p tag=abc'"); -#ifdef FLB_HAVE_STREAM_PROCESSOR - print_opt("-T, --sp-task=SQL", "define a stream processor task"); -#endif - print_opt("-v, --verbose", "increase logging verbosity (default: info)"); -#ifdef FLB_TRACE - print_opt("-vv", "trace mode (available)"); -#endif -#ifdef FLB_HAVE_CHUNK_TRACE - print_opt("-Z, --enable-chunk-trace", "enable chunk tracing, it can be activated either through the http api or the command line"); - print_opt("--trace-input", "input to start tracing on startup."); - print_opt("--trace-output", "output to use for tracing on startup."); - print_opt("--trace-output-property", "set a property for output tracing on startup."); - print_opt("--trace", "setup a trace pipeline on startup. Uses a single line, ie: \"input=dummy.0 output=stdout output.format='json'\""); -#endif - print_opt("-w, --workdir", "set the working directory"); -#ifdef FLB_HAVE_HTTP_SERVER - print_opt("-H, --http", "enable monitoring HTTP server"); - print_opt_s("-P, --port", "set HTTP server TCP port", - FLB_CONFIG_HTTP_PORT); -#endif - print_opt_i("-s, --coro_stack_size", "set coroutines stack size in bytes", - config->coro_stack_size); - print_opt("-q, --quiet", "quiet mode"); - print_opt("-S, --sosreport", "support report for Enterprise customers"); - print_opt("-Y, --enable-hot-reload", "enable for hot reloading"); - print_opt("-W, --disable-thread-safety-on-hot-reloading", "disable thread safety on hot reloading"); - print_opt("-V, --version", "show version number"); - print_opt("-h, --help", "print this help"); - - printf("\n%sInputs%s\n", ANSI_BOLD, ANSI_RESET); - - /* Iterate each supported input */ - mk_list_foreach(head, &config->in_plugins) { - in = mk_list_entry(head, struct flb_input_plugin, _head); - if (strcmp(in->name, "lib") == 0 || (in->flags & FLB_INPUT_PRIVATE)) { - /* useless..., just skip it. */ - continue; - } - print_opt(in->name, in->description); - } - - printf("\n%sProcessors%s\n", ANSI_BOLD, ANSI_RESET); - mk_list_foreach(head, &config->processor_plugins) { - processor = mk_list_entry(head, struct flb_processor_plugin, _head); - print_opt(processor->name, processor->description); - } - - printf("\n%sFilters%s\n", ANSI_BOLD, ANSI_RESET); - mk_list_foreach(head, &config->filter_plugins) { - filter = mk_list_entry(head, struct flb_filter_plugin, _head); - print_opt(filter->name, filter->description); - } - - printf("\n%sOutputs%s\n", ANSI_BOLD, ANSI_RESET); - mk_list_foreach(head, &config->out_plugins) { - out = mk_list_entry(head, struct flb_output_plugin, _head); - if (strcmp(out->name, "lib") == 0 || (out->flags & FLB_OUTPUT_PRIVATE)) { - /* useless..., just skip it. */ - continue; - } - print_opt(out->name, out->description); - } - - printf("\n%sInternal%s\n", ANSI_BOLD, ANSI_RESET); - printf(" Event Loop = %s\n", mk_event_backend()); - printf(" Build Flags =%s\n", FLB_INFO_FLAGS); - exit(rc); -} - -/* - * If the description is larger than the allowed 80 chars including left - * padding, split the content in multiple lines and align it properly. - */ -static void help_plugin_description(int left_padding, flb_sds_t str) -{ - int len; - int max; - int line = 0; - char *c; - char *p; - char *end; - char fmt[32]; - - if (!str) { - printf("no description available\n"); - return; - } - - max = 90 - left_padding; - len = strlen(str); - - if (len <= max) { - printf("%s\n", str); - return; - } - - p = str; - len = flb_sds_len(str); - end = str + len; - - while (p < end) { - if ((p + max) > end) { - c = end; - } - else { - c = p + max; - while (*c != ' ' && c > p) { - c--; - } - } - - if (c == p) { - len = end - p; - } - else { - len = c - p; - } - - snprintf(fmt, sizeof(fmt) - 1, "%%*s%%.%is\n", len); - if (line == 0) { - printf(fmt, 0, "", p); - } - else { - printf(fmt, left_padding, " ", p); - } - line++; - p += len + 1; - } -} - -static flb_sds_t help_get_value(msgpack_object map, char *key) -{ - flb_sds_t k; - flb_sds_t val; - msgpack_object *o; - struct flb_ra_value *rval = NULL; - struct flb_record_accessor *ra = NULL; - - k = flb_sds_create(key); - ra = flb_ra_create(k, FLB_FALSE); - flb_sds_destroy(k); - if (!ra) { - return NULL; - } - - rval = flb_ra_get_value_object(ra, map); - if (!rval) { - flb_ra_destroy(ra); - return NULL; - } - - o = &rval->o; - val = flb_sds_create_len(o->via.str.ptr, o->via.str.size); - - flb_ra_key_value_destroy(rval); - flb_ra_destroy(ra); - - return val; -} - -static void help_print_property(int max, msgpack_object k, msgpack_object v) -{ - int i; - int len = 0; - char buf[32]; - char fmt[32]; - char fmt_prf[32]; - char def[32]; - msgpack_object map; - flb_sds_t tmp; - flb_sds_t name; - flb_sds_t type; - flb_sds_t desc; - flb_sds_t defv; - - /* Convert property type to uppercase and print it */ - for (i = 0; i < k.via.str.size; i++) { - buf[i] = toupper(k.via.str.ptr[i]); - } - buf[k.via.str.size] = '\0'; - printf(ANSI_BOLD "\n%s\n" ANSI_RESET, buf); - - snprintf(fmt, sizeof(fmt) - 1, "%%-%is", max); - snprintf(fmt_prf, sizeof(fmt_prf) - 1, "%%-%is", max); - snprintf(def, sizeof(def) - 1, "%%*s> default: %%s, type: "); - - for (i = 0; i < v.via.array.size; i++) { - map = v.via.array.ptr[i]; - - name = help_get_value(map, "$name"); - type = help_get_value(map, "$type"); - desc = help_get_value(map, "$description"); - defv = help_get_value(map, "$default"); - - if (strcmp(type, "prefix") == 0) { - len = flb_sds_len(name); - tmp = flb_sds_create_size(len + 2); - flb_sds_printf(&tmp, "%sN", name); - printf(fmt_prf, tmp); - flb_sds_destroy(tmp); - } - else { - printf(fmt, name); - } - - help_plugin_description(max, desc); - - if (defv) { - printf(def, max, " ", defv); - } - else { - printf("%*s> type: ", max, " "); - } - printf("%s", type); - printf("\n\n"); - } -} - -static void help_format_json(void *help_buf, size_t help_size) -{ - flb_sds_t json; - - json = flb_msgpack_raw_to_json_sds(help_buf, help_size); - printf("%s\n", json); - flb_sds_destroy(json); -} - -static void help_format_text(void *help_buf, size_t help_size) -{ - int i; - int x; - int max = 0; - int len = 0; - int ret; - size_t off = 0; - flb_sds_t name; - flb_sds_t type; - flb_sds_t desc; - msgpack_unpacked result; - msgpack_object map; - msgpack_object p; - msgpack_object k; - msgpack_object v; - - msgpack_unpacked_init(&result); - ret = msgpack_unpack_next(&result, help_buf, help_size, &off); - if (ret != MSGPACK_UNPACK_SUCCESS) { - return; - } - map = result.data; - - type = help_get_value(map, "$type"); - name = help_get_value(map, "$name"); - desc = help_get_value(map, "$description"); - - printf("%sHELP%s\n%s %s plugin\n", ANSI_BOLD, ANSI_RESET, - name, type); - flb_sds_destroy(type); - flb_sds_destroy(name); - - if (desc) { - printf(ANSI_BOLD "\nDESCRIPTION\n" ANSI_RESET "%s\n", desc); - flb_sds_destroy(desc); - } - - /* Properties */ - p = map.via.map.ptr[3].val; - - /* Calculate padding */ - for (i = 0; i < p.via.map.size; i++) { - v = p.via.map.ptr[i].val; - for (x = 0; x < v.via.map.size; x++) { - msgpack_object ptr = v.via.array.ptr[x]; - name = help_get_value(ptr, "$name"); - len = flb_sds_len(name); - flb_sds_destroy(name); - if (len > max) { - max = len; - } - } - } - max += 2; - - /* Iterate each section of properties */ - for (i = 0; i < p.via.map.size; i++) { - k = p.via.map.ptr[i].key; - v = p.via.map.ptr[i].val; - help_print_property(max, k, v); - } -} - -static void flb_help_plugin(int rc, int format, - struct flb_config *config, int type, - struct flb_cf *cf, - struct flb_cf_section *s) -{ - struct flb_config_map *opt = NULL; - void *help_buf; - size_t help_size; - char *name; - struct flb_custom_instance *c = NULL; - struct flb_input_instance *i = NULL; - struct flb_filter_instance *f = NULL; - struct flb_output_instance *o = NULL; - - flb_version_banner(); - - name = flb_cf_section_property_get_string(cf, s, "name"); - if (!name) { - exit(EXIT_FAILURE); - } - - if (type == PLUGIN_CUSTOM) { - c = flb_custom_new(config, name, NULL); - if (!c) { - fprintf(stderr, "invalid custom plugin '%s'", name); - return; - } - opt = c->p->config_map; - flb_help_custom(c, &help_buf, &help_size); - flb_custom_instance_destroy(c); - } - else if (type == PLUGIN_INPUT) { - i = flb_input_new(config, name, 0, FLB_TRUE); - if (!i) { - fprintf(stderr, "invalid input plugin '%s'", name); - return; - } - opt = i->p->config_map; - flb_help_input(i, &help_buf, &help_size); - flb_input_instance_destroy(i); - } - else if (type == PLUGIN_FILTER) { - f = flb_filter_new(config, name, 0); - if (!f) { - fprintf(stderr, "invalid filter plugin '%s'", name); - return; - } - opt = f->p->config_map; - flb_help_filter(f, &help_buf, &help_size); - flb_filter_instance_destroy(f); - } - else if (type == PLUGIN_OUTPUT) { - o = flb_output_new(config, name, 0, FLB_TRUE); - if (!o) { - fprintf(stderr, "invalid output plugin '%s'", name); - return; - } - opt = o->p->config_map; - flb_help_output(o, &help_buf, &help_size); - flb_output_instance_destroy(o); - } - - if (!opt) { - exit(rc); - } - - if (format == FLB_HELP_TEXT) { - help_format_text(help_buf, help_size); - } - else if (format == FLB_HELP_JSON) { - help_format_json(help_buf, help_size); - } - - flb_free(help_buf); - exit(rc); -} - -#define flb_print_signal(X) case X: \ - write (STDERR_FILENO, #X ")\n", sizeof(#X ")\n")-1); \ - break; - -static void flb_signal_handler_break_loop(int signal) -{ - exit_signal = signal; -} - -static void flb_signal_exit(int signal) -{ - int len; - char ts[32]; - char s[] = "[engine] caught signal ("; - time_t now; - struct tm *cur; - - now = time(NULL); - cur = localtime(&now); - len = snprintf(ts, sizeof(ts) - 1, "[%i/%02i/%02i %02i:%02i:%02i] ", - cur->tm_year + 1900, - cur->tm_mon + 1, - cur->tm_mday, - cur->tm_hour, - cur->tm_min, - cur->tm_sec); - - /* write signal number */ - write(STDERR_FILENO, ts, len); - write(STDERR_FILENO, s, sizeof(s) - 1); - switch (signal) { - flb_print_signal(SIGINT); -#ifndef FLB_SYSTEM_WINDOWS - flb_print_signal(SIGQUIT); - flb_print_signal(SIGHUP); - flb_print_signal(SIGCONT); -#endif - flb_print_signal(SIGTERM); - flb_print_signal(SIGSEGV); - }; -} - -static void flb_signal_handler(int signal) -{ - int len; - char ts[32]; - char s[] = "[engine] caught signal ("; - time_t now; - struct tm *cur; - flb_ctx_t *ctx = flb_context_get(); - struct flb_cf *cf_opts = flb_cf_context_get(); - - now = time(NULL); - cur = localtime(&now); - len = snprintf(ts, sizeof(ts) - 1, "[%i/%02i/%02i %02i:%02i:%02i] ", - cur->tm_year + 1900, - cur->tm_mon + 1, - cur->tm_mday, - cur->tm_hour, - cur->tm_min, - cur->tm_sec); - - /* write signal number */ - write(STDERR_FILENO, ts, len); - write(STDERR_FILENO, s, sizeof(s) - 1); - switch (signal) { - flb_print_signal(SIGINT); -#ifndef FLB_SYSTEM_WINDOWS - flb_print_signal(SIGQUIT); - flb_print_signal(SIGHUP); - flb_print_signal(SIGCONT); -#endif - flb_print_signal(SIGTERM); - flb_print_signal(SIGSEGV); - flb_print_signal(SIGFPE); - }; - - flb_signal_init(); - - switch(signal) { - case SIGSEGV: - case SIGFPE: -#ifdef FLB_HAVE_LIBBACKTRACE - /* To preserve stacktrace */ - flb_stacktrace_print(&flb_st); -#endif - abort(); -#ifndef FLB_SYSTEM_WINDOWS - case SIGCONT: - flb_dump(ctx->config); - break; - case SIGHUP: -#ifndef FLB_HAVE_STATIC_CONF - if (flb_bin_restarting == FLB_RELOAD_IDLE) { - flb_bin_restarting = FLB_RELOAD_IN_PROGRESS; - /* reload by using same config files/path */ - flb_reload(ctx, cf_opts); - flb_bin_restarting = FLB_RELOAD_IDLE; - } - else { - flb_utils_error(FLB_ERR_RELOADING_IN_PROGRESS); - } - break; -#endif -#endif - } -} - -#ifdef FLB_SYSTEM_WINDOWS -#include <ConsoleApi.h> - -static flb_ctx_t *handler_ctx = NULL; -static struct flb_cf *handler_opts = NULL; -static int handler_signal = 0; - -void flb_console_handler_set_ctx(flb_ctx_t *ctx, struct flb_cf *cf_opts) -{ - handler_ctx = ctx; - handler_opts = cf_opts; -} - -static BOOL WINAPI flb_console_handler(DWORD evType) -{ - switch(evType) { - case 1 /* CTRL_BREAK_EVENT_1 */: - if (flb_bin_restarting == FLB_RELOAD_IDLE) { - flb_bin_restarting = FLB_RELOAD_IN_PROGRESS; - /* signal the main loop to execute reload. this is necessary since - * all signal handlers in win32 are executed on their own thread. - */ - handler_signal = 1; - flb_bin_restarting = FLB_RELOAD_IDLE; - } - else { - flb_utils_error(FLB_ERR_RELOADING_IN_PROGRESS); - } - break; - } - return 1; -} -#endif - -static void flb_signal_init() -{ - signal(SIGINT, &flb_signal_handler_break_loop); -#ifndef FLB_SYSTEM_WINDOWS - signal(SIGQUIT, &flb_signal_handler_break_loop); - signal(SIGHUP, &flb_signal_handler); - signal(SIGCONT, &flb_signal_handler); -#else - /* Use SetConsoleCtrlHandler on windows to simulate SIGHUP */ - SetConsoleCtrlHandler(flb_console_handler, 1); -#endif - signal(SIGTERM, &flb_signal_handler_break_loop); - signal(SIGSEGV, &flb_signal_handler); - signal(SIGFPE, &flb_signal_handler); -} - -static int set_property(struct flb_cf *cf, struct flb_cf_section *s, char *kv) -{ - int len; - int sep; - char *key; - char *value; - struct cfl_variant *tmp; - - len = strlen(kv); - sep = mk_string_char_search(kv, '=', len); - if (sep == -1) { - return -1; - } - - key = mk_string_copy_substr(kv, 0, sep); - value = kv + sep + 1; - - if (!key) { - return -1; - } - - tmp = flb_cf_section_property_add(cf, s->properties, key, 0, value, 0); - if (tmp == NULL) { - fprintf(stderr, "[error] setting up section '%s' plugin property '%s'\n", - s->name, key); - } - mk_mem_free(key); - return 0; -} - -static int flb_service_conf_path_set(struct flb_config *config, char *file) -{ - char *end; - char *path; - - path = realpath(file, NULL); - if (!path) { - return -1; - } - - /* lookup path ending and truncate */ - end = strrchr(path, FLB_DIRCHAR); - if (!end) { - free(path); - return -1; - } - - end++; - *end = '\0'; - config->conf_path = flb_strdup(path); - free(path); - - /* Store the relative file path */ - config->conf_path_file = flb_sds_create(file); - - return 0; -} - - -static struct flb_cf *service_configure(struct flb_cf *cf, - struct flb_config *config, char *file) -{ - int ret = -1; - -#ifdef FLB_HAVE_STATIC_CONF - cf = flb_config_static_open(file); -#else - if (file) { - cf = flb_cf_create_from_file(cf, file); - } -#endif - - if (!cf) { - return NULL; - } - - - /* Set configuration root path */ - if (file) { - flb_service_conf_path_set(config, file); - } - - ret = flb_config_load_config_format(config, cf); - if (ret != 0) { - return NULL; - } - - config->cf_main = cf; - return cf; -} - -#ifdef FLB_HAVE_CHUNK_TRACE -static struct flb_input_instance *find_input(flb_ctx_t *ctx, const char *name) -{ - struct mk_list *head; - struct flb_input_instance *in; - - - mk_list_foreach(head, &ctx->config->inputs) { - in = mk_list_entry(head, struct flb_input_instance, _head); - if (strcmp(name, in->name) == 0) { - return in; - } - if (in->alias) { - if (strcmp(name, in->alias) == 0) { - return in; - } - } - } - return NULL; -} - -static int enable_trace_input(flb_ctx_t *ctx, const char *name, const char *prefix, const char *output_name, struct mk_list *props) -{ - struct flb_input_instance *in; - - - in = find_input(ctx, name); - if (in == NULL) { - return FLB_ERROR; - } - - flb_chunk_trace_context_new(in, output_name, prefix, NULL, props); - return (in->chunk_trace_ctxt == NULL ? FLB_ERROR : FLB_OK); -} - -static int disable_trace_input(flb_ctx_t *ctx, const char *name) -{ - struct flb_input_instance *in; - - - in = find_input(ctx, name); - if (in == NULL) { - return FLB_ERROR; - } - - if (in->chunk_trace_ctxt != NULL) { - flb_chunk_trace_context_destroy(in); - } - return FLB_OK; -} - -static int set_trace_property(struct mk_list *props, char *kv) -{ - int len; - int sep; - char *key; - char *value; - - len = strlen(kv); - sep = mk_string_char_search(kv, '=', len); - if (sep == -1) { - return -1; - } - - key = mk_string_copy_substr(kv, 0, sep); - value = kv + sep + 1; - - if (!key) { - return -1; - } - - flb_kv_item_create_len(props, - (char *)key, strlen(key), - (char *)value, strlen(value)); - - mk_mem_free(key); - return 0; -} - -static int parse_trace_pipeline_prop(flb_ctx_t *ctx, const char *kv, char **key, char **value) -{ - int len; - int sep; - - len = strlen(kv); - sep = mk_string_char_search(kv, '=', len); - if (sep == -1) { - return FLB_ERROR; - } - - *key = mk_string_copy_substr(kv, 0, sep); - if (!key) { - return FLB_ERROR; - } - - *value = flb_strdup(kv + sep + 1); - return FLB_OK; -} - -static int parse_trace_pipeline(flb_ctx_t *ctx, const char *pipeline, char **trace_input, char **trace_output, struct mk_list **props) -{ - struct mk_list *parts = NULL; - struct mk_list *cur; - struct flb_split_entry *part; - char *key; - char *value; - const char *propname; - const char *propval; - - - parts = flb_utils_split(pipeline, (int)' ', 0); - if (parts == NULL) { - return FLB_ERROR; - } - - mk_list_foreach(cur, parts) { - key = NULL; - value = NULL; - - part = mk_list_entry(cur, struct flb_split_entry, _head); - - if (parse_trace_pipeline_prop(ctx, part->value, &key, &value) == FLB_ERROR) { - return FLB_ERROR; - } - - if (strcmp(key, "input") == 0) { - if (*trace_input != NULL) { - flb_free(*trace_input); - } - *trace_input = flb_strdup(value); - } - else if (strcmp(key, "output") == 0) { - if (*trace_output != NULL) { - flb_free(*trace_output); - } - *trace_output = flb_strdup(value); - } - else if (strncmp(key, "output.", strlen("output.")) == 0) { - propname = mk_string_copy_substr(key, strlen("output."), strlen(key)); - if (propname == NULL) { - return FLB_ERROR; - } - - propval = flb_strdup(value); - if (propval == NULL) { - return FLB_ERROR; - } - - if (*props == NULL) { - *props = flb_calloc(1, sizeof(struct mk_list)); - flb_kv_init(*props); - } - - flb_kv_item_create_len(*props, - (char *)propname, strlen(propname), - (char *)propval, strlen(propval)); - } - - if (key != NULL) { - mk_mem_free(key); - } - - if (value != NULL) { - flb_free(value); - } - } - - flb_utils_split_free(parts); - return FLB_OK; -} -#endif - -int flb_main(int argc, char **argv) -{ - int opt; - int ret; - flb_sds_t json; - - /* handle plugin properties: -1 = none, 0 = input, 1 = output */ - int last_plugin = -1; - - /* local variables to handle config options */ - char *cfg_file = NULL; - - /* config format context */ - struct flb_cf *cf; - struct flb_cf *tmp; - struct flb_cf_section *service; - struct flb_cf_section *s; - struct flb_cf_section *section; - struct flb_cf *cf_opts; - - prog_name = argv[0]; - - cf_opts = flb_cf_create(); - if (!cf_opts) { - exit(EXIT_FAILURE); - } - section = flb_cf_section_create(cf_opts, "service", 0); - if (!section) { - flb_cf_destroy(cf_opts); - exit(EXIT_FAILURE); - } - -#ifdef FLB_HAVE_LIBBACKTRACE - flb_stacktrace_init(argv[0], &flb_st); -#endif - -#ifdef FLB_HAVE_CHUNK_TRACE - char *trace_input = NULL; - char *trace_output = flb_strdup("stdout"); - struct mk_list *trace_props = NULL; -#endif - - /* Setup long-options */ - static const struct option long_opts[] = { - { "storage_path", required_argument, NULL, 'b' }, - { "config", required_argument, NULL, 'c' }, -#ifdef FLB_HAVE_FORK - { "daemon", no_argument , NULL, 'd' }, -#endif - { "dry-run", no_argument , NULL, 'D' }, - { "flush", required_argument, NULL, 'f' }, - { "http", no_argument , NULL, 'H' }, - { "log_file", required_argument, NULL, 'l' }, - { "port", required_argument, NULL, 'P' }, - { "custom", required_argument, NULL, 'C' }, - { "input", required_argument, NULL, 'i' }, - { "match", required_argument, NULL, 'm' }, - { "output", required_argument, NULL, 'o' }, - { "filter", required_argument, NULL, 'F' }, -#ifdef FLB_HAVE_PARSER - { "parser", required_argument, NULL, 'R' }, -#endif - { "prop", required_argument, NULL, 'p' }, - { "plugin", required_argument, NULL, 'e' }, - { "tag", required_argument, NULL, 't' }, -#ifdef FLB_HAVE_STREAM_PROCESSOR - { "sp-task", required_argument, NULL, 'T' }, -#endif - { "version", no_argument , NULL, 'V' }, - { "verbose", no_argument , NULL, 'v' }, - { "workdir", required_argument, NULL, 'w' }, - { "quiet", no_argument , NULL, 'q' }, - { "help", no_argument , NULL, 'h' }, - { "help-json", no_argument , NULL, 'J' }, - { "coro_stack_size", required_argument, NULL, 's' }, - { "sosreport", no_argument , NULL, 'S' }, -#ifdef FLB_HAVE_HTTP_SERVER - { "http_server", no_argument , NULL, 'H' }, - { "http_listen", required_argument, NULL, 'L' }, - { "http_port", required_argument, NULL, 'P' }, -#endif - { "enable-hot-reload", no_argument, NULL, 'Y' }, -#ifdef FLB_HAVE_CHUNK_TRACE - { "enable-chunk-trace", no_argument, NULL, 'Z' }, - { "trace", required_argument, NULL, FLB_LONG_TRACE }, - { "trace-input", required_argument, NULL, FLB_LONG_TRACE_INPUT }, - { "trace-output", required_argument, NULL, FLB_LONG_TRACE_OUTPUT }, - { "trace-output-property", required_argument, NULL, FLB_LONG_TRACE_OUTPUT_PROPERTY }, -#endif - { "disable-thread-safety-on-hot-reload", no_argument, NULL, 'W' }, - { NULL, 0, NULL, 0 } - }; - - /* Signal handler */ - flb_signal_init(); - - /* Initialize Monkey Core library */ - mk_core_init(); - - /* Create Fluent Bit context */ - ctx = flb_create(); - if (!ctx) { - exit(EXIT_FAILURE); - } - config = ctx->config; - cf = config->cf_main; - service = cf_opts->service; - -#ifdef FLB_SYSTEM_WINDOWS - flb_console_handler_set_ctx(ctx, cf_opts); -#endif - - /* Add reference for cf_opts */ - config->cf_opts = cf_opts; - -#ifndef FLB_HAVE_STATIC_CONF - - /* Parse the command line options */ - while ((opt = getopt_long(argc, argv, - "b:c:dDf:C:i:m:o:R:F:p:e:" - "t:T:l:vw:qVhJL:HP:s:SWYZ", - long_opts, NULL)) != -1) { - - switch (opt) { - case 'b': - flb_cf_section_property_add(cf_opts, service->properties, - "storage.path", 0, optarg, 0); - break; - case 'c': - cfg_file = flb_strdup(optarg); - break; -#ifdef FLB_HAVE_FORK - case 'd': - flb_cf_section_property_add(cf_opts, service->properties, - "daemon", 0, "on", 0); - config->daemon = FLB_TRUE; - break; -#endif - case 'D': - config->dry_run = FLB_TRUE; - break; - case 'e': - ret = flb_plugin_load_router(optarg, config); - if (ret == -1) { - exit(EXIT_FAILURE); - } - /* Store the relative file path for external plugin */ - flb_slist_add(&config->external_plugins, optarg); - break; - case 'f': - flb_cf_section_property_add(cf_opts, service->properties, - "flush", 0, optarg, 0); - break; - case 'C': - s = flb_cf_section_create(cf_opts, "custom", 0); - if (!s) { - flb_utils_error(FLB_ERR_CUSTOM_INVALID); - } - flb_cf_section_property_add(cf_opts, s->properties, "name", 0, optarg, 0); - last_plugin = PLUGIN_CUSTOM; - break; - case 'i': - s = flb_cf_section_create(cf_opts, "input", 0); - if (!s) { - flb_utils_error(FLB_ERR_INPUT_INVALID); - } - flb_cf_section_property_add(cf_opts, s->properties, "name", 0, optarg, 0); - last_plugin = PLUGIN_INPUT; - break; - case 'm': - if (last_plugin == PLUGIN_FILTER || last_plugin == PLUGIN_OUTPUT) { - flb_cf_section_property_add(cf_opts, s->properties, "match", 0, optarg, 0); - } - break; - case 'o': - s = flb_cf_section_create(cf_opts, "output", 0); - if (!s) { - flb_utils_error(FLB_ERR_OUTPUT_INVALID); - } - flb_cf_section_property_add(cf_opts, s->properties, "name", 0, optarg, 0); - last_plugin = PLUGIN_OUTPUT; - break; -#ifdef FLB_HAVE_PARSER - case 'R': - ret = flb_parser_conf_file_stat(optarg, config); - if (ret == -1) { - flb_cf_destroy(cf_opts); - flb_destroy(ctx); - exit(EXIT_FAILURE); - } - flb_cf_section_property_add(cf_opts, service->properties, FLB_CONF_STR_PARSERS_FILE, 0, optarg, 0); - break; -#endif - case 'F': - s = flb_cf_section_create(cf_opts, "filter", 0); - if (!s) { - flb_utils_error(FLB_ERR_FILTER_INVALID); - } - flb_cf_section_property_add(cf_opts, s->properties, "name", 0, optarg, 0); - last_plugin = PLUGIN_FILTER; - break; - case 'l': - flb_cf_section_property_add(cf_opts, service->properties, - "log_file", 0, optarg, 0); - break; - case 'p': - if (s) { - set_property(cf_opts, s, optarg); - } - break; - case 't': - if (s) { - flb_cf_section_property_add(cf_opts, s->properties, "tag", 0, optarg, 0); - } - break; -#ifdef FLB_HAVE_STREAM_PROCESSOR - case 'T': - flb_slist_add(&config->stream_processor_tasks, optarg); - break; -#endif - case 'h': - if (last_plugin == -1) { - flb_help(EXIT_SUCCESS, config); - } - else { - flb_help_plugin(EXIT_SUCCESS, FLB_HELP_TEXT, - config, - last_plugin, cf_opts, s); - } - break; - case 'J': - if (last_plugin == -1) { - json = flb_help_build_json_schema(config); - if (!json) { - exit(EXIT_FAILURE); - } - - printf("%s\n", json); - flb_sds_destroy(json); - exit(EXIT_SUCCESS); - } - else { - flb_help_plugin(EXIT_SUCCESS, FLB_HELP_JSON, config, - last_plugin, cf_opts, s); - } - break; -#ifdef FLB_HAVE_HTTP_SERVER - case 'H': - flb_cf_section_property_add(cf_opts, service->properties, "http_server", 0, "on", 0); - break; - case 'L': - flb_cf_section_property_add(cf_opts, service->properties, FLB_CONF_STR_HTTP_LISTEN, 0, optarg, 0); - break; - case 'P': - flb_cf_section_property_add(cf_opts, service->properties, FLB_CONF_STR_HTTP_PORT, 0, optarg, 0); - break; -#endif - case 'V': - flb_version(); - exit(EXIT_SUCCESS); - case 'v': - config->verbose++; - break; - case 'w': - config->workdir = flb_strdup(optarg); - break; - case 'q': - config->verbose = FLB_LOG_OFF; - break; - case 's': - flb_cf_section_property_add(cf_opts, service->properties, FLB_CONF_STR_CORO_STACK_SIZE, 0, optarg, 0); - break; - case 'S': - config->support_mode = FLB_TRUE; - break; - case 'Y': - flb_cf_section_property_add(cf_opts, service->properties, FLB_CONF_STR_HOT_RELOAD, 0, "on", 0); - break; - case 'W': - flb_cf_section_property_add(cf_opts, service->properties, - FLB_CONF_STR_HOT_RELOAD_ENSURE_THREAD_SAFETY, 0, "off", 0); - break; -#ifdef FLB_HAVE_CHUNK_TRACE - case 'Z': - flb_cf_section_property_add(cf_opts, service->properties, FLB_CONF_STR_ENABLE_CHUNK_TRACE, 0, "on", 0); - break; - case FLB_LONG_TRACE: - parse_trace_pipeline(ctx, optarg, &trace_input, &trace_output, &trace_props); - break; - case FLB_LONG_TRACE_INPUT: - if (trace_input != NULL) { - flb_free(trace_input); - } - trace_input = flb_strdup(optarg); - break; - case FLB_LONG_TRACE_OUTPUT: - if (trace_output != NULL) { - flb_free(trace_output); - } - trace_output = flb_strdup(optarg); - break; - case FLB_LONG_TRACE_OUTPUT_PROPERTY: - if (trace_props == NULL) { - trace_props = flb_calloc(1, sizeof(struct mk_list)); - flb_kv_init(trace_props); - } - set_trace_property(trace_props, optarg); - break; -#endif /* FLB_HAVE_CHUNK_TRACE */ - default: - flb_help(EXIT_FAILURE, config); - } - } -#endif /* !FLB_HAVE_STATIC_CONF */ - - set_log_level_from_env(config); - - if (config->verbose != FLB_LOG_OFF) { - flb_version_banner(); - } - - /* Program name */ - flb_config_set_program_name(config, argv[0]); - - /* Set the current directory */ - if (config->workdir) { - ret = chdir(config->workdir); - if (ret == -1) { - flb_cf_destroy(cf_opts); - flb_errno(); - return -1; - } - } - - /* Validate config file */ -#ifndef FLB_HAVE_STATIC_CONF - if (cfg_file) { - if (access(cfg_file, R_OK) != 0) { - flb_free(cfg_file); - flb_cf_destroy(cf_opts); - flb_utils_error(FLB_ERR_CFG_FILE); - } - } - - if (flb_reload_reconstruct_cf(cf_opts, cf) != 0) { - flb_free(cfg_file); - flb_cf_destroy(cf_opts); - fprintf(stderr, "reconstruct format context is failed\n"); - exit(EXIT_FAILURE); - } - - /* Load the service configuration file */ - tmp = service_configure(cf, config, cfg_file); - flb_free(cfg_file); - if (!tmp) { - flb_cf_destroy(cf_opts); - flb_utils_error(FLB_ERR_CFG_FILE_STOP); - } -#else - tmp = service_configure(cf, config, "fluent-bit.conf"); - if (!tmp) { - flb_cf_destroy(cf_opts); - flb_utils_error(FLB_ERR_CFG_FILE_STOP); - } - - /* destroy previous context and override */ - flb_cf_destroy(cf); - config->cf_main = tmp; - cf = tmp; -#endif - - /* Check co-routine stack size */ - if (config->coro_stack_size < getpagesize()) { - flb_cf_destroy(cf_opts); - flb_utils_error(FLB_ERR_CORO_STACK_SIZE); - } - - /* Validate flush time (seconds) */ - if (config->flush <= (double) 0.0) { - flb_cf_destroy(cf_opts); - flb_utils_error(FLB_ERR_CFG_FLUSH); - } - - /* debug or trace */ - if (config->verbose >= FLB_LOG_DEBUG) { - flb_utils_print_setup(config); - } - -#ifdef FLB_HAVE_FORK - /* Run in background/daemon mode */ - if (config->daemon == FLB_TRUE) { - flb_utils_set_daemon(config); - } -#endif - -#ifdef FLB_SYSTEM_WINDOWS - win32_started(); -#endif - - if (config->dry_run == FLB_TRUE) { - fprintf(stderr, "configuration test is successful\n"); - flb_cf_destroy(cf_opts); - flb_destroy(ctx); - exit(EXIT_SUCCESS); - } - - /* start Fluent Bit library */ - ret = flb_start(ctx); - if (ret != 0) { - flb_cf_destroy(cf_opts); - flb_destroy(ctx); - return ret; - } - - /* Store the current config format context from command line */ - flb_cf_context_set(cf_opts); - - /* - * Always re-set the original context that was started, note that during a flb_start() a 'reload' could happen so the context - * will be different. Use flb_context_get() to get the current context. - */ - ctx = flb_context_get(); - -#ifdef FLB_HAVE_CHUNK_TRACE - if (trace_input != NULL) { - enable_trace_input(ctx, trace_input, NULL /* prefix ... */, trace_output, trace_props); - } -#endif - - while (ctx->status == FLB_LIB_OK && exit_signal == 0) { - sleep(1); - -#ifdef FLB_SYSTEM_WINDOWS - if (handler_signal == 1) { - handler_signal = 0; - flb_reload(ctx, cf_opts); - } -#endif - - /* set the context again before checking the status again */ - ctx = flb_context_get(); - -#ifdef FLB_SYSTEM_WINDOWS - flb_console_handler_set_ctx(ctx, cf_opts); -#endif - } - - if (exit_signal) { - flb_signal_exit(exit_signal); - } - ret = config->exit_status_code; - - cf_opts = flb_cf_context_get(); - - if (cf_opts != NULL) { - flb_cf_destroy(cf_opts); - } - -#ifdef FLB_HAVE_CHUNK_TRACE - if (trace_input != NULL) { - disable_trace_input(ctx, trace_input); - flb_free(trace_input); - } - if (trace_output) { - flb_free(trace_output); - } - if (trace_props != NULL) { - flb_kv_release(trace_props); - flb_free(trace_props); - } -#endif - - flb_stop(ctx); - flb_destroy(ctx); - - return ret; -} - -int main(int argc, char **argv) -{ -#ifdef FLB_SYSTEM_WINDOWS - return win32_main(argc, argv); -#else - return flb_main(argc, argv); -#endif -} diff --git a/fluent-bit/src/http_server/CMakeLists.txt b/fluent-bit/src/http_server/CMakeLists.txt deleted file mode 100644 index acded936d..000000000 --- a/fluent-bit/src/http_server/CMakeLists.txt +++ /dev/null @@ -1,20 +0,0 @@ -if(NOT FLB_METRICS) - message(FATAL_ERROR "FLB_HTTP_SERVER requires FLB_METRICS=On.") -endif() - -# Core Source -set(src - flb_hs.c - flb_hs_endpoints.c - flb_hs_utils.c - ) - -# api/v1 -add_subdirectory(api/v1) - -# api/v2 -add_subdirectory(api/v2) - -include_directories(${MONKEY_INCLUDE_DIR}) -add_library(flb-http-server STATIC ${src}) -target_link_libraries(flb-http-server monkey-core-static api-v1 api-v2) diff --git a/fluent-bit/src/http_server/api/v1/CMakeLists.txt b/fluent-bit/src/http_server/api/v1/CMakeLists.txt deleted file mode 100644 index af86e43f8..000000000 --- a/fluent-bit/src/http_server/api/v1/CMakeLists.txt +++ /dev/null @@ -1,20 +0,0 @@ -# api/v1 -set(src - uptime.c - metrics.c - storage.c - plugins.c - register.c - health.c - ) - -if(FLB_CHUNK_TRACE) - set(src - ${src} - trace.c - ) -endif() - -include_directories(${MONKEY_INCLUDE_DIR}) -add_library(api-v1 STATIC ${src}) -target_link_libraries(api-v1 monkey-core-static fluent-bit-static) diff --git a/fluent-bit/src/http_server/api/v1/health.c b/fluent-bit/src/http_server/api/v1/health.c deleted file mode 100644 index 713d4b877..000000000 --- a/fluent-bit/src/http_server/api/v1/health.c +++ /dev/null @@ -1,335 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include<stdio.h> -#include <stdlib.h> - -#include <fluent-bit/flb_info.h> -#include <fluent-bit/flb_pack.h> -#include <fluent-bit/flb_sds.h> -#include <fluent-bit/flb_macros.h> -#include <fluent-bit/flb_http_server.h> -#include <msgpack.h> - -#include "health.h" - -struct flb_health_check_metrics_counter *metrics_counter; - -pthread_key_t hs_health_key; - -static struct mk_list *hs_health_key_create() -{ - struct mk_list *metrics_list = NULL; - - metrics_list = flb_malloc(sizeof(struct mk_list)); - if (!metrics_list) { - flb_errno(); - return NULL; - } - mk_list_init(metrics_list); - pthread_setspecific(hs_health_key, metrics_list); - - return metrics_list; -} - -static void hs_health_key_destroy(void *data) -{ - struct mk_list *metrics_list = (struct mk_list*)data; - struct mk_list *tmp; - struct mk_list *head; - struct flb_hs_hc_buf *entry; - - if (metrics_list == NULL) { - return; - } - mk_list_foreach_safe(head, tmp, metrics_list) { - entry = mk_list_entry(head, struct flb_hs_hc_buf, _head); - if (entry != NULL) { - mk_list_del(&entry->_head); - flb_free(entry); - } - } - - flb_free(metrics_list); -} - -/* initialize the metrics counters */ -static void counter_init(struct flb_hs *hs) { - - metrics_counter = flb_malloc(sizeof(struct flb_health_check_metrics_counter)); - - if (!metrics_counter) { - flb_errno(); - return; - } - - metrics_counter->error_counter = 0; - metrics_counter->retry_failure_counter = 0; - metrics_counter->error_limit = hs->config->hc_errors_count; - metrics_counter->retry_failure_limit = hs->config->hc_retry_failure_count; - metrics_counter->period_counter = 0; - metrics_counter->period_limit = hs->config->health_check_period; - -} - -/* -* tell what's the current status for health check -* One default background is that the metrics received and saved into -* message queue every time is a accumulation of error numbers, -* not a error number in recent second. So to get the error number -* in a period, we need to use: -* the error number of the newest metrics message minus -* the error number in oldest metrics of period -*/ -static int is_healthy() { - - struct mk_list *metrics_list; - struct flb_hs_hc_buf *buf; - int period_errors; - int period_retry_failure; - - metrics_list = pthread_getspecific(hs_health_key); - if (metrics_list == NULL) { - metrics_list = hs_health_key_create(); - if (metrics_list == NULL) { - return FLB_FALSE; - } - } - - if (mk_list_is_empty(metrics_list) == 0) { - return FLB_TRUE; - } - - /* Get the error metrics entry from the start time of current period */ - buf = mk_list_entry_first(metrics_list, struct flb_hs_hc_buf, _head); - - /* - * increase user so clean up function won't - * free the memory and delete the data - */ - buf->users++; - - /* the error count saved in message queue is the number of - * error count at that time. So the math is that: - * the error count in current period = (current error count in total) - - * (begin error count in the period) - */ - period_errors = metrics_counter->error_counter - buf->error_count; - period_retry_failure = metrics_counter->retry_failure_counter - - buf->retry_failure_count; - buf->users--; - - if (period_errors > metrics_counter->error_limit || - period_retry_failure > metrics_counter->retry_failure_limit) { - - return FLB_FALSE; - } - - return FLB_TRUE; -} - -/* read the metrics from message queue and update the counter*/ -static void read_metrics(void *data, size_t size, int* error_count, - int* retry_failure_count) -{ - int i; - int j; - int m; - msgpack_unpacked result; - msgpack_object map; - size_t off = 0; - int errors = 0; - int retry_failure = 0; - - msgpack_unpacked_init(&result); - msgpack_unpack_next(&result, data, size, &off); - map = result.data; - - for (i = 0; i < map.via.map.size; i++) { - msgpack_object k; - msgpack_object v; - - /* Keys: input, output */ - k = map.via.map.ptr[i].key; - v = map.via.map.ptr[i].val; - if (k.via.str.size != sizeof("output") - 1 || - strncmp(k.via.str.ptr, "output", k.via.str.size) != 0) { - - continue; - } - /* Iterate sub-map */ - for (j = 0; j < v.via.map.size; j++) { - msgpack_object sv; - - /* Keys: plugin name , values: metrics */ - sv = v.via.map.ptr[j].val; - - for (m = 0; m < sv.via.map.size; m++) { - msgpack_object mk; - msgpack_object mv; - - mk = sv.via.map.ptr[m].key; - mv = sv.via.map.ptr[m].val; - - if (mk.via.str.size == sizeof("errors") - 1 && - strncmp(mk.via.str.ptr, "errors", mk.via.str.size) == 0) { - errors += mv.via.u64; - } - else if (mk.via.str.size == sizeof("retries_failed") - 1 && - strncmp(mk.via.str.ptr, "retries_failed", - mk.via.str.size) == 0) { - retry_failure += mv.via.u64; - } - } - } - } - - *error_count = errors; - *retry_failure_count = retry_failure; - msgpack_unpacked_destroy(&result); -} - -/* -* Delete unused metrics, note that we only care about the latest node -* we use this function to maintain the metrics queue only save the metrics -* in a period. The old metrics which is out of period will be removed -*/ -static int cleanup_metrics() -{ - int c = 0; - struct mk_list *tmp; - struct mk_list *head; - struct mk_list *metrics_list; - struct flb_hs_hc_buf *entry; - - metrics_list = pthread_getspecific(hs_health_key); - if (!metrics_list) { - return -1; - } - - if (metrics_counter->period_counter < metrics_counter->period_limit) { - return 0; - } - - /* remove the oldest metrics if it's out of period */ - mk_list_foreach_safe(head, tmp, metrics_list) { - entry = mk_list_entry(head, struct flb_hs_hc_buf, _head); - if (metrics_counter->period_counter > metrics_counter->period_limit && - entry->users == 0) { - metrics_counter->period_counter--; - mk_list_del(&entry->_head); - flb_free(entry); - c++; - } - else { - break; - } - } - - return c; -} - -/* - * Callback invoked every time some metrics are received through a - * message queue channel. This function runs in a Monkey HTTP thread - * worker and it purpose is to take the metrics data and record the health - * status based on the metrics. - * This happens every second based on the event config. - * So we treat period_counter to count the time. - * And we maintain a message queue with the size of period limit number - * so every time we get a new metrics data in, if the message queue size is - * large than period limit, we will do the clean up func to - * remove the oldest metrics. - */ -static void cb_mq_health(mk_mq_t *queue, void *data, size_t size) -{ - struct flb_hs_hc_buf *buf; - struct mk_list *metrics_list = NULL; - int error_count = 0; - int retry_failure_count = 0; - - metrics_list = pthread_getspecific(hs_health_key); - - if (metrics_list == NULL) { - metrics_list = hs_health_key_create(); - if (metrics_list == NULL) { - return; - } - } - - metrics_counter->period_counter++; - - /* this is to remove the metrics out of period*/ - cleanup_metrics(); - - buf = flb_malloc(sizeof(struct flb_hs_hc_buf)); - if (!buf) { - flb_errno(); - return; - } - - buf->users = 0; - - read_metrics(data, size, &error_count, &retry_failure_count); - - metrics_counter->error_counter = error_count; - metrics_counter->retry_failure_counter = retry_failure_count; - - buf->error_count = error_count; - buf->retry_failure_count = retry_failure_count; - - mk_list_add(&buf->_head, metrics_list); -} - -/* API: Get fluent Bit Health Status */ -static void cb_health(mk_request_t *request, void *data) -{ - int status = is_healthy(); - - if (status == FLB_TRUE) { - mk_http_status(request, 200); - mk_http_send(request, "ok\n", strlen("ok\n"), NULL); - mk_http_done(request); - } - else { - mk_http_status(request, 500); - mk_http_send(request, "error\n", strlen("error\n"), NULL); - mk_http_done(request); - } -} - -/* Perform registration */ -int api_v1_health(struct flb_hs *hs) -{ - - pthread_key_create(&hs_health_key, hs_health_key_destroy); - - counter_init(hs); - /* Create a message queue */ - hs->qid_health = mk_mq_create(hs->ctx, "/health", - cb_mq_health, NULL); - - mk_vhost_handler(hs->ctx, hs->vid, "/api/v1/health", cb_health, hs); - return 0; -} - -void flb_hs_health_destroy() -{ - flb_free(metrics_counter); -} diff --git a/fluent-bit/src/http_server/api/v1/health.h b/fluent-bit/src/http_server/api/v1/health.h deleted file mode 100644 index 27a826f43..000000000 --- a/fluent-bit/src/http_server/api/v1/health.h +++ /dev/null @@ -1,73 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - - -#ifndef FLB_HS_API_V1_HEALTH_H -#define FLB_HS_API_V1_HEALTH_H - -#include <fluent-bit/flb_info.h> -#include <fluent-bit/flb_http_server.h> - -struct flb_health_check_metrics_counter { - - /* - * health check error limit, - * setup by customer through config: HC_Errors_Count - */ - int error_limit; - - /* counter the error number in metrics*/ - int error_counter; - - /* - * health check retry failed limit, - * setup by customer through config: HC_Retry_Failure_Count - */ - int retry_failure_limit; - - /* count the retry failed number in metrics*/ - int retry_failure_counter; - - /*period limit, setup by customer through config: HC_Period*/ - int period_limit; - - /* count the seconds in one period*/ - int period_counter; - -}; - - -/* - * error and retry failure buffers that contains certain cached data to be used - * by health check. - */ -struct flb_hs_hc_buf { - int users; - int error_count; - int retry_failure_count; - struct mk_list _head; -}; - -/* health endpoint*/ -int api_v1_health(struct flb_hs *hs); - -/* clean up health resource when shutdown*/ -void flb_hs_health_destroy(); -#endif diff --git a/fluent-bit/src/http_server/api/v1/metrics.c b/fluent-bit/src/http_server/api/v1/metrics.c deleted file mode 100644 index 4a541eaa0..000000000 --- a/fluent-bit/src/http_server/api/v1/metrics.c +++ /dev/null @@ -1,579 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <fluent-bit/flb_info.h> -#include <fluent-bit/flb_pack.h> -#include <fluent-bit/flb_input.h> -#include <fluent-bit/flb_filter.h> -#include <fluent-bit/flb_output.h> -#include <fluent-bit/flb_sds.h> -#include <fluent-bit/flb_version.h> -#include <fluent-bit/flb_time.h> -#include "metrics.h" - -#include <fluent-bit/flb_http_server.h> -#include <msgpack.h> - -#define null_check(x) do { if (!x) { goto error; } else {sds = x;} } while (0) - -pthread_key_t hs_metrics_key; - -static struct mk_list *hs_metrics_key_create() -{ - struct mk_list *metrics_list = NULL; - - metrics_list = flb_malloc(sizeof(struct mk_list)); - if (metrics_list == NULL) { - flb_errno(); - return NULL; - } - mk_list_init(metrics_list); - pthread_setspecific(hs_metrics_key, metrics_list); - - return metrics_list; -} - -static void hs_metrics_key_destroy(void *data) -{ - struct mk_list *metrics_list = (struct mk_list*)data; - struct mk_list *tmp; - struct mk_list *head; - struct flb_hs_buf *entry; - - if (metrics_list == NULL) { - return; - } - mk_list_foreach_safe(head, tmp, metrics_list) { - entry = mk_list_entry(head, struct flb_hs_buf, _head); - if (entry != NULL) { - if (entry->raw_data != NULL) { - flb_free(entry->raw_data); - entry->raw_data = NULL; - } - if (entry->data) { - flb_sds_destroy(entry->data); - entry->data = NULL; - } - mk_list_del(&entry->_head); - flb_free(entry); - } - } - - flb_free(metrics_list); -} - -/* Return the newest metrics buffer */ -static struct flb_hs_buf *metrics_get_latest() -{ - struct flb_hs_buf *buf; - struct mk_list *metrics_list; - - metrics_list = pthread_getspecific(hs_metrics_key); - if (!metrics_list) { - return NULL; - } - - if (mk_list_size(metrics_list) == 0) { - return NULL; - } - - buf = mk_list_entry_last(metrics_list, struct flb_hs_buf, _head); - return buf; -} - -/* Delete unused metrics, note that we only care about the latest node */ -static int cleanup_metrics() -{ - int c = 0; - struct mk_list *tmp; - struct mk_list *head; - struct mk_list *metrics_list; - struct flb_hs_buf *last; - struct flb_hs_buf *entry; - - metrics_list = pthread_getspecific(hs_metrics_key); - if (!metrics_list) { - return -1; - } - - last = metrics_get_latest(); - if (!last) { - return -1; - } - - mk_list_foreach_safe(head, tmp, metrics_list) { - entry = mk_list_entry(head, struct flb_hs_buf, _head); - if (entry != last && entry->users == 0) { - mk_list_del(&entry->_head); - flb_sds_destroy(entry->data); - flb_free(entry->raw_data); - flb_free(entry); - c++; - } - } - - return c; -} - -/* - * Callback invoked every time some metrics are received through a - * message queue channel. This function runs in a Monkey HTTP thread - * worker and it purpose is to take the metrics data and store it - * somewhere so then it can be available by the end-points upon - * HTTP client requests. - */ -static void cb_mq_metrics(mk_mq_t *queue, void *data, size_t size) -{ - flb_sds_t out_data; - struct flb_hs_buf *buf; - struct mk_list *metrics_list = NULL; - - metrics_list = pthread_getspecific(hs_metrics_key); - if (!metrics_list) { - metrics_list = hs_metrics_key_create(); - if (metrics_list == NULL) { - return; - } - } - - /* Convert msgpack to JSON */ - out_data = flb_msgpack_raw_to_json_sds(data, size); - if (!out_data) { - return; - } - - buf = flb_malloc(sizeof(struct flb_hs_buf)); - if (!buf) { - flb_errno(); - flb_sds_destroy(out_data); - return; - } - buf->users = 0; - buf->data = out_data; - - buf->raw_data = flb_malloc(size); - if (!buf->raw_data) { - flb_errno(); - flb_sds_destroy(out_data); - flb_free(buf); - return; - } - memcpy(buf->raw_data, data, size); - buf->raw_size = size; - - mk_list_add(&buf->_head, metrics_list); - - cleanup_metrics(); -} - -int string_cmp(const void* a_arg, const void* b_arg) { - char *a = *(char **)a_arg; - char *b = *(char **)b_arg; - - return strcmp(a, b); -} - -size_t extract_metric_name_end_position(char *s) { - int i; - - for (i = 0; i < flb_sds_len(s); i++) { - if (s[i] == '{') { - return i; - } - } - return 0; -} - -int is_same_metric(char *s1, char *s2) { - int i; - int p1 = extract_metric_name_end_position(s1); - int p2 = extract_metric_name_end_position(s2); - - if (p1 != p2) { - return 0; - } - - for (i = 0; i < p1; i++) { - if (s1[i] != s2[i]) { - return 0; - } - } - return 1; -} - -/* derive HELP text from metricname */ -/* if help text length > 128, increase init memory for metric_helptxt */ -flb_sds_t metrics_help_txt(char *metric_name, flb_sds_t *metric_helptxt) -{ - if (strstr(metric_name, "input_bytes")) { - return flb_sds_cat(*metric_helptxt, " Number of input bytes.\n", 24); - } - else if (strstr(metric_name, "input_records")) { - return flb_sds_cat(*metric_helptxt, " Number of input records.\n", 26); - } - else if (strstr(metric_name, "output_bytes")) { - return flb_sds_cat(*metric_helptxt, " Number of output bytes.\n", 25); - } - else if (strstr(metric_name, "output_records")) { - return flb_sds_cat(*metric_helptxt, " Number of output records.\n", 27); - } - else if (strstr(metric_name, "output_errors")) { - return flb_sds_cat(*metric_helptxt, " Number of output errors.\n", 26); - } - else if (strstr(metric_name, "output_retries_failed")) { - return flb_sds_cat(*metric_helptxt, " Number of abandoned batches because the maximum number of re-tries was reached.\n", 81); - } - else if (strstr(metric_name, "output_retries")) { - return flb_sds_cat(*metric_helptxt, " Number of output retries.\n", 27); - } - else if (strstr(metric_name, "output_proc_records")) { - return flb_sds_cat(*metric_helptxt, " Number of processed output records.\n", 37); - } - else if (strstr(metric_name, "output_proc_bytes")) { - return flb_sds_cat(*metric_helptxt, " Number of processed output bytes.\n", 35); - } - else if (strstr(metric_name, "output_dropped_records")) { - return flb_sds_cat(*metric_helptxt, " Number of dropped records.\n", 28); - } - else if (strstr(metric_name, "output_retried_records")) { - return flb_sds_cat(*metric_helptxt, " Number of retried records.\n", 28); - } - else { - return (flb_sds_cat(*metric_helptxt, " Fluentbit metrics.\n", 20)); - } -} - -/* API: expose metrics in Prometheus format /api/v1/metrics/prometheus */ -void cb_metrics_prometheus(mk_request_t *request, void *data) -{ - int i; - int j; - int m; - int len; - int time_len; - int start_time_len; - uint64_t uptime; - size_t index; - size_t num_metrics = 0; - long now; - flb_sds_t sds; - flb_sds_t sds_metric; - flb_sds_t tmp_sds; - struct flb_sds *metric_helptxt_head; - flb_sds_t metric_helptxt; - size_t off = 0; - struct flb_hs_buf *buf; - msgpack_unpacked result; - msgpack_object map; - char tmp[32]; - char time_str[64]; - char start_time_str[64]; - char* *metrics_arr; - struct flb_time tp; - struct flb_hs *hs = data; - struct flb_config *config = hs->config; - - buf = metrics_get_latest(); - if (!buf) { - mk_http_status(request, 404); - mk_http_done(request); - return; - } - - /* ref count */ - buf->users++; - - /* Compose outgoing buffer string */ - sds = flb_sds_create_size(1024); - if (!sds) { - mk_http_status(request, 500); - mk_http_done(request); - buf->users--; - return; - } - - /* length of HELP text */ - metric_helptxt = flb_sds_create_size(128); - if (!metric_helptxt) { - flb_sds_destroy(sds); - mk_http_status(request, 500); - mk_http_done(request); - buf->users--; - return; - } - metric_helptxt_head = FLB_SDS_HEADER(metric_helptxt); - - /* - * fluentbit_input_records[name="cpu0", hostname="${HOSTNAME}"] NUM TIMESTAMP - * fluentbit_input_bytes[name="cpu0", hostname="${HOSTNAME}"] NUM TIMESTAMP - */ - index = 0; - msgpack_unpacked_init(&result); - msgpack_unpack_next(&result, buf->raw_data, buf->raw_size, &off); - map = result.data; - - /* we need to know number of exposed metrics to reserve a memory */ - for (i = 0; i < map.via.map.size; i++) { - msgpack_object v = map.via.map.ptr[i].val; - /* Iterate sub-map */ - for (j = 0; j < v.via.map.size; j++) { - msgpack_object sv = v.via.map.ptr[j].val; - for (m = 0; m < sv.via.map.size; m++) { - num_metrics++; - } - } - } - metrics_arr = flb_malloc(num_metrics * sizeof(char*)); - if (!metrics_arr) { - flb_errno(); - - mk_http_status(request, 500); - mk_http_done(request); - buf->users--; - - flb_sds_destroy(sds); - flb_sds_destroy(metric_helptxt); - msgpack_unpacked_destroy(&result); - return; - } - - flb_time_get(&tp); - now = flb_time_to_nanosec(&tp) / 1000000; /* in milliseconds */ - time_len = snprintf(time_str, sizeof(time_str) - 1, "%lu", now); - - for (i = 0; i < map.via.map.size; i++) { - msgpack_object k; - msgpack_object v; - - /* Keys: input, output */ - k = map.via.map.ptr[i].key; - v = map.via.map.ptr[i].val; - - /* Iterate sub-map */ - for (j = 0; j < v.via.map.size; j++) { - msgpack_object sk; - msgpack_object sv; - - /* Keys: plugin name , values: metrics */ - sk = v.via.map.ptr[j].key; - sv = v.via.map.ptr[j].val; - - for (m = 0; m < sv.via.map.size; m++) { - msgpack_object mk; - msgpack_object mv; - - mk = sv.via.map.ptr[m].key; - mv = sv.via.map.ptr[m].val; - - /* Convert metric value to string */ - len = snprintf(tmp, sizeof(tmp) - 1, "%" PRIu64 " ", mv.via.u64); - if (len < 0) { - goto error; - } - - /* Allocate buffer */ - sds_metric = flb_sds_create_size(k.via.str.size - + mk.via.str.size - + sk.via.str.size - + len + time_len + 28); - if (sds_metric == NULL) { - goto error; - } - - sds_metric = flb_sds_cat(sds_metric, "fluentbit_", 10); - sds_metric = flb_sds_cat(sds_metric, k.via.str.ptr, k.via.str.size); - sds_metric = flb_sds_cat(sds_metric, "_", 1); - sds_metric = flb_sds_cat(sds_metric, mk.via.str.ptr, mk.via.str.size); - sds_metric = flb_sds_cat(sds_metric, "_total{name=\"", 13); - sds_metric = flb_sds_cat(sds_metric, sk.via.str.ptr, sk.via.str.size); - sds_metric = flb_sds_cat(sds_metric, "\"} ", 3); - sds_metric = flb_sds_cat(sds_metric, tmp, len); - sds_metric = flb_sds_cat(sds_metric, time_str, time_len); - sds_metric = flb_sds_cat(sds_metric, "\n", 1); - metrics_arr[index] = sds_metric; - index++; - } - } - } - - /* Sort metrics in alphabetic order, so we can group them later. */ - qsort(metrics_arr, num_metrics, sizeof(char *), string_cmp); - - /* When a new metric starts add HELP and TYPE annotation. */ - tmp_sds = flb_sds_cat(sds, "# HELP ", 7); - null_check(tmp_sds); - tmp_sds = flb_sds_cat(sds, metrics_arr[0], extract_metric_name_end_position(metrics_arr[0])); - null_check(tmp_sds); - if (!metrics_help_txt(metrics_arr[0], &metric_helptxt)) { - goto error; - } - tmp_sds = flb_sds_cat(sds, metric_helptxt, metric_helptxt_head->len); - null_check(tmp_sds); - tmp_sds = flb_sds_cat(sds, "# TYPE ", 7); - null_check(tmp_sds); - tmp_sds = flb_sds_cat(sds, metrics_arr[0], extract_metric_name_end_position(metrics_arr[0])); - null_check(tmp_sds); - tmp_sds = flb_sds_cat(sds, " counter\n", 9); - null_check(tmp_sds); - - for (i = 0; i < num_metrics; i++) { - tmp_sds = flb_sds_cat(sds, metrics_arr[i], strlen(metrics_arr[i])); - null_check(tmp_sds); - if ((i != num_metrics - 1) && (is_same_metric(metrics_arr[i], metrics_arr[i+1]) == 0)) { - tmp_sds = flb_sds_cat(sds, "# HELP ", 7); - null_check(tmp_sds); - tmp_sds = flb_sds_cat(sds, metrics_arr[i+1], extract_metric_name_end_position(metrics_arr[i+1])); - null_check(tmp_sds); - metric_helptxt_head->len = 0; - if (!metrics_help_txt(metrics_arr[i+1], &metric_helptxt)) { - goto error; - } - tmp_sds = flb_sds_cat(sds, metric_helptxt, metric_helptxt_head->len); - null_check(tmp_sds); - tmp_sds = flb_sds_cat(sds, "# TYPE ", 7); - null_check(tmp_sds); - tmp_sds = flb_sds_cat(sds, metrics_arr[i+1], extract_metric_name_end_position(metrics_arr[i+1])); - null_check(tmp_sds); - tmp_sds = flb_sds_cat(sds, " counter\n", 9); - null_check(tmp_sds); - } - } - - /* Attach uptime */ - uptime = time(NULL) - config->init_time; - len = snprintf(time_str, sizeof(time_str) - 1, "%lu", uptime); - - tmp_sds = flb_sds_cat(sds, - "# HELP fluentbit_uptime Number of seconds that Fluent Bit has " - "been running.\n", 76); - null_check(tmp_sds); - tmp_sds = flb_sds_cat(sds, "# TYPE fluentbit_uptime counter\n", 32); - null_check(tmp_sds); - - tmp_sds = flb_sds_cat(sds, "fluentbit_uptime ", 17); - null_check(tmp_sds); - tmp_sds = flb_sds_cat(sds, time_str, len); - null_check(tmp_sds); - tmp_sds = flb_sds_cat(sds, "\n", 1); - null_check(tmp_sds); - - /* Attach process_start_time_seconds metric. */ - start_time_len = snprintf(start_time_str, sizeof(start_time_str) - 1, - "%lu", config->init_time); - - tmp_sds = flb_sds_cat(sds, "# HELP process_start_time_seconds Start time of the process since unix epoch in seconds.\n", 89); - null_check(tmp_sds); - tmp_sds = flb_sds_cat(sds, "# TYPE process_start_time_seconds gauge\n", 40); - null_check(tmp_sds); - - tmp_sds = flb_sds_cat(sds, "process_start_time_seconds ", 27); - null_check(tmp_sds); - tmp_sds = flb_sds_cat(sds, start_time_str, start_time_len); - null_check(tmp_sds); - tmp_sds = flb_sds_cat(sds, "\n", 1); - null_check(tmp_sds); - - /* Attach fluentbit_build_info metric. */ - tmp_sds = flb_sds_cat(sds, "# HELP fluentbit_build_info Build version information.\n", 55); - null_check(tmp_sds); - tmp_sds = flb_sds_cat(sds, "# TYPE fluentbit_build_info gauge\n", 34); - null_check(tmp_sds); - tmp_sds = flb_sds_cat(sds, "fluentbit_build_info{version=\"", 30); - null_check(tmp_sds); - tmp_sds = flb_sds_cat(sds, FLB_VERSION_STR, sizeof(FLB_VERSION_STR) - 1); - null_check(tmp_sds); - tmp_sds = flb_sds_cat(sds, "\",edition=\"", 11); - null_check(tmp_sds); -#ifdef FLB_ENTERPRISE - tmp_sds = flb_sds_cat(sds, "Enterprise\"} 1\n", 15); - null_check(tmp_sds); -#else - tmp_sds = flb_sds_cat(sds, "Community\"} 1\n", 14); - null_check(tmp_sds); -#endif - - msgpack_unpacked_destroy(&result); - buf->users--; - - mk_http_status(request, 200); - flb_hs_add_content_type_to_req(request, FLB_HS_CONTENT_TYPE_PROMETHEUS); - mk_http_send(request, sds, flb_sds_len(sds), NULL); - for (i = 0; i < num_metrics; i++) { - flb_sds_destroy(metrics_arr[i]); - } - flb_free(metrics_arr); - flb_sds_destroy(sds); - flb_sds_destroy(metric_helptxt); - - mk_http_done(request); - return; - -error: - mk_http_status(request, 500); - mk_http_done(request); - buf->users--; - - for (i = 0; i < index; i++) { - flb_sds_destroy(metrics_arr[i]); - } - flb_free(metrics_arr); - flb_sds_destroy(sds); - flb_sds_destroy(metric_helptxt); - msgpack_unpacked_destroy(&result); -} - -/* API: expose built-in metrics /api/v1/metrics */ -static void cb_metrics(mk_request_t *request, void *data) -{ - struct flb_hs_buf *buf; - - buf = metrics_get_latest(); - if (!buf) { - mk_http_status(request, 404); - mk_http_done(request); - return; - } - - buf->users++; - - mk_http_status(request, 200); - flb_hs_add_content_type_to_req(request, FLB_HS_CONTENT_TYPE_JSON); - mk_http_send(request, buf->data, flb_sds_len(buf->data), NULL); - mk_http_done(request); - - buf->users--; -} - -/* Perform registration */ -int api_v1_metrics(struct flb_hs *hs) -{ - - pthread_key_create(&hs_metrics_key, hs_metrics_key_destroy); - - /* Create a message queue */ - hs->qid_metrics = mk_mq_create(hs->ctx, "/metrics", - cb_mq_metrics, NULL); - - /* HTTP end-points */ - mk_vhost_handler(hs->ctx, hs->vid, "/api/v1/metrics/prometheus", - cb_metrics_prometheus, hs); - mk_vhost_handler(hs->ctx, hs->vid, "/api/v1/metrics", cb_metrics, hs); - - return 0; -} diff --git a/fluent-bit/src/http_server/api/v1/metrics.h b/fluent-bit/src/http_server/api/v1/metrics.h deleted file mode 100644 index f6bb0d017..000000000 --- a/fluent-bit/src/http_server/api/v1/metrics.h +++ /dev/null @@ -1,30 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_HS_API_V1_METRICS_H -#define FLB_HS_API_V1_METRICS_H - -#include <fluent-bit/flb_info.h> -#include <fluent-bit/flb_http_server.h> -#include <fluent-bit/flb_sds.h> - -int api_v1_metrics(struct flb_hs *hs); -flb_sds_t metrics_help_txt(char *metric_name, flb_sds_t *metric_helptxt); - -#endif diff --git a/fluent-bit/src/http_server/api/v1/plugins.c b/fluent-bit/src/http_server/api/v1/plugins.c deleted file mode 100644 index 1b63843cf..000000000 --- a/fluent-bit/src/http_server/api/v1/plugins.c +++ /dev/null @@ -1,109 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <fluent-bit/flb_info.h> -#include <fluent-bit/flb_pack.h> -#include <fluent-bit/flb_input.h> -#include <fluent-bit/flb_filter.h> -#include <fluent-bit/flb_output.h> -#include <fluent-bit/flb_sds.h> -#include <fluent-bit/flb_http_server.h> -#include <msgpack.h> - -/* API: List all built-in plugins */ -static void cb_plugins(mk_request_t *request, void *data) -{ - int len; - flb_sds_t out_buf; - msgpack_packer mp_pck; - msgpack_sbuffer mp_sbuf; - struct mk_list *head; - struct flb_input_plugin *in; - struct flb_filter_plugin *filter; - struct flb_output_plugin *out; - struct flb_hs *hs = data; - struct flb_config *config = hs->config; - - /* initialize buffers */ - msgpack_sbuffer_init(&mp_sbuf); - msgpack_packer_init(&mp_pck, &mp_sbuf, msgpack_sbuffer_write); - - msgpack_pack_map(&mp_pck, 1); - msgpack_pack_str(&mp_pck, 7); - msgpack_pack_str_body(&mp_pck, "plugins", 7); - - /* - * plugins are: inputs, filters, outputs - */ - msgpack_pack_map(&mp_pck, 3); - - /* Inputs */ - msgpack_pack_str(&mp_pck, 6); - msgpack_pack_str_body(&mp_pck, "inputs", 6); - len = mk_list_size(&config->in_plugins); - msgpack_pack_array(&mp_pck, len); - mk_list_foreach(head, &hs->config->in_plugins) { - in = mk_list_entry(head, struct flb_input_plugin, _head); - len = strlen(in->name); - msgpack_pack_str(&mp_pck, len); - msgpack_pack_str_body(&mp_pck, in->name, len); - } - - /* Filters */ - msgpack_pack_str(&mp_pck, 7); - msgpack_pack_str_body(&mp_pck, "filters", 7); - len = mk_list_size(&config->filter_plugins); - msgpack_pack_array(&mp_pck, len); - mk_list_foreach(head, &config->filter_plugins) { - filter = mk_list_entry(head, struct flb_filter_plugin, _head); - len = strlen(filter->name); - msgpack_pack_str(&mp_pck, len); - msgpack_pack_str_body(&mp_pck, filter->name, len); - } - - /* Outputs */ - msgpack_pack_str(&mp_pck, 7); - msgpack_pack_str_body(&mp_pck, "outputs", 7); - len = mk_list_size(&config->out_plugins); - msgpack_pack_array(&mp_pck, len); - mk_list_foreach(head, &config->out_plugins) { - out = mk_list_entry(head, struct flb_output_plugin, _head); - len = strlen(out->name); - msgpack_pack_str(&mp_pck, len); - msgpack_pack_str_body(&mp_pck, out->name, len); - } - - /* Export to JSON */ - out_buf = flb_msgpack_raw_to_json_sds(mp_sbuf.data, mp_sbuf.size); - msgpack_sbuffer_destroy(&mp_sbuf); - - mk_http_status(request, 200); - mk_http_send(request, - out_buf, flb_sds_len(out_buf), NULL); - mk_http_done(request); - - flb_sds_destroy(out_buf); -} - -/* Perform registration */ -int api_v1_plugins(struct flb_hs *hs) -{ - mk_vhost_handler(hs->ctx, hs->vid, "/api/v1/plugins", cb_plugins, hs); - return 0; -} diff --git a/fluent-bit/src/http_server/api/v1/plugins.h b/fluent-bit/src/http_server/api/v1/plugins.h deleted file mode 100644 index d20940321..000000000 --- a/fluent-bit/src/http_server/api/v1/plugins.h +++ /dev/null @@ -1,28 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_HS_API_V1_PLUGINS_H -#define FLB_HS_API_V1_PLUGINS_H - -#include <fluent-bit/flb_info.h> -#include <fluent-bit/flb_http_server.h> - -int api_v1_plugins(struct flb_hs *hs); - -#endif diff --git a/fluent-bit/src/http_server/api/v1/register.c b/fluent-bit/src/http_server/api/v1/register.c deleted file mode 100644 index 093644b3e..000000000 --- a/fluent-bit/src/http_server/api/v1/register.c +++ /dev/null @@ -1,49 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <fluent-bit/flb_info.h> -#include <fluent-bit/flb_http_server.h> - -#include "uptime.h" -#include "metrics.h" -#include "storage.h" -#include "plugins.h" -#include "health.h" -#include "trace.h" - -int api_v1_registration(struct flb_hs *hs) -{ - api_v1_uptime(hs); - api_v1_metrics(hs); - api_v1_plugins(hs); - -#ifdef FLB_HAVE_CHUNK_TRACE - api_v1_trace(hs); -#endif /* FLB_HAVE_CHUNK_TRACE */ - - if (hs->config->health_check == FLB_TRUE) { - api_v1_health(hs); - } - - if (hs->config->storage_metrics == FLB_TRUE) { - api_v1_storage_metrics(hs); - } - - return 0; -} diff --git a/fluent-bit/src/http_server/api/v1/register.h b/fluent-bit/src/http_server/api/v1/register.h deleted file mode 100644 index 978db9e0e..000000000 --- a/fluent-bit/src/http_server/api/v1/register.h +++ /dev/null @@ -1,28 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_API_V1_REG_H -#define FLB_API_V1_REG_H - -#include <fluent-bit/flb_info.h> -#include <fluent-bit/flb_http_server.h> - -int api_v1_registration(struct flb_hs *hs); - -#endif diff --git a/fluent-bit/src/http_server/api/v1/storage.c b/fluent-bit/src/http_server/api/v1/storage.c deleted file mode 100644 index df47859d6..000000000 --- a/fluent-bit/src/http_server/api/v1/storage.c +++ /dev/null @@ -1,204 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <fluent-bit/flb_info.h> -#include <fluent-bit/flb_pack.h> -#include <fluent-bit/flb_sds.h> -#include "storage.h" - -#include <fluent-bit/flb_http_server.h> -#include <msgpack.h> - -pthread_key_t hs_storage_metrics_key; - -/* Return the newest storage metrics buffer */ -static struct flb_hs_buf *storage_metrics_get_latest() -{ - struct flb_hs_buf *buf; - struct mk_list *metrics_list; - - metrics_list = pthread_getspecific(hs_storage_metrics_key); - if (!metrics_list) { - return NULL; - } - - if (mk_list_size(metrics_list) == 0) { - return NULL; - } - - buf = mk_list_entry_last(metrics_list, struct flb_hs_buf, _head); - return buf; -} - -/* Delete unused metrics, note that we only care about the latest node */ -static int cleanup_metrics() -{ - int c = 0; - struct mk_list *tmp; - struct mk_list *head; - struct mk_list *metrics_list; - struct flb_hs_buf *last; - struct flb_hs_buf *entry; - - metrics_list = pthread_getspecific(hs_storage_metrics_key); - if (!metrics_list) { - return -1; - } - - last = storage_metrics_get_latest(); - if (!last) { - return -1; - } - - mk_list_foreach_safe(head, tmp, metrics_list) { - entry = mk_list_entry(head, struct flb_hs_buf, _head); - if (entry != last && entry->users == 0) { - mk_list_del(&entry->_head); - flb_sds_destroy(entry->data); - flb_free(entry->raw_data); - flb_free(entry); - c++; - } - } - - return c; -} - -/* - * Callback invoked every time some storage metrics are received through a - * message queue channel. This function runs in a Monkey HTTP thread - * worker and it purpose is to take the metrics data and store it - * somewhere so then it can be available by the end-points upon - * HTTP client requests. - */ -static void cb_mq_storage_metrics(mk_mq_t *queue, void *data, size_t size) -{ - flb_sds_t out_data; - struct flb_hs_buf *buf; - struct mk_list *metrics_list = NULL; - - metrics_list = pthread_getspecific(hs_storage_metrics_key); - if (!metrics_list) { - metrics_list = flb_malloc(sizeof(struct mk_list)); - if (!metrics_list) { - flb_errno(); - return; - } - mk_list_init(metrics_list); - pthread_setspecific(hs_storage_metrics_key, metrics_list); - } - - /* Convert msgpack to JSON */ - out_data = flb_msgpack_raw_to_json_sds(data, size); - if (!out_data) { - return; - } - - buf = flb_malloc(sizeof(struct flb_hs_buf)); - if (!buf) { - flb_errno(); - flb_sds_destroy(out_data); - return; - } - buf->users = 0; - buf->data = out_data; - - buf->raw_data = flb_malloc(size); - memcpy(buf->raw_data, data, size); - buf->raw_size = size; - - mk_list_add(&buf->_head, metrics_list); - - cleanup_metrics(); -} - -/* FIXME: pending implementation of metrics exit interface -static void cb_mq_storage_metrics_exit(mk_mq_t *queue, void *data) -{ - -} -*/ - -/* API: expose built-in storage metrics /api/v1/storage */ -static void cb_storage(mk_request_t *request, void *data) -{ - struct flb_hs_buf *buf; - - buf = storage_metrics_get_latest(); - if (!buf) { - mk_http_status(request, 404); - mk_http_done(request); - return; - } - - buf->users++; - - mk_http_status(request, 200); - flb_hs_add_content_type_to_req(request, FLB_HS_CONTENT_TYPE_JSON); - mk_http_send(request, buf->data, flb_sds_len(buf->data), NULL); - mk_http_done(request); - - buf->users--; -} - -static void hs_storage_metrics_key_destroy(void *data) -{ - struct mk_list *metrics_list = (struct mk_list*)data; - struct mk_list *tmp; - struct mk_list *head; - struct flb_hs_buf *entry; - - if (metrics_list == NULL) { - return; - } - - mk_list_foreach_safe(head, tmp, metrics_list) { - entry = mk_list_entry(head, struct flb_hs_buf, _head); - if (entry != NULL) { - if (entry->raw_data != NULL) { - flb_free(entry->raw_data); - entry->raw_data = NULL; - } - if (entry->data) { - flb_sds_destroy(entry->data); - entry->data = NULL; - } - mk_list_del(&entry->_head); - flb_free(entry); - } - } - - flb_free(metrics_list); -} - -/* Perform registration */ -int api_v1_storage_metrics(struct flb_hs *hs) -{ - pthread_key_create(&hs_storage_metrics_key, hs_storage_metrics_key_destroy); - - /* Create a message queue */ - hs->qid_storage = mk_mq_create(hs->ctx, "/storage", - cb_mq_storage_metrics, - NULL); - - /* HTTP end-point */ - mk_vhost_handler(hs->ctx, hs->vid, "/api/v1/storage", cb_storage, hs); - - return 0; -} diff --git a/fluent-bit/src/http_server/api/v1/storage.h b/fluent-bit/src/http_server/api/v1/storage.h deleted file mode 100644 index 27410c79b..000000000 --- a/fluent-bit/src/http_server/api/v1/storage.h +++ /dev/null @@ -1,28 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_HS_API_V1_STORAGE_METRICS_H -#define FLB_HS_API_V1_STORAGE_METRICS_H - -#include <fluent-bit/flb_info.h> -#include <fluent-bit/flb_http_server.h> - -int api_v1_storage_metrics(struct flb_hs *hs); - -#endif diff --git a/fluent-bit/src/http_server/api/v1/trace.c b/fluent-bit/src/http_server/api/v1/trace.c deleted file mode 100644 index 95da17343..000000000 --- a/fluent-bit/src/http_server/api/v1/trace.c +++ /dev/null @@ -1,615 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <fluent-bit/flb_info.h> -#include <fluent-bit/flb_pack.h> -#include <fluent-bit/flb_input.h> -#include <fluent-bit/flb_filter.h> -#include <fluent-bit/flb_output.h> -#include <fluent-bit/flb_sds.h> -#include <fluent-bit/flb_http_server.h> -#include <fluent-bit/flb_lib.h> -#include <fluent-bit/flb_chunk_trace.h> -#include <fluent-bit/flb_kv.h> -#include <fluent-bit/flb_utils.h> -#include <msgpack.h> - - -static struct flb_input_instance *find_input(struct flb_hs *hs, const char *name) -{ - struct mk_list *head; - struct flb_input_instance *in; - - - mk_list_foreach(head, &hs->config->inputs) { - in = mk_list_entry(head, struct flb_input_instance, _head); - if (strcmp(name, in->name) == 0) { - return in; - } - if (in->alias) { - if (strcmp(name, in->alias) == 0) { - return in; - } - } - } - return NULL; -} - -static int enable_trace_input(struct flb_hs *hs, const char *name, const char *prefix, const char *output_name, struct mk_list *props) -{ - struct flb_input_instance *in; - - - in = find_input(hs, name); - if (in == NULL) { - return 404; - } - - flb_chunk_trace_context_new(in, output_name, prefix, NULL, props); - return (in->chunk_trace_ctxt == NULL ? 503 : 0); -} - -static int disable_trace_input(struct flb_hs *hs, const char *name) -{ - struct flb_input_instance *in; - - - in = find_input(hs, name); - if (in == NULL) { - return 404; - } - - if (in->chunk_trace_ctxt != NULL) { - flb_chunk_trace_context_destroy(in); - } - return 201; -} - -static flb_sds_t get_input_name(mk_request_t *request) -{ - const char *base = "/api/v1/trace/"; - - - if (request->real_path.data == NULL) { - return NULL; - } - if (request->real_path.len < strlen(base)) { - return NULL; - } - - return flb_sds_create_len(&request->real_path.data[strlen(base)], - request->real_path.len - strlen(base)); -} - -static int http_disable_trace(mk_request_t *request, void *data, const char *input_name, msgpack_packer *mp_pck) -{ - struct flb_hs *hs = data; - int toggled_on = 503; - - - toggled_on = disable_trace_input(hs, input_name); - if (toggled_on < 300) { - msgpack_pack_map(mp_pck, 1); - msgpack_pack_str_with_body(mp_pck, "status", strlen("status")); - msgpack_pack_str_with_body(mp_pck, "ok", strlen("ok")); - return 201; - } - - return toggled_on; -} - -static int msgpack_params_enable_trace(struct flb_hs *hs, msgpack_unpacked *result, const char *input_name) -{ - int ret = -1; - int i; - int x; - flb_sds_t prefix = NULL; - flb_sds_t output_name = NULL; - int toggled_on = -1; - msgpack_object *key; - msgpack_object *val; - struct mk_list *props = NULL; - msgpack_object_kv *param; - msgpack_object_str *param_key; - msgpack_object_str *param_val; - - - if (result->data.type == MSGPACK_OBJECT_MAP) { - for (i = 0; i < result->data.via.map.size; i++) { - key = &result->data.via.map.ptr[i].key; - val = &result->data.via.map.ptr[i].val; - - if (key->type != MSGPACK_OBJECT_STR) { - ret = -1; - goto parse_error; - } - - if (strncmp(key->via.str.ptr, "prefix", key->via.str.size) == 0) { - if (val->type != MSGPACK_OBJECT_STR) { - ret = -1; - goto parse_error; - } - if (prefix != NULL) { - flb_sds_destroy(prefix); - } - prefix = flb_sds_create_len(val->via.str.ptr, val->via.str.size); - } - else if (strncmp(key->via.str.ptr, "output", key->via.str.size) == 0) { - if (val->type != MSGPACK_OBJECT_STR) { - ret = -1; - goto parse_error; - } - if (output_name != NULL) { - flb_sds_destroy(output_name); - } - output_name = flb_sds_create_len(val->via.str.ptr, val->via.str.size); - } - else if (strncmp(key->via.str.ptr, "params", key->via.str.size) == 0) { - if (val->type != MSGPACK_OBJECT_MAP) { - ret = -1; - goto parse_error; - } - if (props != NULL) { - flb_free(props); - } - props = flb_calloc(1, sizeof(struct mk_list)); - flb_kv_init(props); - for (x = 0; x < val->via.map.size; x++) { - param = &val->via.map.ptr[x]; - if (param->val.type != MSGPACK_OBJECT_STR) { - ret = -1; - goto parse_error; - } - if (param->key.type != MSGPACK_OBJECT_STR) { - ret = -1; - goto parse_error; - } - param_key = ¶m->key.via.str; - param_val = ¶m->val.via.str; - flb_kv_item_create_len(props, - (char *)param_key->ptr, param_key->size, - (char *)param_val->ptr, param_val->size); - } - } - } - - if (output_name == NULL) { - output_name = flb_sds_create("stdout"); - } - - toggled_on = enable_trace_input(hs, input_name, prefix, output_name, props); - if (!toggled_on) { - ret = -1; - goto parse_error; - } - } - -parse_error: - if (prefix) flb_sds_destroy(prefix); - if (output_name) flb_sds_destroy(output_name); - if (props != NULL) { - flb_kv_release(props); - flb_free(props); - } - return ret; -} - -static int http_enable_trace(mk_request_t *request, void *data, const char *input_name, msgpack_packer *mp_pck) -{ - char *buf = NULL; - size_t buf_size; - msgpack_unpacked result; - int ret = -1; - int rc = -1; - int i; - int x; - size_t off = 0; - int root_type = MSGPACK_OBJECT_ARRAY; - struct flb_hs *hs = data; - flb_sds_t prefix = NULL; - flb_sds_t output_name = NULL; - msgpack_object *key; - msgpack_object *val; - struct mk_list *props = NULL; - struct flb_chunk_trace_limit limit = { 0 }; - struct flb_input_instance *input_instance; - - - if (request->method == MK_METHOD_GET) { - ret = enable_trace_input(hs, input_name, "trace.", "stdout", NULL); - if (ret == 0) { - msgpack_pack_map(mp_pck, 1); - msgpack_pack_str_with_body(mp_pck, "status", strlen("status")); - msgpack_pack_str_with_body(mp_pck, "ok", strlen("ok")); - return 200; - } - else { - flb_error("unable to enable tracing for %s", input_name); - goto input_error; - } - } - - msgpack_unpacked_init(&result); - rc = flb_pack_json(request->data.data, request->data.len, &buf, &buf_size, - &root_type, NULL); - if (rc == -1) { - ret = 503; - flb_error("unable to parse json parameters"); - goto unpack_error; - } - - rc = msgpack_unpack_next(&result, buf, buf_size, &off); - if (rc != MSGPACK_UNPACK_SUCCESS) { - ret = 503; - flb_error("unable to unpack msgpack parameters for %s", input_name); - goto unpack_error; - } - - if (result.data.type == MSGPACK_OBJECT_MAP) { - for (i = 0; i < result.data.via.map.size; i++) { - key = &result.data.via.map.ptr[i].key; - val = &result.data.via.map.ptr[i].val; - - if (key->type != MSGPACK_OBJECT_STR) { - ret = 503; - flb_error("non string key in parameters"); - goto parse_error; - } - - if (strncmp(key->via.str.ptr, "prefix", key->via.str.size) == 0) { - if (val->type != MSGPACK_OBJECT_STR) { - ret = 503; - flb_error("prefix is not a string"); - goto parse_error; - } - if (prefix != NULL) { - flb_sds_destroy(prefix); - } - prefix = flb_sds_create_len(val->via.str.ptr, val->via.str.size); - } - else if (strncmp(key->via.str.ptr, "output", key->via.str.size) == 0) { - if (val->type != MSGPACK_OBJECT_STR) { - ret = 503; - flb_error("output is not a string"); - goto parse_error; - } - if (output_name != NULL) { - flb_sds_destroy(output_name); - } - output_name = flb_sds_create_len(val->via.str.ptr, val->via.str.size); - } - else if (strncmp(key->via.str.ptr, "params", key->via.str.size) == 0) { - if (val->type != MSGPACK_OBJECT_MAP) { - ret = 503; - flb_error("output params is not a maps"); - goto parse_error; - } - props = flb_calloc(1, sizeof(struct mk_list)); - flb_kv_init(props); - for (x = 0; x < val->via.map.size; x++) { - if (val->via.map.ptr[x].val.type != MSGPACK_OBJECT_STR) { - ret = 503; - flb_error("output parameter key is not a string"); - goto parse_error; - } - if (val->via.map.ptr[x].key.type != MSGPACK_OBJECT_STR) { - ret = 503; - flb_error("output parameter value is not a string"); - goto parse_error; - } - flb_kv_item_create_len(props, - (char *)val->via.map.ptr[x].key.via.str.ptr, val->via.map.ptr[x].key.via.str.size, - (char *)val->via.map.ptr[x].val.via.str.ptr, val->via.map.ptr[x].val.via.str.size); - } - } - else if (strncmp(key->via.str.ptr, "limit", key->via.str.size) == 0) { - if (val->type != MSGPACK_OBJECT_MAP) { - ret = 503; - flb_error("limit must be a map of limit types"); - goto parse_error; - } - if (val->via.map.size != 1) { - ret = 503; - flb_error("limit must have a single limit type"); - goto parse_error; - } - if (val->via.map.ptr[0].key.type != MSGPACK_OBJECT_STR) { - ret = 503; - flb_error("limit type (key) must be a string"); - goto parse_error; - } - if (val->via.map.ptr[0].val.type != MSGPACK_OBJECT_POSITIVE_INTEGER) { - ret = 503; - flb_error("limit type must be an integer"); - goto parse_error; - } - if (strncmp(val->via.map.ptr[0].key.via.str.ptr, "seconds", val->via.map.ptr[0].key.via.str.size) == 0) { - limit.type = FLB_CHUNK_TRACE_LIMIT_TIME; - limit.seconds = val->via.map.ptr[0].val.via.u64; - } - else if (strncmp(val->via.map.ptr[0].key.via.str.ptr, "count", val->via.map.ptr[0].key.via.str.size) == 0) { - limit.type = FLB_CHUNK_TRACE_LIMIT_COUNT; - limit.count = val->via.map.ptr[0].val.via.u64; - } - else { - ret = 503; - flb_error("unknown limit type"); - goto parse_error; - } - } - } - - if (output_name == NULL) { - output_name = flb_sds_create("stdout"); - } - - ret = enable_trace_input(hs, input_name, prefix, output_name, props); - if (ret != 0) { - flb_error("error when enabling tracing"); - goto parse_error; - } - - if (limit.type != 0) { - input_instance = find_input(hs, input_name); - if (limit.type == FLB_CHUNK_TRACE_LIMIT_TIME) { - flb_chunk_trace_context_set_limit(input_instance->chunk_trace_ctxt, limit.type, limit.seconds); - } - else if (limit.type == FLB_CHUNK_TRACE_LIMIT_COUNT) { - flb_chunk_trace_context_set_limit(input_instance->chunk_trace_ctxt, limit.type, limit.count); - } - } - } - - msgpack_pack_map(mp_pck, 1); - msgpack_pack_str_with_body(mp_pck, "status", strlen("status")); - msgpack_pack_str_with_body(mp_pck, "ok", strlen("ok")); - - ret = 200; -parse_error: - if (prefix) flb_sds_destroy(prefix); - if (output_name) flb_sds_destroy(output_name); - if (props != NULL) { - flb_kv_release(props); - flb_free(props); - } -unpack_error: - msgpack_unpacked_destroy(&result); - if (buf != NULL) { - flb_free(buf); - } -input_error: - return ret; -} - -static void cb_trace(mk_request_t *request, void *data) -{ - flb_sds_t out_buf; - msgpack_sbuffer mp_sbuf; - msgpack_packer mp_pck; - int response = 404; - flb_sds_t input_name = NULL; - - - /* initialize buffers */ - msgpack_sbuffer_init(&mp_sbuf); - msgpack_packer_init(&mp_pck, &mp_sbuf, msgpack_sbuffer_write); - - input_name = get_input_name(request); - if (input_name == NULL) { - response = 404; - goto error; - } - - if (request->method == MK_METHOD_POST || request->method == MK_METHOD_GET) { - response = http_enable_trace(request, data, input_name, &mp_pck); - } - else if (request->method == MK_METHOD_DELETE) { - response = http_disable_trace(request, data, input_name, &mp_pck); - } -error: - if (response == 404) { - msgpack_pack_map(&mp_pck, 1); - msgpack_pack_str_with_body(&mp_pck, "status", strlen("status")); - msgpack_pack_str_with_body(&mp_pck, "not found", strlen("not found")); - } - else if (response == 503) { - msgpack_pack_map(&mp_pck, 1); - msgpack_pack_str_with_body(&mp_pck, "status", strlen("status")); - msgpack_pack_str_with_body(&mp_pck, "error", strlen("error")); - } - - if (input_name != NULL) { - flb_sds_destroy(input_name); - } - - /* Export to JSON */ - out_buf = flb_msgpack_raw_to_json_sds(mp_sbuf.data, mp_sbuf.size); - if (out_buf == NULL) { - mk_http_status(request, 503); - mk_http_done(request); - return; - } - - mk_http_status(request, response); - mk_http_send(request, out_buf, flb_sds_len(out_buf), NULL); - mk_http_done(request); - - msgpack_sbuffer_destroy(&mp_sbuf); - flb_sds_destroy(out_buf); -} - -static void cb_traces(mk_request_t *request, void *data) -{ - flb_sds_t out_buf; - msgpack_sbuffer mp_sbuf; - msgpack_packer mp_pck; - int ret; - char *buf = NULL; - size_t buf_size; - int root_type = MSGPACK_OBJECT_ARRAY; - msgpack_unpacked result; - flb_sds_t error_msg = NULL; - int response = 200; - flb_sds_t input_name; - msgpack_object_array *inputs = NULL; - size_t off = 0; - int i; - - - /* initialize buffers */ - msgpack_sbuffer_init(&mp_sbuf); - msgpack_packer_init(&mp_pck, &mp_sbuf, msgpack_sbuffer_write); - - msgpack_unpacked_init(&result); - ret = flb_pack_json(request->data.data, request->data.len, &buf, &buf_size, - &root_type, NULL); - if (ret == -1) { - goto unpack_error; - } - - ret = msgpack_unpack_next(&result, buf, buf_size, &off); - if (ret != MSGPACK_UNPACK_SUCCESS) { - ret = -1; - error_msg = flb_sds_create("unfinished input"); - goto unpack_error; - } - - if (result.data.type != MSGPACK_OBJECT_MAP) { - response = 503; - error_msg = flb_sds_create("input is not an object"); - goto unpack_error; - } - - for (i = 0; i < result.data.via.map.size; i++) { - if (result.data.via.map.ptr[i].val.type != MSGPACK_OBJECT_ARRAY) { - continue; - } - if (result.data.via.map.ptr[i].key.type != MSGPACK_OBJECT_STR) { - continue; - } - if (result.data.via.map.ptr[i].key.via.str.size < strlen("inputs")) { - continue; - } - if (strncmp(result.data.via.map.ptr[i].key.via.str.ptr, "inputs", strlen("inputs"))) { - continue; - } - inputs = &result.data.via.map.ptr[i].val.via.array; - } - - if (inputs == NULL) { - response = 503; - error_msg = flb_sds_create("inputs not found"); - goto unpack_error; - } - - msgpack_pack_map(&mp_pck, 2); - - msgpack_pack_str_with_body(&mp_pck, "inputs", strlen("inputs")); - msgpack_pack_map(&mp_pck, inputs->size); - - for (i = 0; i < inputs->size; i++) { - input_name = flb_sds_create_len(inputs->ptr[i].via.str.ptr, inputs->ptr[i].via.str.size); - msgpack_pack_str_with_body(&mp_pck, input_name, flb_sds_len(input_name)); - - if (inputs->ptr[i].type != MSGPACK_OBJECT_STR) { - msgpack_pack_map(&mp_pck, 1); - msgpack_pack_str_with_body(&mp_pck, "status", strlen("status")); - msgpack_pack_str_with_body(&mp_pck, "error", strlen("error")); - } - else { - if (request->method == MK_METHOD_POST || request->method == MK_METHOD_GET) { - ret = msgpack_params_enable_trace((struct flb_hs *)data, &result, input_name); - if (ret != 0) { - msgpack_pack_map(&mp_pck, 2); - msgpack_pack_str_with_body(&mp_pck, "status", strlen("status")); - msgpack_pack_str_with_body(&mp_pck, "error", strlen("error")); - msgpack_pack_str_with_body(&mp_pck, "returncode", strlen("returncode")); - msgpack_pack_int64(&mp_pck, ret); - } - else { - msgpack_pack_map(&mp_pck, 1); - msgpack_pack_str_with_body(&mp_pck, "status", strlen("status")); - msgpack_pack_str_with_body(&mp_pck, "ok", strlen("ok")); - } - } - else if (request->method == MK_METHOD_DELETE) { - disable_trace_input((struct flb_hs *)data, input_name); - } - else { - msgpack_pack_map(&mp_pck, 2); - msgpack_pack_str_with_body(&mp_pck, "status", strlen("status")); - msgpack_pack_str_with_body(&mp_pck, "error", strlen("error")); - msgpack_pack_str_with_body(&mp_pck, "message", strlen("message")); - msgpack_pack_str_with_body(&mp_pck, "method not allowed", strlen("method not allowed")); - } - } - } - - msgpack_pack_str_with_body(&mp_pck, "result", strlen("result")); -unpack_error: - if (buf != NULL) { - flb_free(buf); - } - msgpack_unpacked_destroy(&result); - if (response == 404) { - msgpack_pack_map(&mp_pck, 1); - msgpack_pack_str_with_body(&mp_pck, "status", strlen("status")); - msgpack_pack_str_with_body(&mp_pck, "not found", strlen("not found")); - } - else if (response == 503) { - msgpack_pack_map(&mp_pck, 2); - msgpack_pack_str_with_body(&mp_pck, "status", strlen("status")); - msgpack_pack_str_with_body(&mp_pck, "error", strlen("error")); - msgpack_pack_str_with_body(&mp_pck, "message", strlen("message")); - if (error_msg) { - msgpack_pack_str_with_body(&mp_pck, error_msg, flb_sds_len(error_msg)); - flb_sds_destroy(error_msg); - } - else { - msgpack_pack_str_with_body(&mp_pck, "unknown error", strlen("unknown error")); - } - } - else { - msgpack_pack_map(&mp_pck, 1); - msgpack_pack_str_with_body(&mp_pck, "status", strlen("status")); - msgpack_pack_str_with_body(&mp_pck, "ok", strlen("ok")); - } - - /* Export to JSON */ - out_buf = flb_msgpack_raw_to_json_sds(mp_sbuf.data, mp_sbuf.size); - if (out_buf == NULL) { - out_buf = flb_sds_create("serialization error"); - } - msgpack_sbuffer_destroy(&mp_sbuf); - - mk_http_status(request, response); - mk_http_send(request, - out_buf, flb_sds_len(out_buf), NULL); - mk_http_done(request); - - flb_sds_destroy(out_buf); -} - -/* Perform registration */ -int api_v1_trace(struct flb_hs *hs) -{ - mk_vhost_handler(hs->ctx, hs->vid, "/api/v1/traces/", cb_traces, hs); - mk_vhost_handler(hs->ctx, hs->vid, "/api/v1/trace/*", cb_trace, hs); - return 0; -} diff --git a/fluent-bit/src/http_server/api/v1/trace.h b/fluent-bit/src/http_server/api/v1/trace.h deleted file mode 100644 index 236925de8..000000000 --- a/fluent-bit/src/http_server/api/v1/trace.h +++ /dev/null @@ -1,28 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_HS_API_V1_TRACE_H -#define FLB_HS_API_V1_TRACE_H - -#include <fluent-bit/flb_info.h> -#include <fluent-bit/flb_http_server.h> - -int api_v1_trace(struct flb_hs *hs); - -#endif diff --git a/fluent-bit/src/http_server/api/v1/uptime.c b/fluent-bit/src/http_server/api/v1/uptime.c deleted file mode 100644 index 37f971871..000000000 --- a/fluent-bit/src/http_server/api/v1/uptime.c +++ /dev/null @@ -1,111 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <fluent-bit/flb_info.h> -#include <fluent-bit/flb_pack.h> -#include <fluent-bit/flb_http_server.h> -#include <fluent-bit/flb_mem.h> - -#define FLB_UPTIME_ONEDAY 86400 -#define FLB_UPTIME_ONEHOUR 3600 -#define FLB_UPTIME_ONEMINUTE 60 - -/* Append human-readable uptime */ -static void uptime_hr(time_t uptime, msgpack_packer *mp_pck) -{ - int len; - int days; - int hours; - int minutes; - int seconds; - long int upmind; - long int upminh; - char buf[256]; - - /* days */ - days = uptime / FLB_UPTIME_ONEDAY; - upmind = uptime - (days * FLB_UPTIME_ONEDAY); - - /* hours */ - hours = upmind / FLB_UPTIME_ONEHOUR; - upminh = upmind - hours * FLB_UPTIME_ONEHOUR; - - /* minutes */ - minutes = upminh / FLB_UPTIME_ONEMINUTE; - seconds = upminh - minutes * FLB_UPTIME_ONEMINUTE; - - len = snprintf(buf, sizeof(buf) - 1, - "Fluent Bit has been running: " - " %i day%s, %i hour%s, %i minute%s and %i second%s", - days, (days > 1) ? "s" : "", hours, \ - (hours > 1) ? "s" : "", minutes, \ - (minutes > 1) ? "s" : "", seconds, \ - (seconds > 1) ? "s" : ""); - msgpack_pack_str(mp_pck, 9); - msgpack_pack_str_body(mp_pck, "uptime_hr", 9); - msgpack_pack_str(mp_pck, len); - msgpack_pack_str_body(mp_pck, buf, len); -} - -/* API: List all built-in plugins */ -static void cb_uptime(mk_request_t *request, void *data) -{ - flb_sds_t out_buf; - size_t out_size; - time_t uptime; - msgpack_packer mp_pck; - msgpack_sbuffer mp_sbuf; - struct flb_hs *hs = data; - struct flb_config *config = hs->config; - - /* initialize buffers */ - msgpack_sbuffer_init(&mp_sbuf); - msgpack_packer_init(&mp_pck, &mp_sbuf, msgpack_sbuffer_write); - - msgpack_pack_map(&mp_pck, 2); - msgpack_pack_str(&mp_pck, 10); - msgpack_pack_str_body(&mp_pck, "uptime_sec", 10); - - uptime = time(NULL) - config->init_time; - msgpack_pack_uint64(&mp_pck, uptime); - - uptime_hr(uptime, &mp_pck); - - /* Export to JSON */ - out_buf = flb_msgpack_raw_to_json_sds(mp_sbuf.data, mp_sbuf.size); - msgpack_sbuffer_destroy(&mp_sbuf); - if (!out_buf) { - return; - } - out_size = flb_sds_len(out_buf); - - mk_http_status(request, 200); - flb_hs_add_content_type_to_req(request, FLB_HS_CONTENT_TYPE_JSON); - mk_http_send(request, out_buf, out_size, NULL); - mk_http_done(request); - - flb_sds_destroy(out_buf); -} - -/* Perform registration */ -int api_v1_uptime(struct flb_hs *hs) -{ - mk_vhost_handler(hs->ctx, hs->vid, "/api/v1/uptime", cb_uptime, hs); - return 0; -} diff --git a/fluent-bit/src/http_server/api/v1/uptime.h b/fluent-bit/src/http_server/api/v1/uptime.h deleted file mode 100644 index 5a4247798..000000000 --- a/fluent-bit/src/http_server/api/v1/uptime.h +++ /dev/null @@ -1,28 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_HS_API_V1_UPTIME_H -#define FLB_HS_API_V1_UPTIME_H - -#include <fluent-bit/flb_info.h> -#include <fluent-bit/flb_http_server.h> - -int api_v1_uptime(struct flb_hs *hs); - -#endif diff --git a/fluent-bit/src/http_server/api/v2/CMakeLists.txt b/fluent-bit/src/http_server/api/v2/CMakeLists.txt deleted file mode 100644 index a9d590fbd..000000000 --- a/fluent-bit/src/http_server/api/v2/CMakeLists.txt +++ /dev/null @@ -1,10 +0,0 @@ -# api/v2 -set(src - metrics.c - reload.c - register.c - ) - -include_directories(${MONKEY_INCLUDE_DIR}) -add_library(api-v2 STATIC ${src}) -target_link_libraries(api-v2 monkey-core-static fluent-bit-static) diff --git a/fluent-bit/src/http_server/api/v2/metrics.c b/fluent-bit/src/http_server/api/v2/metrics.c deleted file mode 100644 index 27513b7a4..000000000 --- a/fluent-bit/src/http_server/api/v2/metrics.c +++ /dev/null @@ -1,259 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <fluent-bit/flb_info.h> -#include <fluent-bit/flb_pack.h> -#include <fluent-bit/flb_input.h> -#include <fluent-bit/flb_filter.h> -#include <fluent-bit/flb_output.h> -#include <fluent-bit/flb_sds.h> -#include <fluent-bit/flb_version.h> -#include <fluent-bit/flb_time.h> -#include "metrics.h" - -#include <fluent-bit/flb_http_server.h> - -#define null_check(x) do { if (!x) { goto error; } else {sds = x;} } while (0) - -pthread_key_t hs_metrics_v2_key; - -static struct mk_list *hs_metrics_v2_key_create() -{ - struct mk_list *metrics_list = NULL; - - metrics_list = flb_malloc(sizeof(struct mk_list)); - if (metrics_list == NULL) { - flb_errno(); - return NULL; - } - mk_list_init(metrics_list); - pthread_setspecific(hs_metrics_v2_key, metrics_list); - - return metrics_list; -} - -static void hs_metrics_v2_key_destroy(void *data) -{ - struct mk_list *metrics_list = (struct mk_list*) data; - struct mk_list *tmp; - struct mk_list *head; - struct flb_hs_buf *entry; - - if (metrics_list == NULL) { - return; - } - mk_list_foreach_safe(head, tmp, metrics_list) { - entry = mk_list_entry(head, struct flb_hs_buf, _head); - if (entry != NULL) { - if (entry->raw_data != NULL) { - cmt_destroy(entry->raw_data); - entry->raw_data = NULL; - } - mk_list_del(&entry->_head); - flb_free(entry); - } - } - - flb_free(metrics_list); -} - -/* Return the newest metrics buffer */ -static struct flb_hs_buf *metrics_get_latest() -{ - struct flb_hs_buf *buf; - struct mk_list *metrics_list; - - metrics_list = pthread_getspecific(hs_metrics_v2_key); - if (!metrics_list) { - return NULL; - } - - if (mk_list_size(metrics_list) == 0) { - return NULL; - } - - buf = mk_list_entry_last(metrics_list, struct flb_hs_buf, _head); - return buf; -} - -/* Delete unused metrics, note that we only care about the latest node */ -static int cleanup_metrics() -{ - int c = 0; - struct mk_list *tmp; - struct mk_list *head; - struct mk_list *metrics_list; - struct flb_hs_buf *last; - struct flb_hs_buf *entry; - - metrics_list = pthread_getspecific(hs_metrics_v2_key); - if (!metrics_list) { - return -1; - } - - last = metrics_get_latest(); - if (!last) { - return -1; - } - - mk_list_foreach_safe(head, tmp, metrics_list) { - entry = mk_list_entry(head, struct flb_hs_buf, _head); - if (entry != last && entry->users == 0) { - mk_list_del(&entry->_head); - cmt_destroy(entry->raw_data); - flb_free(entry); - c++; - } - } - - return c; -} - -/* - * Callback invoked every time some metrics are received through a message queue channel. - * This function runs in a Monkey HTTP thread worker and it purpose is to take the metrics - * data and store it somewhere so then it can be available by the end-points upon - * HTTP client requests. - */ -static void cb_mq_metrics(mk_mq_t *queue, void *data, size_t size) -{ - int ret; - size_t off = 0; - struct cmt *cmt; - struct flb_hs_buf *buf; - struct mk_list *metrics_list = NULL; - - metrics_list = pthread_getspecific(hs_metrics_v2_key); - if (!metrics_list) { - metrics_list = hs_metrics_v2_key_create(); - if (metrics_list == NULL) { - return; - } - } - - /* decode cmetrics */ - ret = cmt_decode_msgpack_create(&cmt, data, size, &off); - if (ret != 0) { - return; - } - - buf = flb_malloc(sizeof(struct flb_hs_buf)); - if (!buf) { - flb_errno(); - return; - } - buf->users = 0; - buf->data = NULL; - - /* Store CMetrics context as the raw_data */ - buf->raw_data = cmt; - buf->raw_size = 0; - - mk_list_add(&buf->_head, metrics_list); - cleanup_metrics(); -} - -/* API: expose metrics in Prometheus format /api/v2/metrics/prometheus */ -static void cb_metrics_prometheus(mk_request_t *request, void *data) -{ - struct cmt *cmt; - struct flb_hs_buf *buf; - cfl_sds_t payload; - - buf = metrics_get_latest(); - if (!buf) { - mk_http_status(request, 404); - mk_http_done(request); - return; - } - - cmt = (struct cmt *) buf->raw_data; - - /* convert CMetrics to text */ - payload = cmt_encode_prometheus_create(cmt, CMT_FALSE); - if (!payload) { - mk_http_status(request, 500); - mk_http_done(request); - return; - } - - buf->users++; - - mk_http_status(request, 200); - flb_hs_add_content_type_to_req(request, FLB_HS_CONTENT_TYPE_PROMETHEUS); - mk_http_send(request, payload, cfl_sds_len(payload), NULL); - mk_http_done(request); - - cmt_encode_prometheus_destroy(payload); - - buf->users--; -} - -/* API: expose built-in metrics /api/v1/metrics (JSON format) */ -static void cb_metrics(mk_request_t *request, void *data) -{ - struct cmt *cmt; - struct flb_hs_buf *buf; - cfl_sds_t payload; - - buf = metrics_get_latest(); - if (!buf) { - mk_http_status(request, 404); - mk_http_done(request); - return; - } - - cmt = (struct cmt *) buf->raw_data; - - /* convert CMetrics to text */ - payload = cmt_encode_text_create(cmt); - if (!payload) { - mk_http_status(request, 500); - mk_http_done(request); - return; - } - - buf->users++; - - mk_http_status(request, 200); - mk_http_send(request, payload, cfl_sds_len(payload), NULL); - mk_http_done(request); - - cmt_encode_text_destroy(payload); - - buf->users--; -} - -/* Perform registration */ -int api_v2_metrics(struct flb_hs *hs) -{ - - pthread_key_create(&hs_metrics_v2_key, hs_metrics_v2_key_destroy); - - /* Create a message queue */ - hs->qid_metrics_v2 = mk_mq_create(hs->ctx, "/metrics_v2", - cb_mq_metrics, NULL); - /* HTTP end-points */ - mk_vhost_handler(hs->ctx, hs->vid, "/api/v2/metrics/prometheus", - cb_metrics_prometheus, hs); - - mk_vhost_handler(hs->ctx, hs->vid, "/api/v2/metrics", cb_metrics, hs); - - return 0; -} diff --git a/fluent-bit/src/http_server/api/v2/metrics.h b/fluent-bit/src/http_server/api/v2/metrics.h deleted file mode 100644 index 5336865dd..000000000 --- a/fluent-bit/src/http_server/api/v2/metrics.h +++ /dev/null @@ -1,28 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_HS_API_V2_METRICS_H -#define FLB_HS_API_V2_METRICS_H - -#include <fluent-bit/flb_info.h> -#include <fluent-bit/flb_http_server.h> - -int api_v2_metrics(struct flb_hs *hs); - -#endif diff --git a/fluent-bit/src/http_server/api/v2/register.c b/fluent-bit/src/http_server/api/v2/register.c deleted file mode 100644 index 7a0956fbf..000000000 --- a/fluent-bit/src/http_server/api/v2/register.c +++ /dev/null @@ -1,31 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2023 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <fluent-bit/flb_info.h> -#include <fluent-bit/flb_http_server.h> - -#include "metrics.h" -#include "reload.h" - -int api_v2_registration(struct flb_hs *hs) -{ - api_v2_reload(hs); - api_v2_metrics(hs); - return 0; -} diff --git a/fluent-bit/src/http_server/api/v2/register.h b/fluent-bit/src/http_server/api/v2/register.h deleted file mode 100644 index da6d78f3a..000000000 --- a/fluent-bit/src/http_server/api/v2/register.h +++ /dev/null @@ -1,28 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2023 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_API_V2_REG_H -#define FLB_API_V2_REG_H - -#include <fluent-bit/flb_info.h> -#include <fluent-bit/flb_http_server.h> - -int api_v2_registration(struct flb_hs *hs); - -#endif diff --git a/fluent-bit/src/http_server/api/v2/reload.c b/fluent-bit/src/http_server/api/v2/reload.c deleted file mode 100644 index 3bb5159fa..000000000 --- a/fluent-bit/src/http_server/api/v2/reload.c +++ /dev/null @@ -1,161 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2023 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <fluent-bit/flb_info.h> -#include <fluent-bit/flb_pack.h> -#include <fluent-bit/flb_input.h> -#include <fluent-bit/flb_filter.h> -#include <fluent-bit/flb_output.h> -#include <fluent-bit/flb_sds.h> -#include <fluent-bit/flb_version.h> -#include <fluent-bit/flb_time.h> -#include <fluent-bit/flb_lib.h> -#include <fluent-bit/flb_reload.h> -#include "reload.h" - -#include <signal.h> - -#include <fluent-bit/flb_http_server.h> - -static void handle_reload_request(mk_request_t *request, struct flb_config *config) -{ - int ret; - flb_sds_t out_buf; - size_t out_size; - msgpack_packer mp_pck; - msgpack_sbuffer mp_sbuf; - - /* initialize buffers */ - msgpack_sbuffer_init(&mp_sbuf); - msgpack_packer_init(&mp_pck, &mp_sbuf, msgpack_sbuffer_write); - - msgpack_pack_map(&mp_pck, 2); - msgpack_pack_str(&mp_pck, 6); - msgpack_pack_str_body(&mp_pck, "reload", 6); - -#ifdef FLB_SYSTEM_WINDOWS - ret = -1; - - msgpack_pack_str(&mp_pck, 11); - msgpack_pack_str_body(&mp_pck, "unsupported", 11); - msgpack_pack_str(&mp_pck, 6); - msgpack_pack_str_body(&mp_pck, "status", 6); - msgpack_pack_int64(&mp_pck, ret); -#else - if (config->enable_hot_reload != FLB_TRUE) { - msgpack_pack_str(&mp_pck, 11); - msgpack_pack_str_body(&mp_pck, "not enabled", 11); - msgpack_pack_str(&mp_pck, 6); - msgpack_pack_str_body(&mp_pck, "status", 6); - msgpack_pack_int64(&mp_pck, -1); - } - else { - ret = kill(getpid(), SIGHUP); - if (ret != 0) { - mk_http_status(request, 500); - mk_http_done(request); - return; - } - - msgpack_pack_str(&mp_pck, 4); - msgpack_pack_str_body(&mp_pck, "done", 4); - msgpack_pack_str(&mp_pck, 6); - msgpack_pack_str_body(&mp_pck, "status", 6); - msgpack_pack_int64(&mp_pck, ret); - } - -#endif - - /* Export to JSON */ - out_buf = flb_msgpack_raw_to_json_sds(mp_sbuf.data, mp_sbuf.size); - msgpack_sbuffer_destroy(&mp_sbuf); - if (!out_buf) { - mk_http_status(request, 400); - mk_http_done(request); - return; - } - out_size = flb_sds_len(out_buf); - - mk_http_status(request, 200); - flb_hs_add_content_type_to_req(request, FLB_HS_CONTENT_TYPE_JSON); - mk_http_send(request, out_buf, out_size, NULL); - mk_http_done(request); - - flb_sds_destroy(out_buf); -} - -static void handle_get_reload_status(mk_request_t *request, struct flb_config *config) -{ - flb_sds_t out_buf; - size_t out_size; - msgpack_packer mp_pck; - msgpack_sbuffer mp_sbuf; - - /* initialize buffers */ - msgpack_sbuffer_init(&mp_sbuf); - msgpack_packer_init(&mp_pck, &mp_sbuf, msgpack_sbuffer_write); - - msgpack_pack_map(&mp_pck, 1); - msgpack_pack_str(&mp_pck, 16); - msgpack_pack_str_body(&mp_pck, "hot_reload_count", 16); - msgpack_pack_int64(&mp_pck, config->hot_reloaded_count); - - /* Export to JSON */ - out_buf = flb_msgpack_raw_to_json_sds(mp_sbuf.data, mp_sbuf.size); - msgpack_sbuffer_destroy(&mp_sbuf); - if (!out_buf) { - mk_http_status(request, 400); - mk_http_done(request); - return; - } - out_size = flb_sds_len(out_buf); - - mk_http_status(request, 200); - flb_hs_add_content_type_to_req(request, FLB_HS_CONTENT_TYPE_JSON); - mk_http_send(request, out_buf, out_size, NULL); - mk_http_done(request); - - flb_sds_destroy(out_buf); -} - -static void cb_reload(mk_request_t *request, void *data) -{ - struct flb_hs *hs = data; - struct flb_config *config = hs->config; - - if (request->method == MK_METHOD_POST || - request->method == MK_METHOD_PUT) { - handle_reload_request(request, config); - } - else if (request->method == MK_METHOD_GET) { - handle_get_reload_status(request, config); - } - else { - mk_http_status(request, 400); - mk_http_done(request); - } -} - -/* Perform registration */ -int api_v2_reload(struct flb_hs *hs) -{ - mk_vhost_handler(hs->ctx, hs->vid, "/api/v2/reload", cb_reload, hs); - - return 0; -} diff --git a/fluent-bit/src/http_server/api/v2/reload.h b/fluent-bit/src/http_server/api/v2/reload.h deleted file mode 100644 index e64e867d6..000000000 --- a/fluent-bit/src/http_server/api/v2/reload.h +++ /dev/null @@ -1,28 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2023 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_HS_API_V2_RELOAD_H -#define FLB_HS_API_V2_RELOAD_H - -#include <fluent-bit/flb_info.h> -#include <fluent-bit/flb_http_server.h> - -int api_v2_reload(struct flb_hs *hs); - -#endif diff --git a/fluent-bit/src/http_server/flb_hs.c b/fluent-bit/src/http_server/flb_hs.c deleted file mode 100644 index 40cc8467a..000000000 --- a/fluent-bit/src/http_server/flb_hs.c +++ /dev/null @@ -1,147 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2019-2020 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <fluent-bit/flb_info.h> -#include <fluent-bit/flb_mem.h> -#include <fluent-bit/flb_log.h> -#include <fluent-bit/flb_utils.h> -#include <fluent-bit/flb_http_server.h> - -#include <monkey/mk_lib.h> - -/* v1 */ -#include "api/v1/register.h" -#include "api/v1/health.h" - -/* v2 */ -#include "api/v2/register.h" - -static void cb_root(mk_request_t *request, void *data) -{ - struct flb_hs *hs = data; - - mk_http_status(request, 200); - flb_hs_add_content_type_to_req(request, FLB_HS_CONTENT_TYPE_JSON); - mk_http_send(request, hs->ep_root_buf, hs->ep_root_size, NULL); - mk_http_done(request); -} - -/* Ingest health metrics into the web service context */ -int flb_hs_push_health_metrics(struct flb_hs *hs, void *data, size_t size) -{ - return mk_mq_send(hs->ctx, hs->qid_health, data, size); -} - -/* Ingest pipeline metrics into the web service context */ -int flb_hs_push_pipeline_metrics(struct flb_hs *hs, void *data, size_t size) -{ - return mk_mq_send(hs->ctx, hs->qid_metrics, data, size); -} - -/* Ingest pipeline metrics into the web service context */ -int flb_hs_push_metrics(struct flb_hs *hs, void *data, size_t size) -{ - return mk_mq_send(hs->ctx, hs->qid_metrics_v2, data, size); -} - -/* Ingest storage metrics into the web service context */ -int flb_hs_push_storage_metrics(struct flb_hs *hs, void *data, size_t size) -{ - return mk_mq_send(hs->ctx, hs->qid_storage, data, size); -} - -/* Create ROOT endpoints */ -struct flb_hs *flb_hs_create(const char *listen, const char *tcp_port, - struct flb_config *config) -{ - int vid; - char tmp[32]; - struct flb_hs *hs; - - hs = flb_calloc(1, sizeof(struct flb_hs)); - if (!hs) { - flb_errno(); - return NULL; - } - hs->config = config; - - /* Setup endpoint specific data */ - flb_hs_endpoints(hs); - - /* Create HTTP server context */ - hs->ctx = mk_create(); - if (!hs->ctx) { - flb_error("[http_server] could not create context"); - flb_free(hs); - return NULL; - } - - /* Compose listen address */ - snprintf(tmp, sizeof(tmp) -1, "%s:%s", listen, tcp_port); - mk_config_set(hs->ctx, "Listen", tmp, NULL); - vid = mk_vhost_create(hs->ctx, NULL); - hs->vid = vid; - - /* Setup virtual host */ - mk_vhost_set(hs->ctx, vid, - "Name", "fluent-bit", - NULL); - - - /* Register endpoints for /api/v1 */ - api_v1_registration(hs); - - /* Register endpoints for /api/v2 */ - api_v2_registration(hs); - - /* Root */ - mk_vhost_handler(hs->ctx, vid, "/", cb_root, hs); - - return hs; -} - -int flb_hs_start(struct flb_hs *hs) -{ - int ret; - struct flb_config *config = hs->config; - - ret = mk_start(hs->ctx); - - if (ret == 0) { - flb_info("[http_server] listen iface=%s tcp_port=%s", - config->http_listen, config->http_port); - } - return ret; -} - -int flb_hs_destroy(struct flb_hs *hs) -{ - if (!hs) { - return 0; - } - flb_hs_health_destroy(); - mk_stop(hs->ctx); - mk_destroy(hs->ctx); - - flb_hs_endpoints_free(hs); - flb_free(hs); - - - return 0; -} diff --git a/fluent-bit/src/http_server/flb_hs_endpoints.c b/fluent-bit/src/http_server/flb_hs_endpoints.c deleted file mode 100644 index 2ea12a981..000000000 --- a/fluent-bit/src/http_server/flb_hs_endpoints.c +++ /dev/null @@ -1,121 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2019-2020 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <fluent-bit/flb_info.h> -#include <fluent-bit/flb_mem.h> -#include <fluent-bit/flb_pack.h> -#include <fluent-bit/flb_utils.h> -#include <fluent-bit/flb_version.h> -#include <fluent-bit/flb_http_server.h> -#include <msgpack.h> - -/* Create a JSON buffer with informational data about the running service */ -static int endpoint_root(struct flb_hs *hs) -{ - int c; - flb_sds_t out_buf; - msgpack_packer mp_pck; - msgpack_sbuffer mp_sbuf; - struct mk_list *head; - struct mk_list *list; - struct flb_split_entry *entry; - - /* initialize buffers */ - msgpack_sbuffer_init(&mp_sbuf); - msgpack_packer_init(&mp_pck, &mp_sbuf, msgpack_sbuffer_write); - - msgpack_pack_map(&mp_pck, 1); - msgpack_pack_str(&mp_pck, 10); - msgpack_pack_str_body(&mp_pck, "fluent-bit", 10); - - /* entries under fluent-bit parent: - * - * - version - * - edition - * - built flags - */ - msgpack_pack_map(&mp_pck, 3); - - /* fluent-bit['version'] */ - msgpack_pack_str(&mp_pck, 7); - msgpack_pack_str_body(&mp_pck, "version", 7); - msgpack_pack_str(&mp_pck, sizeof(FLB_VERSION_STR) - 1); - msgpack_pack_str_body(&mp_pck, FLB_VERSION_STR, sizeof(FLB_VERSION_STR) - 1); - - /* fluent-bit['edition'] */ - msgpack_pack_str(&mp_pck, 7); - msgpack_pack_str_body(&mp_pck, "edition", 7); -#ifdef FLB_ENTERPRISE - msgpack_pack_str(&mp_pck, 10); - msgpack_pack_str_body(&mp_pck, "Enterprise", 10); -#else - msgpack_pack_str(&mp_pck, 9); - msgpack_pack_str_body(&mp_pck, "Community", 9); -#endif - - /* fluent-bit['flags'] */ - msgpack_pack_str(&mp_pck, 5); - msgpack_pack_str_body(&mp_pck, "flags", 5); - - c = 0; - list = flb_utils_split(FLB_INFO_FLAGS, ' ', -1); - mk_list_foreach(head, list) { - entry = mk_list_entry(head, struct flb_split_entry, _head); - if (strncmp(entry->value, "FLB_", 4) == 0) { - c++; - } - } - - msgpack_pack_array(&mp_pck, c); - mk_list_foreach(head, list) { - entry = mk_list_entry(head, struct flb_split_entry, _head); - if (strncmp(entry->value, "FLB_", 4) == 0) { - msgpack_pack_str(&mp_pck, entry->len); - msgpack_pack_str_body(&mp_pck, entry->value, entry->len); - } - } - flb_utils_split_free(list); - - /* export as JSON */ - out_buf = flb_msgpack_raw_to_json_sds(mp_sbuf.data, mp_sbuf.size); - msgpack_sbuffer_destroy(&mp_sbuf); - - if (out_buf) { - hs->ep_root_buf = out_buf; - hs->ep_root_size = flb_sds_len(out_buf); - } - - return -1; -} - -int flb_hs_endpoints(struct flb_hs *hs) -{ - endpoint_root(hs); - return 0; -} - -/* Release cached data from endpoints */ -int flb_hs_endpoints_free(struct flb_hs *hs) -{ - if (hs->ep_root_buf) { - flb_sds_destroy(hs->ep_root_buf); - } - - return 0; -} diff --git a/fluent-bit/src/http_server/flb_hs_utils.c b/fluent-bit/src/http_server/flb_hs_utils.c deleted file mode 100644 index 7b39bff28..000000000 --- a/fluent-bit/src/http_server/flb_hs_utils.c +++ /dev/null @@ -1,48 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -#include <fluent-bit/flb_log.h> -#include <fluent-bit/flb_http_server.h> -#include <monkey/mk_lib.h> - -int flb_hs_add_content_type_to_req(mk_request_t *request, int type) -{ - if (request == NULL) { - return -1; - } - - switch (type) { - case FLB_HS_CONTENT_TYPE_JSON: - mk_http_header(request, - FLB_HS_CONTENT_TYPE_KEY_STR, FLB_HS_CONTENT_TYPE_KEY_LEN, - FLB_HS_CONTENT_TYPE_JSON_STR, FLB_HS_CONTENT_TYPE_JSON_LEN); - break; - case FLB_HS_CONTENT_TYPE_PROMETHEUS: - mk_http_header(request, - FLB_HS_CONTENT_TYPE_KEY_STR, FLB_HS_CONTENT_TYPE_KEY_LEN, - FLB_HS_CONTENT_TYPE_PROMETHEUS_STR, FLB_HS_CONTENT_TYPE_PROMETHEUS_LEN); - break; - default: - flb_error("[%s] unknown type=%d", __FUNCTION__, type); - return -1; - } - - return 0; -} diff --git a/fluent-bit/src/multiline/CMakeLists.txt b/fluent-bit/src/multiline/CMakeLists.txt deleted file mode 100644 index 294ef3e8f..000000000 --- a/fluent-bit/src/multiline/CMakeLists.txt +++ /dev/null @@ -1,15 +0,0 @@ -set(src_multiline - # built-in parsers - multiline/flb_ml_parser_cri.c - multiline/flb_ml_parser_docker.c - multiline/flb_ml_parser_python.c - multiline/flb_ml_parser_java.c - multiline/flb_ml_parser_go.c - multiline/flb_ml_parser_ruby.c - # core - multiline/flb_ml_stream.c - multiline/flb_ml_parser.c - multiline/flb_ml_group.c - multiline/flb_ml_rule.c - multiline/flb_ml.c PARENT_SCOPE - ) diff --git a/fluent-bit/src/multiline/flb_ml.c b/fluent-bit/src/multiline/flb_ml.c deleted file mode 100644 index 28e123cee..000000000 --- a/fluent-bit/src/multiline/flb_ml.c +++ /dev/null @@ -1,1562 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <fluent-bit/flb_info.h> -#include <fluent-bit/flb_mem.h> -#include <fluent-bit/flb_log.h> -#include <fluent-bit/flb_regex.h> -#include <fluent-bit/flb_time.h> -#include <fluent-bit/flb_pack.h> -#include <fluent-bit/flb_scheduler.h> -#include <fluent-bit/multiline/flb_ml.h> -#include <fluent-bit/multiline/flb_ml_rule.h> -#include <fluent-bit/multiline/flb_ml_group.h> - -#include <stdarg.h> -#include <math.h> - -static inline int match_negate(struct flb_ml_parser *ml_parser, int matched) -{ - int rule_match = matched; - - /* Validate pattern matching against expected 'negate' condition */ - if (matched == FLB_TRUE) { - if (ml_parser->negate == FLB_FALSE) { - rule_match = FLB_TRUE; - } - else { - rule_match = FLB_FALSE; - } - } - else { - if (ml_parser->negate == FLB_TRUE) { - rule_match = FLB_TRUE; - } - } - - return rule_match; -} - -static uint64_t time_ms_now() -{ - uint64_t ms; - struct flb_time tm; - - flb_time_get(&tm); - ms = (tm.tm.tv_sec * 1000) + lround(tm.tm.tv_nsec/1.0e6); - return ms; -} - - -int flb_ml_flush_stdout(struct flb_ml_parser *parser, - struct flb_ml_stream *mst, - void *data, char *buf_data, size_t buf_size) -{ - fprintf(stdout, "\n%s----- MULTILINE FLUSH (stream_id=%" PRIu64 ") -----%s\n", - ANSI_GREEN, mst->id, ANSI_RESET); - - /* Print incoming flush buffer */ - flb_pack_print(buf_data, buf_size); - - fprintf(stdout, "%s----------- EOF -----------%s\n", - ANSI_GREEN, ANSI_RESET); - return 0; -} - -int flb_ml_type_lookup(char *str) -{ - int type = -1; - - if (strcasecmp(str, "regex") == 0) { - type = FLB_ML_REGEX; - } - else if (strcasecmp(str, "endswith") == 0) { - type = FLB_ML_ENDSWITH; - } - else if (strcasecmp(str, "equal") == 0 || strcasecmp(str, "eq") == 0) { - type = FLB_ML_EQ; - } - - return type; -} - -void flb_ml_flush_parser_instance(struct flb_ml *ml, - struct flb_ml_parser_ins *parser_i, - uint64_t stream_id, int forced_flush) -{ - struct mk_list *head; - struct mk_list *head_group; - struct flb_ml_stream *mst; - struct flb_ml_stream_group *group; - - mk_list_foreach(head, &parser_i->streams) { - mst = mk_list_entry(head, struct flb_ml_stream, _head); - if (stream_id != 0 && mst->id != stream_id) { - continue; - } - - /* Iterate stream groups */ - mk_list_foreach(head_group, &mst->groups) { - group = mk_list_entry(head_group, struct flb_ml_stream_group, _head); - flb_ml_flush_stream_group(parser_i->ml_parser, mst, group, forced_flush); - } - } -} - -void flb_ml_flush_pending(struct flb_ml *ml, uint64_t now, int forced_flush) -{ - struct mk_list *head; - struct flb_ml_parser_ins *parser_i; - struct flb_ml_group *group; - - /* set the last flush time */ - ml->last_flush = now; - - /* flush only the first group of the context */ - group = mk_list_entry_first(&ml->groups, struct flb_ml_group, _head); - - /* iterate group parser instances */ - mk_list_foreach(head, &group->parsers) { - parser_i = mk_list_entry(head, struct flb_ml_parser_ins, _head); - flb_ml_flush_parser_instance(ml, parser_i, 0, forced_flush); - } -} - -void flb_ml_flush_pending_now(struct flb_ml *ml) -{ - uint64_t now; - - now = time_ms_now(); - flb_ml_flush_pending(ml, now, FLB_TRUE); -} - -static void cb_ml_flush_timer(struct flb_config *ctx, void *data) -{ - uint64_t now; - struct flb_ml *ml = data; - - now = time_ms_now(); - if (ml->last_flush + ml->flush_ms > now) { - return; - } - - /* - * Iterate over all streams and groups and for a flush for expired groups - * which has not flushed in the last N milliseconds. - */ - flb_ml_flush_pending(ml, now, FLB_TRUE); -} - -int flb_ml_register_context(struct flb_ml_stream_group *group, - struct flb_time *tm, msgpack_object *map) -{ - if (tm) { - flb_time_copy(&group->mp_time, tm); - } - - if (map) { - msgpack_pack_object(&group->mp_pck, *map); - } - - return 0; -} - -static inline void breakline_prepare(struct flb_ml_parser_ins *parser_i, - struct flb_ml_stream_group *stream_group) -{ - int len; - - if (parser_i->key_content) { - return; - } - - len = flb_sds_len(stream_group->buf); - if (len <= 0) { - return; - } - - if (stream_group->buf[len - 1] != '\n') { - flb_sds_cat_safe(&stream_group->buf, "\n", 1); - } -} - -/* - * package content into a multiline stream: - * - * full_map: if the original content to process comes in msgpack map, this variable - * reference the map. It's only used in case we will package a first line so we - * store a copy of the other key values in the map for flush time. - */ -static int package_content(struct flb_ml_stream *mst, - msgpack_object *metadata, - msgpack_object *full_map, - void *buf, size_t size, struct flb_time *tm, - msgpack_object *val_content, - msgpack_object *val_pattern, - msgpack_object *val_group) -{ - int len; - int ret; - int rule_match = FLB_FALSE; - int processed = FLB_FALSE; - int type; - size_t offset = 0; - size_t buf_size; - char *buf_data; - msgpack_object *val = val_content; - struct flb_ml_parser *parser; - struct flb_ml_parser_ins *parser_i; - struct flb_ml_stream_group *stream_group; - - parser_i = mst->parser; - parser = parser_i->ml_parser; - - /* Get stream group */ - stream_group = flb_ml_stream_group_get(mst->parser, mst, val_group); - if (!mst->last_stream_group) { - mst->last_stream_group = stream_group; - } - else { - if (mst->last_stream_group != stream_group) { - mst->last_stream_group = stream_group; - } - } - - /* Set the parser type */ - type = parser->type; - - if (val_pattern) { - val = val_pattern; - } - - if (val) { - buf_data = (char *) val->via.str.ptr; - buf_size = val->via.str.size; - } - else { - buf_data = buf; - buf_size = size; - - } - if (type == FLB_ML_REGEX) { - ret = flb_ml_rule_process(parser, mst, - stream_group, full_map, buf, size, tm, - val_content, val_pattern); - if (ret == -1) { - processed = FLB_FALSE; - } - else { - processed = FLB_TRUE; - } - } - else if (type == FLB_ML_ENDSWITH) { - len = flb_sds_len(parser->match_str); - if (buf_data && len <= buf_size) { - /* Validate if content ends with expected string */ - offset = buf_size - len; - ret = memcmp(buf_data + offset, parser->match_str, len); - if (ret == 0) { - rule_match = match_negate(parser, FLB_TRUE); - } - else { - rule_match = match_negate(parser, FLB_FALSE); - } - - if (stream_group->mp_sbuf.size == 0) { - flb_ml_register_context(stream_group, tm, full_map); - } - - /* Prepare concatenation */ - breakline_prepare(parser_i, stream_group); - - /* Concatenate value */ - if (val_content) { - flb_sds_cat_safe(&stream_group->buf, - val_content->via.str.ptr, - val_content->via.str.size); - } - else { - flb_sds_cat_safe(&stream_group->buf, buf_data, buf_size); - } - - /* on ENDSWITH mode, a rule match means flush the content */ - if (rule_match) { - flb_ml_flush_stream_group(parser, mst, stream_group, FLB_FALSE); - } - processed = FLB_TRUE; - } - } - else if (type == FLB_ML_EQ) { - if (buf_size == flb_sds_len(parser->match_str) && - memcmp(buf_data, parser->match_str, buf_size) == 0) { - /* EQ match */ - rule_match = match_negate(parser, FLB_TRUE); - } - else { - rule_match = match_negate(parser, FLB_FALSE); - } - - if (stream_group->mp_sbuf.size == 0) { - flb_ml_register_context(stream_group, tm, full_map); - } - - /* Prepare concatenation */ - breakline_prepare(parser_i, stream_group); - - /* Concatenate value */ - if (val_content) { - flb_sds_cat_safe(&stream_group->buf, - val_content->via.str.ptr, - val_content->via.str.size); - } - else { - flb_sds_cat_safe(&stream_group->buf, buf_data, buf_size); - } - - /* on ENDSWITH mode, a rule match means flush the content */ - if (rule_match) { - flb_ml_flush_stream_group(parser, mst, stream_group, FLB_FALSE); - } - processed = FLB_TRUE; - } - - if (processed && metadata != NULL) { - msgpack_pack_object(&stream_group->mp_md_pck, *metadata); - } - - return processed; -} - -/* - * Retrieve the ID of a specific key name in a map. This function might be - * extended later to use record accessor, since all current cases are solved - * now quering the first level of keys in the map, we avoid record accessor - * to avoid extra memory allocations. - */ -static int get_key_id(msgpack_object *map, flb_sds_t key_name) -{ - int i; - int len; - int found = FLB_FALSE; - msgpack_object key; - msgpack_object val; - - if (!key_name) { - return -1; - } - - len = flb_sds_len(key_name); - for (i = 0; i < map->via.map.size; i++) { - key = map->via.map.ptr[i].key; - val = map->via.map.ptr[i].val; - - if (key.type != MSGPACK_OBJECT_STR || val.type != MSGPACK_OBJECT_STR) { - continue; - } - - if (key.via.str.size != len) { - continue; - } - - if (strncmp(key.via.str.ptr, key_name, len) == 0) { - found = FLB_TRUE; - break; - } - } - - if (found) { - return i; - } - - return -1; -} - -static int process_append(struct flb_ml_parser_ins *parser_i, - struct flb_ml_stream *mst, - int type, - struct flb_time *tm, - msgpack_object *metadata, - msgpack_object *obj, - void *buf, size_t size) -{ - int ret; - int id_content = -1; - int id_pattern = -1; - int id_group = -1; - int unpacked = FLB_FALSE; - size_t off = 0; - msgpack_object *full_map = NULL; - msgpack_object *val_content = NULL; - msgpack_object *val_pattern = NULL; - msgpack_object *val_group = NULL; - msgpack_unpacked result; - - /* Lookup the key */ - if (type == FLB_ML_TYPE_TEXT) { - ret = package_content(mst, NULL, NULL, buf, size, tm, NULL, NULL, NULL); - if (ret == FLB_FALSE) { - return -1; - } - return 0; - } - else if (type == FLB_ML_TYPE_MAP) { - full_map = obj; - /* - * When full_map and buf is not NULL, - * we use 'buf' since buf is already processed from full_map at - * ml_append_try_parser_type_map. - */ - if (!full_map || (buf != NULL && full_map != NULL)) { - off = 0; - msgpack_unpacked_init(&result); - ret = msgpack_unpack_next(&result, buf, size, &off); - if (ret != MSGPACK_UNPACK_SUCCESS) { - return -1; - } - full_map = &result.data; - unpacked = FLB_TRUE; - } - else if (full_map->type != MSGPACK_OBJECT_MAP) { - msgpack_unpacked_destroy(&result); - return -1; - } - } - - /* Lookup for key_content entry */ - id_content = get_key_id(full_map, parser_i->key_content); - if (id_content == -1) { - if (unpacked) { - msgpack_unpacked_destroy(&result); - } - return -1; - } - - val_content = &full_map->via.map.ptr[id_content].val; - if (val_content->type != MSGPACK_OBJECT_STR) { - val_content = NULL; - } - - /* Optional: Lookup for key_pattern entry */ - if (parser_i->key_pattern) { - id_pattern = get_key_id(full_map, parser_i->key_pattern); - if (id_pattern >= 0) { - val_pattern = &full_map->via.map.ptr[id_pattern].val; - if (val_pattern->type != MSGPACK_OBJECT_STR) { - val_pattern = NULL; - } - } - } - - /* Optional: lookup for key_group entry */ - if (parser_i->key_group) { - id_group = get_key_id(full_map, parser_i->key_group); - if (id_group >= 0) { - val_group = &full_map->via.map.ptr[id_group].val; - if (val_group->type != MSGPACK_OBJECT_STR) { - val_group = NULL; - } - } - } - - /* Package the content */ - ret = package_content(mst, metadata, full_map, buf, size, tm, - val_content, val_pattern, val_group); - if (unpacked) { - msgpack_unpacked_destroy(&result); - } - if (ret == FLB_FALSE) { - return -1; - } - return 0; -} - -static int ml_append_try_parser_type_text(struct flb_ml_parser_ins *parser, - uint64_t stream_id, - int *type, - struct flb_time *tm, void *buf, size_t size, - msgpack_object *map, - void **out_buf, size_t *out_size, int *out_release, - struct flb_time *out_time) -{ - int ret; - - if (parser->ml_parser->parser) { - /* Parse incoming content */ - ret = flb_parser_do(parser->ml_parser->parser, (char *) buf, size, - out_buf, out_size, out_time); - if (flb_time_to_nanosec(out_time) == 0L) { - flb_time_copy(out_time, tm); - } - if (ret >= 0) { - *out_release = FLB_TRUE; - *type = FLB_ML_TYPE_MAP; - } - else { - *out_buf = buf; - *out_size = size; - return -1; - } - } - else { - *out_buf = buf; - *out_size = size; - } - return 0; -} - -static int ml_append_try_parser_type_map(struct flb_ml_parser_ins *parser, - uint64_t stream_id, - int *type, - struct flb_time *tm, void *buf, size_t size, - msgpack_object *map, - void **out_buf, size_t *out_size, int *out_release, - struct flb_time *out_time) -{ - int map_size; - int i; - int len; - msgpack_object key; - msgpack_object val; - - if (map == NULL || map->type != MSGPACK_OBJECT_MAP) { - flb_error("%s:invalid map", __FUNCTION__); - return -1; - } - - if (parser->ml_parser->parser) { - /* lookup key_content */ - - len = flb_sds_len(parser->key_content); - map_size = map->via.map.size; - for(i = 0; i < map_size; i++) { - key = map->via.map.ptr[i].key; - val = map->via.map.ptr[i].val; - if (key.type == MSGPACK_OBJECT_STR && - parser->key_content && - key.via.str.size == len && - strncmp(key.via.str.ptr, parser->key_content, len) == 0) { - /* key_content found */ - if (val.type == MSGPACK_OBJECT_STR) { - /* try parse the value of key_content e*/ - return ml_append_try_parser_type_text(parser, stream_id, type, - tm, (void*) val.via.str.ptr, - val.via.str.size, - map, - out_buf, out_size, out_release, - out_time); - } else { - flb_error("%s: not string", __FUNCTION__); - return -1; - } - } - } - } - else { - *out_buf = buf; - *out_size = size; - } - return 0; -} - -static int ml_append_try_parser(struct flb_ml_parser_ins *parser, - uint64_t stream_id, - int type, - struct flb_time *tm, void *buf, size_t size, - msgpack_object *metadata, - msgpack_object *map) -{ - int ret; - int release = FLB_FALSE; - void *out_buf = NULL; - size_t out_size = 0; - struct flb_time out_time; - struct flb_ml_stream *mst; - - flb_time_zero(&out_time); - - switch (type) { - case FLB_ML_TYPE_TEXT: - ret = ml_append_try_parser_type_text(parser, stream_id, &type, - tm, buf, size, map, - &out_buf, &out_size, &release, - &out_time); - if (ret < 0) { - return -1; - } - break; - case FLB_ML_TYPE_MAP: - ret = ml_append_try_parser_type_map(parser, stream_id, &type, - tm, buf, size, map, - &out_buf, &out_size, &release, - &out_time); - if (ret < 0) { - return -1; - } - break; - - default: - flb_error("[multiline] unknown type=%d", type); - return -1; - } - - if (flb_time_to_nanosec(&out_time) == 0L) { - if (tm && flb_time_to_nanosec(tm) != 0L) { - flb_time_copy(&out_time, tm); - } - else { - flb_time_get(&out_time); - } - } - - /* Get the stream */ - mst = flb_ml_stream_get(parser, stream_id); - if (!mst) { - flb_error("[multiline] invalid stream_id %" PRIu64 ", could not " - "append content to multiline context", stream_id); - goto exit; - } - - /* Process the binary record */ - ret = process_append(parser, mst, type, &out_time, metadata, map, out_buf, out_size); - if (ret == -1) { - if (release == FLB_TRUE) { - flb_free(out_buf); - } - return -1; - } - - exit: - if (release == FLB_TRUE) { - flb_free(out_buf); - } - - return 0; -} - -int flb_ml_append_text(struct flb_ml *ml, uint64_t stream_id, - struct flb_time *tm, void *buf, size_t size) -{ - int ret; - int processed = FLB_FALSE; - struct mk_list *head; - struct mk_list *head_group; - struct flb_ml_group *group; - struct flb_ml_stream *mst; - struct flb_ml_parser_ins *lru_parser = NULL; - struct flb_ml_parser_ins *parser_i; - struct flb_time out_time; - struct flb_ml_stream_group *st_group; - int type; - - type = FLB_ML_TYPE_TEXT; - - flb_time_zero(&out_time); - - mk_list_foreach(head, &ml->groups) { - group = mk_list_entry(head, struct flb_ml_group, _head); - - /* Check if the incoming data matches the last recently used parser */ - lru_parser = group->lru_parser; - - if (lru_parser && lru_parser->last_stream_id == stream_id) { - ret = ml_append_try_parser(lru_parser, lru_parser->last_stream_id, type, - tm, buf, size, NULL, NULL); - if (ret == 0) { - processed = FLB_TRUE; - break; - } - else { - flb_ml_flush_parser_instance(ml, - lru_parser, - lru_parser->last_stream_id, - FLB_FALSE); - } - } - else if (lru_parser && lru_parser->last_stream_id > 0) { - /* - * Clear last recently used parser to match new parser. - * Do not flush last_stream_id since it should continue to parsing. - */ - lru_parser = NULL; - } - } - - mk_list_foreach(head_group, &group->parsers) { - parser_i = mk_list_entry(head_group, struct flb_ml_parser_ins, _head); - if (lru_parser && lru_parser == parser_i && - lru_parser->last_stream_id == stream_id) { - continue; - } - - ret = ml_append_try_parser(parser_i, stream_id, type, - tm, buf, size, NULL, NULL); - if (ret == 0) { - group->lru_parser = parser_i; - group->lru_parser->last_stream_id = stream_id; - lru_parser = parser_i; - processed = FLB_TRUE; - break; - } - else { - parser_i = NULL; - } - } - - if (!processed) { - if (lru_parser) { - flb_ml_flush_parser_instance(ml, lru_parser, stream_id, FLB_FALSE); - parser_i = lru_parser; - } - else { - /* get the first parser (just to make use of it buffers) */ - parser_i = mk_list_entry_first(&group->parsers, - struct flb_ml_parser_ins, - _head); - } - - flb_ml_flush_parser_instance(ml, parser_i, stream_id, FLB_FALSE); - mst = flb_ml_stream_get(parser_i, stream_id); - if (!mst) { - flb_error("[multiline] invalid stream_id %" PRIu64 ", could not " - "append content to multiline context", stream_id); - return -1; - } - - /* Get stream group */ - st_group = flb_ml_stream_group_get(mst->parser, mst, NULL); - flb_sds_cat_safe(&st_group->buf, buf, size); - flb_ml_flush_stream_group(parser_i->ml_parser, mst, st_group, FLB_FALSE); - } - - return 0; -} - - - -int flb_ml_append_object(struct flb_ml *ml, - uint64_t stream_id, - struct flb_time *tm, - msgpack_object *metadata, - msgpack_object *obj) -{ - int ret; - int type; - int processed = FLB_FALSE; - struct mk_list *head; - struct mk_list *head_group; - struct flb_ml_group *group; - struct flb_ml_parser_ins *lru_parser = NULL; - struct flb_ml_parser_ins *parser_i; - struct flb_ml_stream *mst; - struct flb_ml_stream_group *st_group; - struct flb_log_event event; - - if (metadata == NULL) { - metadata = ml->log_event_decoder.empty_map; - } - - /* - * As incoming objects, we accept packed events - * and msgpack Maps containing key/value pairs. - */ - if (obj->type == MSGPACK_OBJECT_ARRAY) { - flb_error("[multiline] appending object with invalid type, expected " - "map, received type=%i", obj->type); - return -1; - - - flb_log_event_decoder_reset(&ml->log_event_decoder, NULL, 0); - - ret = flb_event_decoder_decode_object(&ml->log_event_decoder, - &event, - obj); - - if (ret != FLB_EVENT_DECODER_SUCCESS) { - flb_error("[multiline] invalid event object"); - - return -1; - } - - tm = &event.timestamp; - obj = event.body; - metadata = event.metadata; - - type = FLB_ML_TYPE_MAP; - } - else if (obj->type == MSGPACK_OBJECT_MAP) { - type = FLB_ML_TYPE_MAP; - } - else { - flb_error("[multiline] appending object with invalid type, expected " - "array or map, received type=%i", obj->type); - return -1; - } - - mk_list_foreach(head, &ml->groups) { - group = mk_list_entry(head, struct flb_ml_group, _head); - - /* Check if the incoming data matches the last recently used parser */ - lru_parser = group->lru_parser; - - if (lru_parser && lru_parser->last_stream_id == stream_id) { - ret = ml_append_try_parser(lru_parser, lru_parser->last_stream_id, type, - tm, NULL, 0, metadata, obj); - if (ret == 0) { - processed = FLB_TRUE; - break; - } - else { - flb_ml_flush_parser_instance(ml, - lru_parser, - lru_parser->last_stream_id, - FLB_FALSE); - } - } - else if (lru_parser && lru_parser->last_stream_id > 0) { - /* - * Clear last recently used parser to match new parser. - * Do not flush last_stream_id since it should continue to parsing. - */ - lru_parser = NULL; - } - } - - mk_list_foreach(head_group, &group->parsers) { - parser_i = mk_list_entry(head_group, struct flb_ml_parser_ins, _head); - if (lru_parser && parser_i == lru_parser) { - continue; - } - - ret = ml_append_try_parser(parser_i, stream_id, type, - tm, NULL, 0, metadata, obj); - if (ret == 0) { - group->lru_parser = parser_i; - group->lru_parser->last_stream_id = stream_id; - lru_parser = parser_i; - processed = FLB_TRUE; - break; - } - else { - parser_i = NULL; - } - } - - if (!processed) { - if (lru_parser) { - flb_ml_flush_parser_instance(ml, lru_parser, stream_id, FLB_FALSE); - parser_i = lru_parser; - } - else { - /* get the first parser (just to make use of it buffers) */ - parser_i = mk_list_entry_first(&group->parsers, - struct flb_ml_parser_ins, - _head); - } - - flb_ml_flush_parser_instance(ml, parser_i, stream_id, FLB_FALSE); - mst = flb_ml_stream_get(parser_i, stream_id); - if (!mst) { - flb_error("[multiline] invalid stream_id %" PRIu64 ", could not " - "append content to multiline context", stream_id); - - return -1; - } - - /* Get stream group */ - st_group = flb_ml_stream_group_get(mst->parser, mst, NULL); - - ret = flb_log_event_encoder_begin_record(&ml->log_event_encoder); - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_set_timestamp( - &ml->log_event_encoder, tm); - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - if (metadata != ml->log_event_decoder.empty_map) { - ret = flb_log_event_encoder_set_metadata_from_msgpack_object( - &ml->log_event_encoder, metadata); - } - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_set_body_from_msgpack_object( - &ml->log_event_encoder, obj); - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_commit_record(&ml->log_event_encoder); - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - mst->cb_flush(parser_i->ml_parser, - mst, - mst->cb_data, - ml->log_event_encoder.output_buffer, - ml->log_event_encoder.output_length); - } - else { - flb_error("[multiline] log event encoder error : %d", ret); - } - - flb_log_event_encoder_reset(&ml->log_event_encoder); - - /* reset group buffer counters */ - st_group->mp_sbuf.size = 0; - flb_sds_len_set(st_group->buf, 0); - - /* Update last flush time */ - st_group->last_flush = time_ms_now(); - } - - return 0; -} - -int flb_ml_append_event(struct flb_ml *ml, uint64_t stream_id, - struct flb_log_event *event) -{ - return flb_ml_append_object(ml, - stream_id, - &event->timestamp, - event->metadata, - event->body); -} - - -struct flb_ml *flb_ml_create(struct flb_config *ctx, char *name) -{ - int result; - struct flb_ml *ml; - - ml = flb_calloc(1, sizeof(struct flb_ml)); - if (!ml) { - flb_errno(); - return NULL; - } - ml->name = flb_sds_create(name); - if (!ml) { - flb_free(ml); - return NULL; - } - - ml->config = ctx; - ml->last_flush = time_ms_now(); - mk_list_init(&ml->groups); - - result = flb_log_event_decoder_init(&ml->log_event_decoder, - NULL, - 0); - - if (result != FLB_EVENT_DECODER_SUCCESS) { - flb_error("cannot initialize log event decoder"); - - flb_ml_destroy(ml); - - return NULL; - } - - result = flb_log_event_encoder_init(&ml->log_event_encoder, - FLB_LOG_EVENT_FORMAT_DEFAULT); - - if (result != FLB_EVENT_ENCODER_SUCCESS) { - flb_error("cannot initialize log event encoder"); - - flb_ml_destroy(ml); - - return NULL; - } - - return ml; -} - -/* - * Some multiline contexts might define a parser name but not a parser context, - * for missing contexts, just lookup the parser and perform the assignment. - * - * The common use case is when reading config files with [PARSER] and [MULTILINE_PARSER] - * definitions, so we need to delay the parser loading. - */ -int flb_ml_parsers_init(struct flb_config *ctx) -{ - struct mk_list *head; - struct flb_parser *p; - struct flb_ml_parser *ml_parser; - - mk_list_foreach(head, &ctx->multiline_parsers) { - ml_parser = mk_list_entry(head, struct flb_ml_parser, _head); - if (ml_parser->parser_name && !ml_parser->parser) { - p = flb_parser_get(ml_parser->parser_name, ctx); - if (!p) { - flb_error("multiline parser '%s' points to an undefined parser '%s'", - ml_parser->name, ml_parser->parser_name); - return -1; - } - ml_parser->parser = p; - } - } - - return 0; -} - -int flb_ml_auto_flush_init(struct flb_ml *ml) -{ - struct flb_sched *scheduler; - int ret; - - if (ml == NULL) { - return -1; - } - - scheduler = flb_sched_ctx_get(); - - if (scheduler == NULL) { - flb_error("[multiline] scheduler context has not been created"); - return -1; - } - - if (ml->flush_ms < 500) { - flb_error("[multiline] flush timeout '%i' is too low", ml->flush_ms); - return -1; - } - - /* Create flush timer */ - ret = flb_sched_timer_cb_create(scheduler, - FLB_SCHED_TIMER_CB_PERM, - ml->flush_ms, - cb_ml_flush_timer, - ml, NULL); - return ret; -} - -int flb_ml_destroy(struct flb_ml *ml) -{ - struct mk_list *head; - struct mk_list *tmp; - struct flb_ml_group *group; - - if (!ml) { - return 0; - } - - flb_log_event_decoder_destroy(&ml->log_event_decoder); - flb_log_event_encoder_destroy(&ml->log_event_encoder); - - if (ml->name) { - flb_sds_destroy(ml->name); - } - - /* destroy groups */ - mk_list_foreach_safe(head, tmp, &ml->groups) { - group = mk_list_entry(head, struct flb_ml_group, _head); - flb_ml_group_destroy(group); - } - - flb_free(ml); - return 0; -} - -static int flb_msgpack_object_hash_internal(cfl_hash_state_t *state, - msgpack_object *object) -{ - void *dummy_pointer; - int result; - int index; - - if (object == NULL) { - return 0; - } - - dummy_pointer = NULL; - result = 0; - - if (object->type == MSGPACK_OBJECT_NIL) { - cfl_hash_64bits_update(state, - &dummy_pointer, - sizeof(dummy_pointer)); - } - else if (object->type == MSGPACK_OBJECT_BOOLEAN) { - cfl_hash_64bits_update(state, - &object->via.boolean, - sizeof(object->via.boolean)); - } - else if (object->type == MSGPACK_OBJECT_POSITIVE_INTEGER || - object->type == MSGPACK_OBJECT_NEGATIVE_INTEGER) { - cfl_hash_64bits_update(state, - &object->via.u64, - sizeof(object->via.u64)); - } - else if (object->type == MSGPACK_OBJECT_FLOAT32 || - object->type == MSGPACK_OBJECT_FLOAT64 || - object->type == MSGPACK_OBJECT_FLOAT) { - cfl_hash_64bits_update(state, - &object->via.f64, - sizeof(object->via.f64)); - } - else if (object->type == MSGPACK_OBJECT_STR) { - cfl_hash_64bits_update(state, - object->via.str.ptr, - object->via.str.size); - } - else if (object->type == MSGPACK_OBJECT_ARRAY) { - for (index = 0 ; - index < object->via.array.size && - result == 0; - index++) { - result = flb_msgpack_object_hash_internal( - state, - &object->via.array.ptr[index]); - } - } - else if (object->type == MSGPACK_OBJECT_MAP) { - for (index = 0 ; - index < object->via.map.size && - result == 0; - index++) { - result = flb_msgpack_object_hash_internal( - state, - &object->via.map.ptr[index].key); - - if (result == 0) { - result = flb_msgpack_object_hash_internal( - state, - &object->via.map.ptr[index].val); - } - } - } - else if (object->type == MSGPACK_OBJECT_BIN) { - cfl_hash_64bits_update(state, - object->via.bin.ptr, - object->via.bin.size); - } - else if (object->type == MSGPACK_OBJECT_EXT) { - cfl_hash_64bits_update(state, - &object->via.ext.type, - sizeof(object->via.ext.type)); - - cfl_hash_64bits_update(state, - object->via.ext.ptr, - object->via.ext.size); - } - - return result; -} - -static int flb_hash_msgpack_object_list(cfl_hash_64bits_t *hash, - size_t entry_count, - ...) -{ - cfl_hash_state_t hash_state; - va_list arguments; - msgpack_object *object; - int result; - size_t index; - - cfl_hash_64bits_reset(&hash_state); - - va_start(arguments, entry_count); - - result = 0; - - for (index = 0 ; - index < entry_count && - result == 0 ; - index++) { - object = va_arg(arguments, msgpack_object *); - - if (object == NULL) { - break; - } - - result = flb_msgpack_object_hash_internal(&hash_state, object); - } - - va_end(arguments); - - if (result == 0) { - *hash = cfl_hash_64bits_digest(&hash_state); - } - - return result; -} - -struct flb_deduplication_list_entry { - cfl_hash_64bits_t hash; - struct cfl_list _head; -}; - -void flb_deduplication_list_init(struct cfl_list *deduplication_list) -{ - cfl_list_init(deduplication_list); -} - -int flb_deduplication_list_validate(struct cfl_list *deduplication_list, - cfl_hash_64bits_t hash) -{ - struct cfl_list *iterator; - struct flb_deduplication_list_entry *entry; - - cfl_list_foreach(iterator, deduplication_list) { - entry = cfl_list_entry(iterator, - struct flb_deduplication_list_entry, - _head); - - if (entry->hash == hash) { - return FLB_TRUE; - } - } - - return FLB_FALSE; -} - -int flb_deduplication_list_add(struct cfl_list *deduplication_list, - cfl_hash_64bits_t hash) -{ - struct flb_deduplication_list_entry *entry; - - entry = (struct flb_deduplication_list_entry *) - flb_calloc(1, - sizeof(struct flb_deduplication_list_entry)); - - if (entry == NULL) { - return -1; - } - - cfl_list_entry_init(&entry->_head); - entry->hash = hash; - - cfl_list_append(&entry->_head, deduplication_list); - - return 0; -} - -void flb_deduplication_list_purge(struct cfl_list *deduplication_list) -{ - struct cfl_list *iterator; - struct cfl_list *backup; - struct flb_deduplication_list_entry *entry; - - cfl_list_foreach_safe(iterator, backup, deduplication_list) { - entry = cfl_list_entry(iterator, - struct flb_deduplication_list_entry, - _head); - - cfl_list_del(&entry->_head); - - free(entry); - } -} - -int flb_ml_flush_metadata_buffer(struct flb_ml_stream *mst, - struct flb_ml_stream_group *group, - int deduplicate_metadata) -{ - int append_metadata_entry; - cfl_hash_64bits_t metadata_entry_hash; - struct cfl_list deduplication_list; - msgpack_unpacked metadata_map; - size_t offset; - size_t index; - msgpack_object value; - msgpack_object key; - int ret; - - ret = FLB_EVENT_ENCODER_SUCCESS; - - if (deduplicate_metadata) { - flb_deduplication_list_init(&deduplication_list); - } - - msgpack_unpacked_init(&metadata_map); - - offset = 0; - while (ret == FLB_EVENT_ENCODER_SUCCESS && - msgpack_unpack_next(&metadata_map, - group->mp_md_sbuf.data, - group->mp_md_sbuf.size, - &offset) == MSGPACK_UNPACK_SUCCESS) { - - for (index = 0; - index < metadata_map.data.via.map.size && - ret == FLB_EVENT_ENCODER_SUCCESS; - index++) { - key = metadata_map.data.via.map.ptr[index].key; - value = metadata_map.data.via.map.ptr[index].val; - - append_metadata_entry = FLB_TRUE; - - if (deduplicate_metadata) { - ret = flb_hash_msgpack_object_list(&metadata_entry_hash, - 2, - &key, - &value); - if (ret != 0) { - ret = FLB_EVENT_ENCODER_ERROR_INVALID_ARGUMENT; - } - else { - ret = flb_deduplication_list_validate( - &deduplication_list, - metadata_entry_hash); - - if (ret) { - append_metadata_entry = FLB_FALSE; - - ret = FLB_EVENT_ENCODER_SUCCESS; - } - else { - ret = flb_deduplication_list_add( - &deduplication_list, - metadata_entry_hash); - - if (ret == 0) { - ret = FLB_EVENT_ENCODER_SUCCESS; - } - else { - ret = FLB_EVENT_ENCODER_ERROR_ALLOCATION_ERROR; - } - } - } - } - - if (append_metadata_entry) { - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_append_metadata_values( - &mst->ml->log_event_encoder, - FLB_LOG_EVENT_MSGPACK_OBJECT_VALUE(&key), - FLB_LOG_EVENT_MSGPACK_OBJECT_VALUE(&value)); - } - } - } - } - - msgpack_unpacked_destroy(&metadata_map); - - if (deduplicate_metadata) { - flb_deduplication_list_purge(&deduplication_list); - } - - return ret; -} - -int flb_ml_flush_stream_group(struct flb_ml_parser *ml_parser, - struct flb_ml_stream *mst, - struct flb_ml_stream_group *group, - int forced_flush) -{ - int i; - int ret; - int size; - int len; - size_t off = 0; - msgpack_object map; - msgpack_object k; - msgpack_object v; - msgpack_sbuffer mp_sbuf; - msgpack_packer mp_pck; - msgpack_unpacked result; - struct flb_ml_parser_ins *parser_i = mst->parser; - struct flb_time *group_time; - struct flb_time now; - - breakline_prepare(parser_i, group); - len = flb_sds_len(group->buf); - - /* init msgpack buffer */ - msgpack_sbuffer_init(&mp_sbuf); - msgpack_packer_init(&mp_pck, &mp_sbuf, msgpack_sbuffer_write); - - /* if the group don't have a time set, use current time */ - if (flb_time_to_nanosec(&group->mp_time) == 0L) { - flb_time_get(&now); - group_time = &now; - } else { - group_time = &group->mp_time; - } - - /* compose final record if we have a first line context */ - if (group->mp_sbuf.size > 0) { - msgpack_unpacked_init(&result); - ret = msgpack_unpack_next(&result, - group->mp_sbuf.data, group->mp_sbuf.size, - &off); - if (ret != MSGPACK_UNPACK_SUCCESS) { - flb_error("[multiline] could not unpack first line state buffer"); - msgpack_unpacked_destroy(&result); - group->mp_sbuf.size = 0; - return -1; - } - map = result.data; - - if (map.type != MSGPACK_OBJECT_MAP) { - flb_error("[multiline] expected MAP type in first line state buffer"); - msgpack_unpacked_destroy(&result); - group->mp_sbuf.size = 0; - return -1; - } - - /* Take the first line keys and repack */ - len = flb_sds_len(parser_i->key_content); - size = map.via.map.size; - msgpack_pack_map(&mp_pck, size); - - for (i = 0; i < size; i++) { - k = map.via.map.ptr[i].key; - v = map.via.map.ptr[i].val; - - /* - * Check if the current key is the key that will contain the - * concatenated multiline buffer - */ - if (k.type == MSGPACK_OBJECT_STR && - parser_i->key_content && - k.via.str.size == len && - strncmp(k.via.str.ptr, parser_i->key_content, len) == 0) { - - /* key */ - msgpack_pack_object(&mp_pck, k); - - /* value */ - len = flb_sds_len(group->buf); - msgpack_pack_str(&mp_pck, len); - msgpack_pack_str_body(&mp_pck, group->buf, len); - } - else { - /* key / val */ - msgpack_pack_object(&mp_pck, k); - msgpack_pack_object(&mp_pck, v); - } - } - msgpack_unpacked_destroy(&result); - group->mp_sbuf.size = 0; - } - else if (len > 0) { - /* Pack raw content as Fluent Bit record */ - msgpack_pack_map(&mp_pck, 1); - - /* key */ - if (parser_i->key_content) { - len = flb_sds_len(parser_i->key_content); - msgpack_pack_str(&mp_pck, len); - msgpack_pack_str_body(&mp_pck, parser_i->key_content, len); - } - else { - msgpack_pack_str(&mp_pck, 3); - msgpack_pack_str_body(&mp_pck, "log", 3); - } - - /* val */ - len = flb_sds_len(group->buf); - msgpack_pack_str(&mp_pck, len); - msgpack_pack_str_body(&mp_pck, group->buf, len); - } - - if (mp_sbuf.size > 0) { - /* - * a 'forced_flush' means to alert the caller that the data 'must be flushed to it destination'. This flag is - * only enabled when the flush process has been triggered by the multiline timer, e.g: - * - * - the message is complete or incomplete and its time to dispatch it. - */ - if (forced_flush) { - mst->forced_flush = FLB_TRUE; - } - - /* encode and invoke the user callback */ - - ret = flb_log_event_encoder_begin_record( - &mst->ml->log_event_encoder); - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_set_timestamp( - &mst->ml->log_event_encoder, - group_time); - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_ml_flush_metadata_buffer(mst, - group, - FLB_TRUE); - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_set_body_from_raw_msgpack( - &mst->ml->log_event_encoder, - mp_sbuf.data, - mp_sbuf.size); - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_commit_record( - &mst->ml->log_event_encoder); - } - - if (ret != FLB_EVENT_ENCODER_SUCCESS) { - flb_error("[multiline] error packing event"); - - return -1; - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - mst->cb_flush(ml_parser, - mst, - mst->cb_data, - mst->ml->log_event_encoder.output_buffer, - mst->ml->log_event_encoder.output_length); - } - else { - flb_error("[multiline] log event encoder error : %d", ret); - } - - flb_log_event_encoder_reset(&mst->ml->log_event_encoder); - - if (forced_flush) { - mst->forced_flush = FLB_FALSE; - } - } - - msgpack_sbuffer_destroy(&mp_sbuf); - flb_sds_len_set(group->buf, 0); - - /* Update last flush time */ - group->last_flush = time_ms_now(); - - return 0; -} - -/* - * Initialize multiline global environment. - * - * note: function must be invoked before any flb_ml_create() call. - */ -int flb_ml_init(struct flb_config *config) -{ - int ret; - - ret = flb_ml_parser_builtin_create(config); - if (ret == -1) { - return -1; - } - - return 0; -} - -int flb_ml_exit(struct flb_config *config) -{ - flb_ml_parser_destroy_all(&config->multiline_parsers); - return 0; -} diff --git a/fluent-bit/src/multiline/flb_ml_group.c b/fluent-bit/src/multiline/flb_ml_group.c deleted file mode 100644 index 895a71056..000000000 --- a/fluent-bit/src/multiline/flb_ml_group.c +++ /dev/null @@ -1,86 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <fluent-bit/flb_info.h> -#include <fluent-bit/flb_mem.h> -#include <fluent-bit/flb_log.h> -#include <fluent-bit/multiline/flb_ml.h> -#include <fluent-bit/multiline/flb_ml_parser.h> - -struct flb_ml_group *flb_ml_group_create(struct flb_ml *ml) -{ - struct flb_ml_group *group; - - group = flb_calloc(1, sizeof(struct flb_ml_group)); - if (!group) { - flb_errno(); - return NULL; - } - group->id = mk_list_size(&ml->groups); - group->ml = ml; - group->lru_parser = NULL; - mk_list_init(&group->parsers); - - mk_list_add(&group->_head, &ml->groups); - - return group; -} - -/* - * Link a parser instance into the active group, if no group exists, a default - * one is created. - */ -int flb_ml_group_add_parser(struct flb_ml *ctx, struct flb_ml_parser_ins *p) -{ - struct flb_ml_group *group = NULL; - - if (mk_list_size(&ctx->groups) == 0) { - group = flb_ml_group_create(ctx); - if (!group) { - return -1; - } - } - else { - /* retrieve the latest active group */ - group = mk_list_entry_last(&ctx->groups, struct flb_ml_group, _head); - } - - if (!group) { - return -1; - } - - mk_list_add(&p->_head, &group->parsers); - return 0; -} - -void flb_ml_group_destroy(struct flb_ml_group *group) -{ - struct mk_list *head; - struct mk_list *tmp; - struct flb_ml_parser_ins *parser_i; - - /* destroy parser instances */ - mk_list_foreach_safe(head, tmp, &group->parsers) { - parser_i = mk_list_entry(head, struct flb_ml_parser_ins, _head); - flb_ml_parser_instance_destroy(parser_i); - } - - mk_list_del(&group->_head); - flb_free(group); -} diff --git a/fluent-bit/src/multiline/flb_ml_mode.c b/fluent-bit/src/multiline/flb_ml_mode.c deleted file mode 100644 index 964672d82..000000000 --- a/fluent-bit/src/multiline/flb_ml_mode.c +++ /dev/null @@ -1,111 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <fluent-bit/flb_info.h> -#include <fluent-bit/flb_log.h> -#include <fluent-bit/multiline/flb_ml.h> -#include <fluent-bit/multiline/flb_ml_mode.h> - -struct flb_ml *flb_ml_mode_create(struct flb_config *config, char *mode, int flush_ms, - char *key) -{ - if (strcmp(mode, "docker") == 0) { - return flb_ml_mode_docker(config, flush_ms); - } - else if (strcmp(mode, "cri") == 0) { - return flb_ml_mode_cri(config, flush_ms); - } - else if (strcmp(mode, "python") == 0) { - return flb_ml_mode_python(config, flush_ms, key); - } - else if (strcmp(mode, "java") == 0) { - return flb_ml_mode_java(config, flush_ms, key); - } - else if (strcmp(mode, "go") == 0) { - return flb_ml_mode_go(config, flush_ms, key); - } - - flb_error("[multiline] built-in mode '%s' not found", mode); - return NULL; -} - - -struct flb_ml_mode *flb_ml_parser_create(struct flb_config *ctx, - char *name, - int type, char *match_str, int negate, - int flush_ms, - char *key_content, - char *key_group, - char *key_pattern, - struct flb_parser *parser_ctx, - char *parser_name) -{ - struct flb_ml_mode *ml; - - ml = flb_calloc(1, sizeof(struct flb_ml)); - if (!ml) { - flb_errno(); - return NULL; - } - ml->name = flb_sds_create(name); - ml->type = type; - - if (match_str) { - ml->match_str = flb_sds_create(match_str); - if (!ml->match_str) { - flb_free(ml); - return NULL; - } - } - - ml->parser = parser_ctx; - if (parser_name) { - ml->parser_name = flb_sds_create(parser_name); - } - - ml->negate = negate; - mk_list_init(&ml->streams); - mk_list_init(&ml->regex_rules); - mk_list_add(&ml->_head, &ctx->multilines); - - if (key_content) { - ml->key_content = flb_sds_create(key_content); - if (!ml->key_content) { - flb_ml_destroy(ml); - return NULL; - } - } - - if (key_group) { - ml->key_group = flb_sds_create(key_group); - if (!ml->key_group) { - flb_ml_destroy(ml); - return NULL; - } - } - - if (key_pattern) { - ml->key_pattern = flb_sds_create(key_pattern); - if (!ml->key_pattern) { - flb_ml_destroy(ml); - return NULL; - } - } - return ml; -} diff --git a/fluent-bit/src/multiline/flb_ml_parser.c b/fluent-bit/src/multiline/flb_ml_parser.c deleted file mode 100644 index 7aa33789d..000000000 --- a/fluent-bit/src/multiline/flb_ml_parser.c +++ /dev/null @@ -1,347 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <fluent-bit/flb_info.h> -#include <fluent-bit/flb_log.h> -#include <fluent-bit/multiline/flb_ml.h> -#include <fluent-bit/multiline/flb_ml_rule.h> -#include <fluent-bit/multiline/flb_ml_mode.h> -#include <fluent-bit/multiline/flb_ml_group.h> - -int flb_ml_parser_init(struct flb_ml_parser *ml_parser) -{ - int ret; - - ret = flb_ml_rule_init(ml_parser); - if (ret == -1) { - return -1; - } - - return 0; -} - -/* Create built-in multiline parsers */ -int flb_ml_parser_builtin_create(struct flb_config *config) -{ - struct flb_ml_parser *mlp; - - /* Docker */ - mlp = flb_ml_parser_docker(config); - if (!mlp) { - flb_error("[multiline] could not init 'docker' built-in parser"); - return -1; - } - - /* CRI */ - mlp = flb_ml_parser_cri(config); - if (!mlp) { - flb_error("[multiline] could not init 'cri' built-in parser"); - return -1; - } - - /* Java */ - mlp = flb_ml_parser_java(config, NULL); - if (!mlp) { - flb_error("[multiline] could not init 'java' built-in parser"); - return -1; - } - - /* Go */ - mlp = flb_ml_parser_go(config, NULL); - if (!mlp) { - flb_error("[multiline] could not init 'go' built-in parser"); - return -1; - } - - /* Ruby */ - mlp = flb_ml_parser_ruby(config, NULL); - if (!mlp) { - flb_error("[multiline] could not init 'ruby' built-in parser"); - return -1; - } - - /* Python */ - mlp = flb_ml_parser_python(config, NULL); - if (!mlp) { - flb_error("[multiline] could not init 'python' built-in parser"); - return -1; - } - - return 0; -} - -struct flb_ml_parser *flb_ml_parser_create(struct flb_config *ctx, - char *name, - int type, char *match_str, int negate, - int flush_ms, - char *key_content, - char *key_group, - char *key_pattern, - struct flb_parser *parser_ctx, - char *parser_name) -{ - struct flb_ml_parser *ml_parser; - - ml_parser = flb_calloc(1, sizeof(struct flb_ml_parser)); - if (!ml_parser) { - flb_errno(); - return NULL; - } - ml_parser->name = flb_sds_create(name); - ml_parser->type = type; - - if (match_str) { - ml_parser->match_str = flb_sds_create(match_str); - if (!ml_parser->match_str) { - if (ml_parser->name) { - flb_sds_destroy(ml_parser->name); - } - flb_free(ml_parser); - return NULL; - } - } - - ml_parser->parser = parser_ctx; - - if (parser_name) { - ml_parser->parser_name = flb_sds_create(parser_name); - } - ml_parser->negate = negate; - ml_parser->flush_ms = flush_ms; - mk_list_init(&ml_parser->regex_rules); - mk_list_add(&ml_parser->_head, &ctx->multiline_parsers); - - if (key_content) { - ml_parser->key_content = flb_sds_create(key_content); - if (!ml_parser->key_content) { - flb_ml_parser_destroy(ml_parser); - return NULL; - } - } - - if (key_group) { - ml_parser->key_group = flb_sds_create(key_group); - if (!ml_parser->key_group) { - flb_ml_parser_destroy(ml_parser); - return NULL; - } - } - - if (key_pattern) { - ml_parser->key_pattern = flb_sds_create(key_pattern); - if (!ml_parser->key_pattern) { - flb_ml_parser_destroy(ml_parser); - return NULL; - } - } - - return ml_parser; -} - -struct flb_ml_parser *flb_ml_parser_get(struct flb_config *ctx, char *name) -{ - struct mk_list *head; - struct flb_ml_parser *ml_parser; - - mk_list_foreach(head, &ctx->multiline_parsers) { - ml_parser = mk_list_entry(head, struct flb_ml_parser, _head); - if (strcasecmp(ml_parser->name, name) == 0) { - return ml_parser; - } - } - - return NULL; -} - -int flb_ml_parser_instance_has_data(struct flb_ml_parser_ins *ins) -{ - struct mk_list *head; - struct mk_list *head_group; - struct flb_ml_stream *st; - struct flb_ml_stream_group *st_group; - - mk_list_foreach(head, &ins->streams) { - st = mk_list_entry(head, struct flb_ml_stream, _head); - mk_list_foreach(head_group, &st->groups) { - st_group = mk_list_entry(head_group, struct flb_ml_stream_group, _head); - if (st_group->mp_sbuf.size > 0) { - return FLB_TRUE; - } - } - } - - return FLB_FALSE; -} - -struct flb_ml_parser_ins *flb_ml_parser_instance_create(struct flb_ml *ml, - char *name) -{ - int ret; - struct flb_ml_parser_ins *ins; - struct flb_ml_parser *parser; - - parser = flb_ml_parser_get(ml->config, name); - if (!parser) { - flb_error("[multiline] parser '%s' not registered", name); - return NULL; - } - - ins = flb_calloc(1, sizeof(struct flb_ml_parser_ins)); - if (!ins) { - flb_errno(); - return NULL; - } - ins->last_stream_id = 0; - ins->ml_parser = parser; - mk_list_init(&ins->streams); - - /* Copy parent configuration */ - if (parser->key_content) { - ins->key_content = flb_sds_create(parser->key_content); - } - if (parser->key_pattern) { - ins->key_pattern = flb_sds_create(parser->key_pattern); - } - if (parser->key_group) { - ins->key_group = flb_sds_create(parser->key_group); - } - - /* Append this multiline parser instance to the active multiline group */ - ret = flb_ml_group_add_parser(ml, ins); - if (ret != 0) { - flb_error("[multiline] could not register parser '%s' on " - "multiline '%s 'group", name, ml->name); - flb_free(ins); - return NULL; - } - - /* - * Update flush_interval for pending records on multiline context. We always - * use the greater value found. - */ - if (parser->flush_ms > ml->flush_ms) { - ml->flush_ms = parser->flush_ms; - } - - return ins; -} - -/* Override a fixed parser property for the instance only*/ -int flb_ml_parser_instance_set(struct flb_ml_parser_ins *p, char *prop, char *val) -{ - if (strcasecmp(prop, "key_content") == 0) { - if (p->key_content) { - flb_sds_destroy(p->key_content); - } - p->key_content = flb_sds_create(val); - } - else if (strcasecmp(prop, "key_pattern") == 0) { - if (p->key_pattern) { - flb_sds_destroy(p->key_pattern); - } - p->key_pattern = flb_sds_create(val); - } - else if (strcasecmp(prop, "key_group") == 0) { - if (p->key_group) { - flb_sds_destroy(p->key_group); - } - p->key_group = flb_sds_create(val); - } - else { - return -1; - } - - return 0; -} - -int flb_ml_parser_destroy(struct flb_ml_parser *ml_parser) -{ - if (!ml_parser) { - return 0; - } - - if (ml_parser->name) { - flb_sds_destroy(ml_parser->name); - } - - if (ml_parser->parser_name) { - flb_sds_destroy(ml_parser->parser_name); - } - - if (ml_parser->match_str) { - flb_sds_destroy(ml_parser->match_str); - } - if (ml_parser->key_content) { - flb_sds_destroy(ml_parser->key_content); - } - if (ml_parser->key_group) { - flb_sds_destroy(ml_parser->key_group); - } - if (ml_parser->key_pattern) { - flb_sds_destroy(ml_parser->key_pattern); - } - - /* Regex rules */ - flb_ml_rule_destroy_all(ml_parser); - - /* Unlink from struct flb_config->multiline_parsers */ - mk_list_del(&ml_parser->_head); - - flb_free(ml_parser); - return 0; -} - -int flb_ml_parser_instance_destroy(struct flb_ml_parser_ins *ins) -{ - struct mk_list *tmp; - struct mk_list *head; - struct flb_ml_stream *stream; - - /* Destroy streams */ - mk_list_foreach_safe(head, tmp, &ins->streams) { - stream = mk_list_entry(head, struct flb_ml_stream, _head); - flb_ml_stream_destroy(stream); - } - - if (ins->key_content) { - flb_sds_destroy(ins->key_content); - } - if (ins->key_pattern) { - flb_sds_destroy(ins->key_pattern); - } - if (ins->key_group) { - flb_sds_destroy(ins->key_group); - } - - flb_free(ins); - - return 0; -} - -void flb_ml_parser_destroy_all(struct mk_list *list) -{ - struct mk_list *tmp; - struct mk_list *head; - struct flb_ml_parser *parser; - - mk_list_foreach_safe(head, tmp, list) { - parser = mk_list_entry(head, struct flb_ml_parser, _head); - flb_ml_parser_destroy(parser); - } -} diff --git a/fluent-bit/src/multiline/flb_ml_parser_cri.c b/fluent-bit/src/multiline/flb_ml_parser_cri.c deleted file mode 100644 index 669fa39a2..000000000 --- a/fluent-bit/src/multiline/flb_ml_parser_cri.c +++ /dev/null @@ -1,81 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <fluent-bit/flb_info.h> -#include <fluent-bit/multiline/flb_ml.h> -#include <fluent-bit/multiline/flb_ml_parser.h> - -#define FLB_ML_CRI_REGEX \ - "^(?<time>.+?) (?<stream>stdout|stderr) (?<_p>F|P) (?<log>.*)$" -#define FLB_ML_CRI_TIME \ - "%Y-%m-%dT%H:%M:%S.%L%z" - -/* Creates a parser for Docker */ -static struct flb_parser *cri_parser_create(struct flb_config *config) -{ - struct flb_parser *p; - - p = flb_parser_create("_ml_cri", /* parser name */ - "regex", /* backend type */ - FLB_ML_CRI_REGEX, /* regex */ - FLB_FALSE, /* skip_empty */ - FLB_ML_CRI_TIME, /* time format */ - "time", /* time key */ - NULL, /* time offset */ - FLB_TRUE, /* time keep */ - FLB_FALSE, /* time strict */ - FLB_FALSE, /* no bare keys */ - NULL, /* parser types */ - 0, /* types len */ - NULL, /* decoders */ - config); /* Fluent Bit context */ - return p; -} - -/* Our first multiline mode: 'docker' */ -struct flb_ml_parser *flb_ml_parser_cri(struct flb_config *config) -{ - struct flb_parser *parser; - struct flb_ml_parser *mlp; - - /* Create a Docker parser */ - parser = cri_parser_create(config); - if (!parser) { - return NULL; - } - - mlp = flb_ml_parser_create(config, - "cri", /* name */ - FLB_ML_EQ, /* type */ - "F", /* match_str */ - FLB_FALSE, /* negate */ - FLB_ML_FLUSH_TIMEOUT, /* flush_ms */ - "log", /* key_content */ - "stream", /* key_group */ - "_p", /* key_pattern */ - parser, /* parser ctx */ - NULL); /* parser name */ - - if (!mlp) { - flb_error("[multiline] could not create 'cri mode'"); - return NULL; - } - - return mlp; -} diff --git a/fluent-bit/src/multiline/flb_ml_parser_docker.c b/fluent-bit/src/multiline/flb_ml_parser_docker.c deleted file mode 100644 index 5b622d32c..000000000 --- a/fluent-bit/src/multiline/flb_ml_parser_docker.c +++ /dev/null @@ -1,110 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <fluent-bit/flb_info.h> -#include <fluent-bit/multiline/flb_ml.h> -#include <fluent-bit/multiline/flb_ml_parser.h> - -/* Creates a parser for Docker */ -static struct flb_parser *docker_parser_create(struct flb_config *config) -{ - struct flb_parser *p; - - p = flb_parser_create("_ml_json_docker", /* parser name */ - "json", /* backend type */ - NULL, /* regex */ - FLB_TRUE, /* skip_empty */ - "%Y-%m-%dT%H:%M:%S.%L", /* time format */ - "time", /* time key */ - NULL, /* time offset */ - FLB_TRUE, /* time keep */ - FLB_FALSE, /* time strict */ - FLB_FALSE, /* no bare keys */ - NULL, /* parser types */ - 0, /* types len */ - NULL, /* decoders */ - config); /* Fluent Bit context */ - return p; -} - -/* Our first multiline mode: 'docker' */ -struct flb_ml_parser *flb_ml_parser_docker(struct flb_config *config) -{ - struct flb_parser *parser; - struct flb_ml_parser *mlp; - - /* Create a Docker parser */ - parser = docker_parser_create(config); - if (!parser) { - return NULL; - } - - /* - * Let's explain this multiline mode, then you (the reader) might want - * to submit a PR with new built-in modes :) - * - * Containerized apps under Docker writes logs to stdout/stderr. These streams - * (stdout/stderr) are handled by Docker, in most of cases the content is - * stored in a .json file in your file system. A message like "hey!" gets into - * a JSON map like this: - * - * {"log": "hey!\n", "stream": "stdout", "time": "2021-02-01T01:40:03.53412Z"} - * - * By Docker log spec, any 'log' key that "ends with a \n" it's a complete - * log record, but Docker also limits the log record size to 16KB, so a long - * message that does not fit into 16KB can be split in multiple JSON lines, - * the following example use short words to describe the context: - * - * - original message: 'one, two, three\n' - * - * Docker log interpretation: - * - * - {"log": "one, ", "stream": "stdout", "time": "2021-02-01T01:40:03.53413Z"} - * - {"log": "two, ", "stream": "stdout", "time": "2021-02-01T01:40:03.53414Z"} - * - {"log": "three\n", "stream": "stdout", "time": "2021-02-01T01:40:03.53415Z"} - * - * So every 'log' key that does not ends with '\n', it's a partial log record - * and for logging purposes it needs to be concatenated with further messages - * until a final '\n' is found. - * - * We setup the Multiline mode as follows: - * - * - Use the type 'FLB_ML_ENDSWITH' to specify that we expect the 'log' - * key must ends with a '\n' for complete messages, otherwise it means is - * a continuation message. In case a message is not complete just wait until - * 500 milliseconds (0.5 second) and flush the buffer. - */ - mlp = flb_ml_parser_create(config, /* Fluent Bit context */ - "docker", /* name */ - FLB_ML_ENDSWITH, /* type */ - "\n", /* match_str */ - FLB_FALSE, /* negate */ - FLB_ML_FLUSH_TIMEOUT, /* flush_ms */ - "log", /* key_content */ - "stream", /* key_group */ - NULL, /* key_pattern */ - parser, /* parser ctx */ - NULL); /* parser name */ - if (!mlp) { - flb_error("[multiline] could not create 'docker mode'"); - return NULL; - } - - return mlp; -} diff --git a/fluent-bit/src/multiline/flb_ml_parser_go.c b/fluent-bit/src/multiline/flb_ml_parser_go.c deleted file mode 100644 index f1cd5407f..000000000 --- a/fluent-bit/src/multiline/flb_ml_parser_go.c +++ /dev/null @@ -1,140 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <fluent-bit/flb_info.h> -#include <fluent-bit/multiline/flb_ml.h> -#include <fluent-bit/multiline/flb_ml_rule.h> -#include <fluent-bit/multiline/flb_ml_parser.h> - -#define rule flb_ml_rule_create - -static void rule_error(struct flb_ml_parser *mlp) -{ - int id; - - id = mk_list_size(&mlp->regex_rules); - flb_error("[multiline: go] rule #%i could not be created", id); - flb_ml_parser_destroy(mlp); -} - -/* Go mode */ -struct flb_ml_parser *flb_ml_parser_go(struct flb_config *config, char *key) -{ - int ret; - struct flb_ml_parser *mlp; - - mlp = flb_ml_parser_create(config, /* Fluent Bit context */ - "go", /* name */ - FLB_ML_REGEX, /* type */ - NULL, /* match_str */ - FLB_FALSE, /* negate */ - FLB_ML_FLUSH_TIMEOUT, /* flush_ms */ - key, /* key_content */ - NULL, /* key_group */ - NULL, /* key_pattern */ - NULL, /* parser ctx */ - NULL); /* parser name */ - - if (!mlp) { - flb_error("[multiline] could not create 'go mode'"); - return NULL; - } - - ret = rule(mlp, - "start_state", - "/\\bpanic: /", - "go_after_panic", NULL); - if (ret != 0) { - rule_error(mlp); - return NULL; - } - - ret = rule(mlp, - "start_state", - "/http: panic serving/", - "go_goroutine", NULL); - if (ret != 0) { - rule_error(mlp); - return NULL; - } - - ret = rule(mlp, - "go_after_panic", - "/^$/", - "go_goroutine", NULL); - if (ret != 0) { - rule_error(mlp); - return NULL; - } - - ret = rule(mlp, - "go_after_panic, go_after_signal, go_frame_1", - "/^$/", - "go_goroutine", NULL); - if (ret != 0) { - rule_error(mlp); - return NULL; - } - - ret = rule(mlp, - "go_after_panic", - "/^\\[signal /", - "go_after_signal", NULL); - if (ret != 0) { - rule_error(mlp); - return NULL; - } - - ret = rule(mlp, - "go_goroutine", - "/^goroutine \\d+ \\[[^\\]]+\\]:$/", - "go_frame_1", NULL); - if (ret != 0) { - rule_error(mlp); - return NULL; - } - - ret = rule(mlp, - "go_frame_1", - "/^(?:[^\\s.:]+\\.)*[^\\s.():]+\\(|^created by /", - "go_frame_2", NULL); - if (ret != 0) { - rule_error(mlp); - return NULL; - } - - ret = rule(mlp, - "go_frame_2", - "/^\\s/", - "go_frame_1", NULL); - if (ret != 0) { - rule_error(mlp); - return NULL; - } - - /* Map the rules (mandatory for regex rules) */ - ret = flb_ml_parser_init(mlp); - if (ret != 0) { - flb_error("[multiline: go] error on mapping rules"); - flb_ml_parser_destroy(mlp); - return NULL; - } - - return mlp; -} diff --git a/fluent-bit/src/multiline/flb_ml_parser_java.c b/fluent-bit/src/multiline/flb_ml_parser_java.c deleted file mode 100644 index 4df5a00f7..000000000 --- a/fluent-bit/src/multiline/flb_ml_parser_java.c +++ /dev/null @@ -1,143 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <fluent-bit/flb_info.h> -#include <fluent-bit/multiline/flb_ml.h> -#include <fluent-bit/multiline/flb_ml_rule.h> -#include <fluent-bit/multiline/flb_ml_parser.h> - -#define rule flb_ml_rule_create - -static void rule_error(struct flb_ml_parser *ml_parser) -{ - int id; - - id = mk_list_size(&ml_parser->regex_rules); - flb_error("[multiline: java] rule #%i could not be created", id); - flb_ml_parser_destroy(ml_parser); -} - -/* Java mode */ -struct flb_ml_parser *flb_ml_parser_java(struct flb_config *config, char *key) -{ - int ret; - struct flb_ml_parser *mlp; - - mlp = flb_ml_parser_create(config, /* Fluent Bit context */ - "java", /* name */ - FLB_ML_REGEX, /* type */ - NULL, /* match_str */ - FLB_FALSE, /* negate */ - FLB_ML_FLUSH_TIMEOUT, /* flush_ms */ - key, /* key_content */ - NULL, /* key_group */ - NULL, /* key_pattern */ - NULL, /* parser ctx */ - NULL); /* parser name */ - - if (!mlp) { - flb_error("[multiline] could not create 'java mode'"); - return NULL; - } - - ret = rule(mlp, - "start_state, java_start_exception", - "/(.)(?:Exception|Error|Throwable|V8 errors stack trace)[:\\r\\n]/", - "java_after_exception", NULL); - if (ret != 0) { - rule_error(mlp); - return NULL; - } - - ret = rule(mlp, - "java_after_exception", - "/^[\\t ]*nested exception is:[\\t ]*/", - "java_start_exception", NULL); - if (ret != 0) { - rule_error(mlp); - return NULL; - } - - ret = rule(mlp, - "java_after_exception", - "/^[\\r\\n]*$/", - "java_after_exception", NULL); - if (ret != 0) { - rule_error(mlp); - return NULL; - } - - ret = rule(mlp, - "java_after_exception, java", - "/^[\\t ]+(?:eval )?at /", - "java", NULL); - if (ret != 0) { - rule_error(mlp); - return NULL; - } - - ret = rule(mlp, - "java_after_exception, java", - /* C# nested exception */ - "/^[\\t ]+--- End of inner exception stack trace ---$/", - "java", NULL); - if (ret != 0) { - rule_error(mlp); - return NULL; - } - - ret = rule(mlp, - "java_after_exception, java", - /* C# exception from async code */ - "/^--- End of stack trace from previous (?x:" - ")location where exception was thrown ---$/", - "java", NULL); - if (ret != 0) { - rule_error(mlp); - return NULL; - } - - ret = rule(mlp, - "java_after_exception, java", - "/^[\\t ]*(?:Caused by|Suppressed):/", - "java_after_exception", NULL); - if (ret != 0) { - rule_error(mlp); - return NULL; - } - - ret = rule(mlp, - "java_after_exception, java", - "/^[\\t ]*... \\d+ (?:more|common frames omitted)/", - "java", NULL); - if (ret != 0) { - rule_error(mlp); - return NULL; - } - - /* Map the rules (mandatory for regex rules) */ - ret = flb_ml_parser_init(mlp); - if (ret != 0) { - flb_error("[multiline: java] error on mapping rules"); - flb_ml_parser_destroy(mlp); - return NULL; - } - - return mlp; -} diff --git a/fluent-bit/src/multiline/flb_ml_parser_python.c b/fluent-bit/src/multiline/flb_ml_parser_python.c deleted file mode 100644 index a92088397..000000000 --- a/fluent-bit/src/multiline/flb_ml_parser_python.c +++ /dev/null @@ -1,98 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <fluent-bit/flb_info.h> -#include <fluent-bit/multiline/flb_ml.h> -#include <fluent-bit/multiline/flb_ml_rule.h> -#include <fluent-bit/multiline/flb_ml_parser.h> - -#define rule flb_ml_rule_create - -static void rule_error(struct flb_ml_parser *mlp) -{ - int id; - - id = mk_list_size(&mlp->regex_rules); - flb_error("[multiline: python] rule #%i could not be created", id); - flb_ml_parser_destroy(mlp); -} - -/* Python */ -struct flb_ml_parser *flb_ml_parser_python(struct flb_config *config, char *key) -{ - int ret; - struct flb_ml_parser *mlp; - - mlp = flb_ml_parser_create(config, /* Fluent Bit context */ - "python", /* name */ - FLB_ML_REGEX, /* type */ - NULL, /* match_str */ - FLB_FALSE, /* negate */ - FLB_ML_FLUSH_TIMEOUT, /* flush_ms */ - key, /* key_content */ - NULL, /* key_group */ - NULL, /* key_pattern */ - NULL, /* parser ctx */ - NULL); /* parser name */ - - if (!mlp) { - flb_error("[multiline] could not create 'python mode'"); - return NULL; - } - - /* rule(:start_state, /^Traceback \(most recent call last\):$/, :python) */ - ret = rule(mlp, - "start_state", "/^Traceback \\(most recent call last\\):$/", - "python", NULL); - if (ret != 0) { - rule_error(mlp); - return NULL; - } - - /* rule(:python, /^[\t ]+File /, :python_code) */ - ret = rule(mlp, "python", "/^[\\t ]+File /", "python_code", NULL); - if (ret != 0) { - rule_error(mlp); - return NULL; - } - - /* rule(:python_code, /[^\t ]/, :python) */ - ret = rule(mlp, "python_code", "/[^\\t ]/", "python", NULL); - if (ret != 0) { - rule_error(mlp); - return NULL; - } - - /* rule(:python, /^(?:[^\s.():]+\.)*[^\s.():]+:/, :start_state) */ - ret = rule(mlp, "python", "/^(?:[^\\s.():]+\\.)*[^\\s.():]+:/", "start_state", NULL); - if (ret != 0) { - rule_error(mlp); - return NULL; - } - - /* Map the rules (mandatory for regex rules) */ - ret = flb_ml_parser_init(mlp); - if (ret != 0) { - flb_error("[multiline: python] error on mapping rules"); - flb_ml_parser_destroy(mlp); - return NULL; - } - - return mlp; -} diff --git a/fluent-bit/src/multiline/flb_ml_parser_ruby.c b/fluent-bit/src/multiline/flb_ml_parser_ruby.c deleted file mode 100644 index 780f829d0..000000000 --- a/fluent-bit/src/multiline/flb_ml_parser_ruby.c +++ /dev/null @@ -1,87 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <fluent-bit/flb_info.h> -#include <fluent-bit/multiline/flb_ml.h> -#include <fluent-bit/multiline/flb_ml_rule.h> -#include <fluent-bit/multiline/flb_ml_parser.h> - -#define rule flb_ml_rule_create - -static void rule_error(struct flb_ml_parser *mlp) -{ - int id; - - id = mk_list_size(&mlp->regex_rules); - flb_error("[multiline: ruby] rule #%i could not be created", id); - flb_ml_parser_destroy(mlp); -} - -/* Ruby mode */ -struct flb_ml_parser *flb_ml_parser_ruby(struct flb_config *config, char *key) -{ - int ret; - struct flb_ml_parser *mlp; - - mlp = flb_ml_parser_create(config, /* Fluent Bit context */ - "ruby", /* name */ - FLB_ML_REGEX, /* type */ - NULL, /* match_str */ - FLB_FALSE, /* negate */ - FLB_ML_FLUSH_TIMEOUT, /* flush_ms */ - key, /* key_content */ - NULL, /* key_group */ - NULL, /* key_pattern */ - NULL, /* parser ctx */ - NULL); /* parser name */ - - if (!mlp) { - flb_error("[multiline] could not create 'ruby mode'"); - return NULL; - } - - ret = rule(mlp, - "start_state, ruby_start_exception", - "/^.+:\\d+:in\\s+.*/", - "ruby_after_exception", NULL); - if (ret != 0) { - rule_error(mlp); - return NULL; - } - - ret = rule(mlp, - "ruby_after_exception, ruby", - "/^\\s+from\\s+.*:\\d+:in\\s+.*/", - "ruby", NULL); - if (ret != 0) { - rule_error(mlp); - return NULL; - } - - - /* Map the rules (mandatory for regex rules) */ - ret = flb_ml_parser_init(mlp); - if (ret != 0) { - flb_error("[multiline: ruby] error on mapping rules"); - flb_ml_parser_destroy(mlp); - return NULL; - } - - return mlp; -} diff --git a/fluent-bit/src/multiline/flb_ml_rule.c b/fluent-bit/src/multiline/flb_ml_rule.c deleted file mode 100644 index 26520dfed..000000000 --- a/fluent-bit/src/multiline/flb_ml_rule.c +++ /dev/null @@ -1,421 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <fluent-bit/flb_info.h> -#include <fluent-bit/flb_regex.h> -#include <fluent-bit/flb_slist.h> - -#include <fluent-bit/multiline/flb_ml.h> -#include <fluent-bit/multiline/flb_ml_rule.h> - -struct to_state { - struct flb_ml_rule *rule; - struct mk_list _head; -}; - -struct flb_slist_entry *get_start_state(struct mk_list *list) -{ - struct mk_list *head; - struct flb_slist_entry *e; - - mk_list_foreach(head, list) { - e = mk_list_entry(head, struct flb_slist_entry, _head); - if (strcmp(e->str, "start_state") == 0) { - return e; - } - } - - return NULL; -} - -int flb_ml_rule_create(struct flb_ml_parser *ml_parser, - flb_sds_t from_states, - char *regex_pattern, - flb_sds_t to_state, - char *end_pattern) -{ - int ret; - int first_rule = FLB_FALSE; - struct flb_ml_rule *rule; - - rule = flb_calloc(1, sizeof(struct flb_ml_rule)); - if (!rule) { - flb_errno(); - return -1; - } - flb_slist_create(&rule->from_states); - mk_list_init(&rule->to_state_map); - - if (mk_list_size(&ml_parser->regex_rules) == 0) { - first_rule = FLB_TRUE; - } - mk_list_add(&rule->_head, &ml_parser->regex_rules); - - /* from_states */ - ret = flb_slist_split_string(&rule->from_states, from_states, ',', -1); - if (ret <= 0) { - flb_error("[multiline] rule is empty or has invalid 'from_states' tokens"); - flb_ml_rule_destroy(rule); - return -1; - } - - /* Check if the rule contains a 'start_state' */ - if (get_start_state(&rule->from_states)) { - rule->start_state = FLB_TRUE; - } - else if (first_rule) { - flb_error("[multiline] rule don't contain a 'start_state'"); - flb_ml_rule_destroy(rule); - return -1; - } - - /* regex content pattern */ - rule->regex = flb_regex_create(regex_pattern); - if (!rule->regex) { - flb_ml_rule_destroy(rule); - return -1; - } - - /* to_state */ - if (to_state) { - rule->to_state = flb_sds_create(to_state); - if (!rule->to_state) { - flb_ml_rule_destroy(rule); - return -1; - } - } - - /* regex end pattern */ - if (end_pattern) { - rule->regex_end = flb_regex_create(end_pattern); - if (!rule->regex_end) { - flb_ml_rule_destroy(rule); - return -1; - } - } - - return 0; -} - -void flb_ml_rule_destroy(struct flb_ml_rule *rule) -{ - struct mk_list *tmp; - struct mk_list *head; - struct to_state *st; - - flb_slist_destroy(&rule->from_states); - - if (rule->regex) { - flb_regex_destroy(rule->regex); - } - - - if (rule->to_state) { - flb_sds_destroy(rule->to_state); - } - - mk_list_foreach_safe(head, tmp, &rule->to_state_map) { - st = mk_list_entry(head, struct to_state, _head); - mk_list_del(&st->_head); - flb_free(st); - } - - if (rule->regex_end) { - flb_regex_destroy(rule->regex_end); - } - - mk_list_del(&rule->_head); - flb_free(rule); -} - -void flb_ml_rule_destroy_all(struct flb_ml_parser *ml_parser) -{ - struct mk_list *tmp; - struct mk_list *head; - struct flb_ml_rule *rule; - - mk_list_foreach_safe(head, tmp, &ml_parser->regex_rules) { - rule = mk_list_entry(head, struct flb_ml_rule, _head); - flb_ml_rule_destroy(rule); - } -} - -static inline int to_states_exists(struct flb_ml_parser *ml_parser, - flb_sds_t state) -{ - struct mk_list *head; - struct mk_list *r_head; - struct flb_ml_rule *rule; - struct flb_slist_entry *e; - - mk_list_foreach(head, &ml_parser->regex_rules) { - rule = mk_list_entry(head, struct flb_ml_rule, _head); - - mk_list_foreach(r_head, &rule->from_states) { - e = mk_list_entry(r_head, struct flb_slist_entry, _head); - if (strcmp(e->str, state) == 0) { - return FLB_TRUE; - } - } - } - - return FLB_FALSE; -} - -static inline int to_states_matches_rule(struct flb_ml_rule *rule, - flb_sds_t state) -{ - struct mk_list *head; - struct flb_slist_entry *e; - - mk_list_foreach(head, &rule->from_states) { - e = mk_list_entry(head, struct flb_slist_entry, _head); - if (strcmp(e->str, state) == 0) { - return FLB_TRUE; - } - } - - return FLB_FALSE; -} - -static int set_to_state_map(struct flb_ml_parser *ml_parser, - struct flb_ml_rule *rule) -{ - int ret; - struct to_state *s; - struct mk_list *head; - struct flb_ml_rule *r; - - if (!rule->to_state) { - /* no to_state */ - return 0; - } - - /* Iterate all rules that matches the to_state */ - mk_list_foreach(head, &ml_parser->regex_rules) { - r = mk_list_entry(head, struct flb_ml_rule, _head); - - /* Check if rule->to_state, matches an existing (registered) from_state */ - ret = to_states_exists(ml_parser, rule->to_state); - if (!ret) { - flb_error("[multiline parser: %s] to_state='%s' is not registered", - ml_parser->name, rule->to_state); - return -1; - } - - /* - * A rule can have many 'from_states', check if the current 'rule->to_state' - * matches any 'r->from_states' - */ - ret = to_states_matches_rule(r, rule->to_state); - if (!ret) { - continue; - } - - /* We have a match. Create a 'to_state' entry into the 'to_state_map' list */ - s = flb_malloc(sizeof(struct to_state)); - if (!s) { - flb_errno(); - return -1; - } - s->rule = r; - mk_list_add(&s->_head, &rule->to_state_map); - } - - return 0; -} - -static int try_flushing_buffer(struct flb_ml_parser *ml_parser, - struct flb_ml_stream *mst, - struct flb_ml_stream_group *group) -{ - int next_start = FLB_FALSE; - struct mk_list *head; - struct to_state *st; - struct flb_ml_rule *rule; - - rule = group->rule_to_state; - if (!rule) { - if (flb_sds_len(group->buf) > 0) { - flb_ml_flush_stream_group(ml_parser, mst, group, FLB_FALSE); - group->first_line = FLB_TRUE; - } - return 0; - } - - /* Check if any 'to_state_map' referenced rules is a possible start */ - mk_list_foreach(head, &rule->to_state_map) { - st = mk_list_entry(head, struct to_state, _head); - if (st->rule->start_state) { - next_start = FLB_TRUE; - break; - } - } - - if (next_start && flb_sds_len(group->buf) > 0) { - flb_ml_flush_stream_group(ml_parser, mst, group, FLB_FALSE); - group->first_line = FLB_TRUE; - } - - return 0; -} - -/* Initialize all rules */ -int flb_ml_rule_init(struct flb_ml_parser *ml_parser) -{ - int ret; - struct mk_list *head; - struct flb_ml_rule *rule; - - /* FIXME: sort rules by start_state first (let's trust in the caller) */ - - /* For each rule, compose it to_state_map list */ - mk_list_foreach(head, &ml_parser->regex_rules) { - rule = mk_list_entry(head, struct flb_ml_rule, _head); - /* Populate 'rule->to_state_map' list */ - ret = set_to_state_map(ml_parser, rule); - if (ret == -1) { - return -1; - } - } - - return 0; -} - -/* Search any 'start_state' matching the incoming 'buf_data' */ -static struct flb_ml_rule *try_start_state(struct flb_ml_parser *ml_parser, - char *buf_data, size_t buf_size) -{ - int ret = -1; - struct mk_list *head; - struct flb_ml_rule *rule = NULL; - - mk_list_foreach(head, &ml_parser->regex_rules) { - rule = mk_list_entry(head, struct flb_ml_rule, _head); - - /* Is this rule matching a start_state ? */ - if (!rule->start_state) { - rule = NULL; - continue; - } - - /* Matched a start_state. Check if we have a regex match */ - ret = flb_regex_match(rule->regex, (unsigned char *) buf_data, buf_size); - if (ret) { - return rule; - } - } - - return NULL; -} - -int flb_ml_rule_process(struct flb_ml_parser *ml_parser, - struct flb_ml_stream *mst, - struct flb_ml_stream_group *group, - msgpack_object *full_map, - void *buf, size_t size, struct flb_time *tm, - msgpack_object *val_content, - msgpack_object *val_pattern) -{ - int ret; - int len; - char *buf_data = NULL; - size_t buf_size = 0; - struct mk_list *head; - struct to_state *st = NULL; - struct flb_ml_rule *rule = NULL; - struct flb_ml_rule *tmp_rule = NULL; - - if (val_content) { - buf_data = (char *) val_content->via.str.ptr; - buf_size = val_content->via.str.size; - } - else { - buf_data = buf; - buf_size = size; - } - - if (group->rule_to_state) { - /* are we in a continuation ? */ - tmp_rule = group->rule_to_state; - - /* Lookup all possible next rules by state reference */ - rule = NULL; - mk_list_foreach(head, &tmp_rule->to_state_map) { - st = mk_list_entry(head, struct to_state, _head); - - /* skip start states */ - if (st->rule->start_state) { - continue; - } - - /* Try regex match */ - ret = flb_regex_match(st->rule->regex, - (unsigned char *) buf_data, buf_size); - if (ret) { - /* Regex matched */ - len = flb_sds_len(group->buf); - if (len >= 1 && group->buf[len - 1] != '\n') { - flb_sds_cat_safe(&group->buf, "\n", 1); - } - - if (buf_size == 0) { - flb_sds_cat_safe(&group->buf, "\n", 1); - } - else { - flb_sds_cat_safe(&group->buf, buf_data, buf_size); - } - rule = st->rule; - break; - } - rule = NULL; - } - - } - - if (!rule) { - /* Check if we are in a 'start_state' */ - rule = try_start_state(ml_parser, buf_data, buf_size); - if (rule) { - /* if the group buffer has any previous data just flush it */ - if (flb_sds_len(group->buf) > 0) { - flb_ml_flush_stream_group(ml_parser, mst, group, FLB_FALSE); - } - - /* set the rule state */ - group->rule_to_state = rule; - - /* concatenate the data */ - flb_sds_cat_safe(&group->buf, buf_data, buf_size); - - /* Copy full map content in stream buffer */ - flb_ml_register_context(group, tm, full_map); - - return 0; - } - } - - if (rule) { - group->rule_to_state = rule; - try_flushing_buffer(ml_parser, mst, group); - return 0; - } - - return -1; -} diff --git a/fluent-bit/src/multiline/flb_ml_stream.c b/fluent-bit/src/multiline/flb_ml_stream.c deleted file mode 100644 index c92ba3220..000000000 --- a/fluent-bit/src/multiline/flb_ml_stream.c +++ /dev/null @@ -1,338 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <fluent-bit/flb_info.h> -#include <fluent-bit/flb_mem.h> -#include <fluent-bit/flb_log.h> -#include <fluent-bit/flb_pack.h> -#include <fluent-bit/multiline/flb_ml.h> -#include <fluent-bit/multiline/flb_ml_rule.h> -#include <cfl/cfl.h> - -static int ml_flush_stdout(struct flb_ml_parser *parser, - struct flb_ml_stream *mst, - void *data, char *buf_data, size_t buf_size) -{ - fprintf(stdout, "\n%s----- MULTILINE FLUSH -----%s\n", - ANSI_GREEN, ANSI_RESET); - - /* Print incoming flush buffer */ - flb_pack_print(buf_data, buf_size); - - fprintf(stdout, "%s----------- EOF -----------%s\n", - ANSI_GREEN, ANSI_RESET); - return 0; -} - -static struct flb_ml_stream_group *stream_group_create(struct flb_ml_stream *mst, - char *name, int len) -{ - struct flb_ml_stream_group *group; - - if (!name) { - name = "_default"; - } - - group = flb_calloc(1, sizeof(struct flb_ml_stream_group)); - if (!group) { - flb_errno(); - return NULL; - } - group->name = flb_sds_create_len(name, len); - if (!group->name) { - flb_free(group); - return NULL; - } - - /* status */ - group->first_line = FLB_TRUE; - - /* multiline buffer */ - group->buf = flb_sds_create_size(FLB_ML_BUF_SIZE); - if (!group->buf) { - flb_error("cannot allocate multiline stream buffer in group %s", name); - flb_sds_destroy(group->name); - flb_free(group); - return NULL; - } - - /* msgpack buffer */ - msgpack_sbuffer_init(&group->mp_md_sbuf); - msgpack_packer_init(&group->mp_md_pck, &group->mp_md_sbuf, msgpack_sbuffer_write); - - msgpack_sbuffer_init(&group->mp_sbuf); - msgpack_packer_init(&group->mp_pck, &group->mp_sbuf, msgpack_sbuffer_write); - - mk_list_add(&group->_head, &mst->groups); - - return group; -} - -struct flb_ml_stream_group *flb_ml_stream_group_get(struct flb_ml_parser_ins *parser_i, - struct flb_ml_stream *mst, - msgpack_object *group_name) -{ - int len; - char *name; - struct flb_ml_parser *mlp; - struct mk_list *head; - struct flb_ml_stream_group *group = NULL; - - mlp = parser_i->ml_parser; - - /* If key_group was not defined, we already have a default group */ - if (!mlp->key_group || !group_name) { - group = mk_list_entry_first(&mst->groups, - struct flb_ml_stream_group, - _head); - return group; - } - - /* Lookup for a candidate group */ - len = group_name->via.str.size; - name = (char *)group_name->via.str.ptr; - - mk_list_foreach(head, &mst->groups) { - group = mk_list_entry(head, struct flb_ml_stream_group, _head); - if (flb_sds_cmp(group->name, name, len) == 0) { - return group; - } - else { - group = NULL; - continue; - } - } - - /* No group has been found, create a new one */ - if (mk_list_size(&mst->groups) >= FLB_ML_MAX_GROUPS) { - flb_error("[multiline] stream %s exceeded number of allowed groups (%i)", - mst->name, FLB_ML_MAX_GROUPS); - return NULL; - } - - group = stream_group_create(mst, name, len); - return group; -} - -static void stream_group_destroy(struct flb_ml_stream_group *group) -{ - if (group->name) { - flb_sds_destroy(group->name); - } - if (group->buf) { - flb_sds_destroy(group->buf); - } - - msgpack_sbuffer_destroy(&group->mp_md_sbuf); - msgpack_sbuffer_destroy(&group->mp_sbuf); - - mk_list_del(&group->_head); - flb_free(group); -} - -static void stream_group_destroy_all(struct flb_ml_stream *mst) -{ - struct mk_list *tmp; - struct mk_list *head; - struct flb_ml_stream_group *group; - - mk_list_foreach_safe(head, tmp, &mst->groups) { - group = mk_list_entry(head, struct flb_ml_stream_group, _head); - stream_group_destroy(group); - } -} - -static int stream_group_init(struct flb_ml_stream *stream) -{ - struct flb_ml_stream_group *group = NULL; - - mk_list_init(&stream->groups); - - /* create a default group */ - group = stream_group_create(stream, NULL, 0); - if (!group) { - flb_error("[multiline] error initializing default group for " - "stream '%s'", stream->name); - return -1; - } - - return 0; -} - -static struct flb_ml_stream *stream_create(struct flb_ml *ml, - uint64_t id, - struct flb_ml_parser_ins *parser, - int (*cb_flush) (struct flb_ml_parser *, - struct flb_ml_stream *, - void *cb_data, - char *buf_data, - size_t buf_size), - void *cb_data) -{ - int ret; - struct flb_ml_stream *stream; - - stream = flb_calloc(1, sizeof(struct flb_ml_stream)); - if (!stream) { - flb_errno(); - return NULL; - } - stream->ml = ml; - stream->id = id; - stream->parser = parser; - - /* Flush Callback and opaque data type */ - if (cb_flush) { - stream->cb_flush = cb_flush; - } - else { - stream->cb_flush = ml_flush_stdout; - } - stream->cb_data = cb_data; - - ret = stream_group_init(stream); - if (ret != 0) { - flb_free(stream); - return NULL; - } - - mk_list_add(&stream->_head, &parser->streams); - return stream; -} - -int flb_ml_stream_create(struct flb_ml *ml, - char *name, - int name_len, - int (*cb_flush) (struct flb_ml_parser *, - struct flb_ml_stream *, - void *cb_data, - char *buf_data, - size_t buf_size), - void *cb_data, - uint64_t *stream_id) -{ - uint64_t id; - struct mk_list *head; - struct mk_list *head_group; - struct flb_ml_stream *mst; - struct flb_ml_group *group; - struct flb_ml_parser_ins *parser; - - if (!name) { - return -1; - } - - if (name_len <= 0) { - name_len = strlen(name); - } - - /* Set the stream id by creating a hash using the name */ - id = cfl_hash_64bits(name, name_len); - - /* For every group and parser, create a stream for this stream_id/hash */ - mk_list_foreach(head, &ml->groups) { - group = mk_list_entry(head, struct flb_ml_group, _head); - mk_list_foreach(head_group, &group->parsers) { - parser = mk_list_entry(head_group, struct flb_ml_parser_ins, _head); - - /* Check if the stream already exists on the parser */ - if (flb_ml_stream_get(parser, id) != NULL) { - continue; - } - - /* Create the stream */ - mst = stream_create(ml, id, parser, cb_flush, cb_data); - if (!mst) { - flb_error("[multiline] could not create stream_id=%" PRIu64 - "for stream '%s' on parser '%s'", - *stream_id, name, parser->ml_parser->name); - return -1; - } - } - } - - *stream_id = id; - return 0; -} - -struct flb_ml_stream *flb_ml_stream_get(struct flb_ml_parser_ins *parser, - uint64_t stream_id) -{ - struct mk_list *head; - struct flb_ml_stream *mst = NULL; - - mk_list_foreach(head, &parser->streams) { - mst = mk_list_entry(head, struct flb_ml_stream, _head); - if (mst->id == stream_id) { - return mst; - } - } - - return NULL; -} - -void flb_ml_stream_id_destroy_all(struct flb_ml *ml, uint64_t stream_id) -{ - struct mk_list *tmp; - struct mk_list *head; - struct mk_list *head_group; - struct mk_list *head_stream; - struct flb_ml_group *group; - struct flb_ml_stream *mst; - struct flb_ml_parser_ins *parser_i; - - /* groups */ - mk_list_foreach(head, &ml->groups) { - group = mk_list_entry(head, struct flb_ml_group, _head); - - /* parser instances */ - mk_list_foreach(head_group, &group->parsers) { - parser_i = mk_list_entry(head_group, struct flb_ml_parser_ins, _head); - - /* streams */ - mk_list_foreach_safe(head_stream, tmp, &parser_i->streams) { - mst = mk_list_entry(head_stream, struct flb_ml_stream, _head); - if (mst->id != stream_id) { - continue; - } - - /* flush any pending data */ - flb_ml_flush_parser_instance(ml, parser_i, stream_id, FLB_TRUE); - - /* destroy internal groups of the stream */ - flb_ml_stream_destroy(mst); - } - } - } -} - -int flb_ml_stream_destroy(struct flb_ml_stream *mst) -{ - mk_list_del(&mst->_head); - if (mst->name) { - flb_sds_destroy(mst->name); - } - - /* destroy groups */ - stream_group_destroy_all(mst); - - flb_free(mst); - - return 0; -} diff --git a/fluent-bit/src/proxy/CMakeLists.txt b/fluent-bit/src/proxy/CMakeLists.txt deleted file mode 100644 index 2e07e7681..000000000 --- a/fluent-bit/src/proxy/CMakeLists.txt +++ /dev/null @@ -1,3 +0,0 @@ -if(FLB_PROXY_GO) - add_subdirectory(go) -endif() diff --git a/fluent-bit/src/proxy/go/CMakeLists.txt b/fluent-bit/src/proxy/go/CMakeLists.txt deleted file mode 100644 index 93e500353..000000000 --- a/fluent-bit/src/proxy/go/CMakeLists.txt +++ /dev/null @@ -1,10 +0,0 @@ -set(src - go.c) - -add_library(flb-plugin-proxy-go STATIC ${src}) -if(FLB_JEMALLOC) - target_link_libraries(flb-plugin-proxy-go libjemalloc) -endif() -if(FLB_REGEX) - target_link_libraries(flb-plugin-proxy-go onigmo-static) -endif() diff --git a/fluent-bit/src/proxy/go/go.c b/fluent-bit/src/proxy/go/go.c deleted file mode 100644 index 9d95a88af..000000000 --- a/fluent-bit/src/proxy/go/go.c +++ /dev/null @@ -1,289 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <fluent-bit/flb_info.h> -#include <fluent-bit/flb_config.h> -#include <fluent-bit/flb_plugin_proxy.h> -#include <fluent-bit/flb_output.h> -#include "./go.h" - -/* - * These functions needs to be moved to a better place, still in - * experimental mode. - * - * ------------------------start------------------------------------------------ - */ - -/* - * Go Plugin phases - * ================ - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * 1. FLBPluginRegister(context) - * 2. Inside FLBPluginRegister, it needs to register it self using Fluent Bit API - * where it basically set: - * - * - name: shortname of the plugin. - * - description: plugin description. - * - type: input, output, filter, whatever. - * - proxy: type of proxy e.g. GOLANG - * - flags: optional flags, not used by Go plugins at the moment. - * - * this is done through Go Wrapper: - * - * output.FLBPluginRegister(ctx, name, description, type, flags); - * - * 3. Plugin Initialization - */ -/*------------------------EOF------------------------------------------------*/ - -int proxy_go_output_register(struct flb_plugin_proxy *proxy, - struct flb_plugin_proxy_def *def) -{ - struct flbgo_output_plugin *plugin; - - plugin = flb_malloc(sizeof(struct flbgo_output_plugin)); - if (!plugin) { - return -1; - } - - /* - * Lookup the entry point function: - * - * - FLBPluginInit - * - FLBPluginFlush - * - FLBPluginFlushCtx - * - FLBPluginExit - * - * note: registration callback FLBPluginRegister() is resolved by the - * parent proxy interface. - */ - - plugin->cb_init = flb_plugin_proxy_symbol(proxy, "FLBPluginInit"); - if (!plugin->cb_init) { - flb_error("[go proxy]: could not load FLBPluginInit symbol"); - flb_free(plugin); - return -1; - } - - plugin->cb_flush = flb_plugin_proxy_symbol(proxy, "FLBPluginFlush"); - plugin->cb_flush_ctx = flb_plugin_proxy_symbol(proxy, "FLBPluginFlushCtx"); - plugin->cb_exit = flb_plugin_proxy_symbol(proxy, "FLBPluginExit"); - plugin->cb_exit_ctx = flb_plugin_proxy_symbol(proxy, "FLBPluginExitCtx"); - plugin->name = flb_strdup(def->name); - - /* This Go plugin context is an opaque data for the parent proxy */ - proxy->data = plugin; - - return 0; -} - -int proxy_go_output_init(struct flb_plugin_proxy *proxy) -{ - int ret; - struct flbgo_output_plugin *plugin = proxy->data; - - /* set the API */ - plugin->api = proxy->api; - plugin->o_ins = proxy->instance; - // In order to avoid having the whole instance as part of the ABI we - // copy the context pointer into the plugin. - plugin->context = ((struct flb_output_instance *)proxy->instance)->context; - - ret = plugin->cb_init(plugin); - if (ret <= 0) { - flb_error("[go proxy]: plugin '%s' failed to initialize", - plugin->name); - flb_free(plugin); - return -1; - } - - return ret; -} - -int proxy_go_output_flush(struct flb_plugin_proxy_context *ctx, - const void *data, size_t size, - const char *tag, int tag_len) -{ - int ret; - char *buf; - struct flbgo_output_plugin *plugin = ctx->proxy->data; - - /* temporary buffer for the tag */ - buf = flb_malloc(tag_len + 1); - if (!buf) { - flb_errno(); - return -1; - } - - memcpy(buf, tag, tag_len); - buf[tag_len] = '\0'; - - if (plugin->cb_flush_ctx) { - ret = plugin->cb_flush_ctx(ctx->remote_context, data, size, buf); - } - else { - ret = plugin->cb_flush(data, size, buf); - } - flb_free(buf); - return ret; -} - -int proxy_go_output_destroy(struct flb_plugin_proxy_context *ctx) -{ - int ret = 0; - struct flbgo_output_plugin *plugin; - - plugin = (struct flbgo_output_plugin *) ctx->proxy->data; - flb_debug("[GO] running exit callback"); - - if (plugin->cb_exit_ctx) { - ret = plugin->cb_exit_ctx(ctx->remote_context); - } - else if (plugin->cb_exit) { - ret = plugin->cb_exit(); - } - return ret; -} - -void proxy_go_output_unregister(void *data) { - struct flbgo_output_plugin *plugin; - - plugin = (struct flbgo_output_plugin *) data; - flb_free(plugin->name); - flb_free(plugin); -} - -int proxy_go_input_register(struct flb_plugin_proxy *proxy, - struct flb_plugin_proxy_def *def) -{ - struct flbgo_input_plugin *plugin; - - plugin = flb_malloc(sizeof(struct flbgo_input_plugin)); - if (!plugin) { - return -1; - } - - /* - * Lookup the entry point function: - * - * - FLBPluginInit - * - FLBPluginInputCallback - * - FLBPluginExit - * - * note: registration callback FLBPluginRegister() is resolved by the - * parent proxy interface. - */ - - plugin->cb_init = flb_plugin_proxy_symbol(proxy, "FLBPluginInit"); - if (!plugin->cb_init) { - flb_error("[go proxy]: could not load FLBPluginInit symbol"); - flb_free(plugin); - return -1; - } - - plugin->cb_collect = flb_plugin_proxy_symbol(proxy, "FLBPluginInputCallback"); - plugin->cb_cleanup = flb_plugin_proxy_symbol(proxy, "FLBPluginInputCleanupCallback"); - plugin->cb_exit = flb_plugin_proxy_symbol(proxy, "FLBPluginExit"); - plugin->name = flb_strdup(def->name); - - /* This Go plugin context is an opaque data for the parent proxy */ - proxy->data = plugin; - - return 0; -} - -int proxy_go_input_init(struct flb_plugin_proxy *proxy) -{ - int ret; - struct flbgo_input_plugin *plugin = proxy->data; - - /* set the API */ - plugin->api = proxy->api; - plugin->i_ins = proxy->instance; - // In order to avoid having the whole instance as part of the ABI we - // copy the context pointer into the plugin. - plugin->context = ((struct flb_input_instance *)proxy->instance)->context; - - ret = plugin->cb_init(plugin); - if (ret <= 0) { - flb_error("[go proxy]: plugin '%s' failed to initialize", - plugin->name); - flb_free(plugin); - return -1; - } - - return ret; -} - -int proxy_go_input_collect(struct flb_plugin_proxy *ctx, - void **collected_data, size_t *len) -{ - int ret; - void *data = NULL; - struct flbgo_input_plugin *plugin = ctx->data; - - ret = plugin->cb_collect(&data, len); - - *collected_data = data; - - return ret; -} - -int proxy_go_input_cleanup(struct flb_plugin_proxy *ctx, - void *allocated_data) -{ - int ret = 0; - struct flbgo_input_plugin *plugin = ctx->data; - - if (plugin->cb_cleanup) { - ret = plugin->cb_cleanup(allocated_data); - } - else { - /* If cleanup callback is not registered, we need to cleanup - * allocated memory on fluent-bit side. */ - if (allocated_data != NULL) { - free(allocated_data); - } - } - - return ret; -} - -int proxy_go_input_destroy(struct flb_plugin_input_proxy_context *ctx) -{ - int ret = 0; - struct flbgo_input_plugin *plugin; - - plugin = (struct flbgo_input_plugin *) ctx->proxy->data; - flb_debug("[GO] running exit callback"); - - if (plugin->cb_exit) { - ret = plugin->cb_exit(); - } - return ret; -} - -void proxy_go_input_unregister(void *data) { - struct flbgo_input_plugin *plugin; - - plugin = (struct flbgo_input_plugin *) data; - flb_free(plugin->name); - flb_free(plugin); -} diff --git a/fluent-bit/src/proxy/go/go.h b/fluent-bit/src/proxy/go/go.h deleted file mode 100644 index 4c3fedb23..000000000 --- a/fluent-bit/src/proxy/go/go.h +++ /dev/null @@ -1,72 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_PROXY_GO_H -#define FLB_PROXY_GO_H - -#include <fluent-bit/flb_config.h> -#include <fluent-bit/flb_plugin_proxy.h> - -struct flbgo_output_plugin { - char *name; - void *api; - void *o_ins; - struct flb_plugin_proxy_context *context; - - int (*cb_init)(); - int (*cb_flush)(const void *, size_t, const char *); - int (*cb_flush_ctx)(void *, const void *, size_t, char *); - int (*cb_exit)(); - int (*cb_exit_ctx)(void *); -}; - -struct flbgo_input_plugin { - char *name; - void *api; - void *i_ins; - struct flb_plugin_proxy_context *context; - - int (*cb_init)(); - int (*cb_collect)(void **, size_t *); - int (*cb_cleanup)(void *); - int (*cb_exit)(); -}; - -int proxy_go_output_register(struct flb_plugin_proxy *proxy, - struct flb_plugin_proxy_def *def); - -int proxy_go_output_init(struct flb_plugin_proxy *proxy); - -int proxy_go_output_flush(struct flb_plugin_proxy_context *ctx, - const void *data, size_t size, - const char *tag, int tag_len); -int proxy_go_output_destroy(struct flb_plugin_proxy_context *ctx); -void proxy_go_output_unregister(void *data); - -int proxy_go_input_register(struct flb_plugin_proxy *proxy, - struct flb_plugin_proxy_def *def); - -int proxy_go_input_init(struct flb_plugin_proxy *proxy); -int proxy_go_input_collect(struct flb_plugin_proxy *ctx, - void **collected_data, size_t *len); -int proxy_go_input_cleanup(struct flb_plugin_proxy *ctx, - void *allocated_data); -int proxy_go_input_destroy(struct flb_plugin_input_proxy_context *ctx); -void proxy_go_input_unregister(void *data); -#endif diff --git a/fluent-bit/src/record_accessor/CMakeLists.txt b/fluent-bit/src/record_accessor/CMakeLists.txt deleted file mode 100644 index 9eb3825d9..000000000 --- a/fluent-bit/src/record_accessor/CMakeLists.txt +++ /dev/null @@ -1,31 +0,0 @@ -flex_target(lexer ra.l "${CMAKE_CURRENT_BINARY_DIR}/ra_lex.c" - DEFINES_FILE "${CMAKE_CURRENT_BINARY_DIR}/ra_lex.h" - ) -bison_target(parser ra.y "${CMAKE_CURRENT_BINARY_DIR}/ra_parser.c") - -set(sources - flb_ra_parser.c - ) - -if(CMAKE_SYSTEM_NAME MATCHES "Windows") - FLB_DEFINITION(YY_NO_UNISTD_H) - message(STATUS "Specifying YY_NO_UNISTD_H") -endif() - -include_directories( - ${CMAKE_CURRENT_SOURCE_DIR} - ${CMAKE_CURRENT_BINARY_DIR} - ) - -add_library(flb-ra-parser STATIC - ${sources} - "${CMAKE_CURRENT_BINARY_DIR}/ra_lex.c" - "${CMAKE_CURRENT_BINARY_DIR}/ra_parser.c" - ) - -add_flex_bison_dependency(lexer parser) -add_dependencies(flb-ra-parser onigmo-static) - -if(FLB_JEMALLOC) - target_link_libraries(flb-ra-parser libjemalloc) -endif() diff --git a/fluent-bit/src/record_accessor/flb_ra_parser.c b/fluent-bit/src/record_accessor/flb_ra_parser.c deleted file mode 100644 index 9f7142950..000000000 --- a/fluent-bit/src/record_accessor/flb_ra_parser.c +++ /dev/null @@ -1,365 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <fluent-bit/flb_info.h> -#include <fluent-bit/flb_slist.h> -#include <fluent-bit/flb_mem.h> -#include <fluent-bit/flb_sds.h> -#include <fluent-bit/flb_log.h> -#include <fluent-bit/record_accessor/flb_ra_parser.h> - -#include "ra_parser.h" -#include "ra_lex.h" - -int flb_ra_parser_subkey_count(struct flb_ra_parser *rp) -{ - if (rp == NULL || rp->key == NULL) { - return -1; - } - else if (rp->type != FLB_RA_PARSER_KEYMAP) { - return 0; - } - else if(rp->key->subkeys == NULL) { - return -1; - } - - return mk_list_size(rp->key->subkeys); -} - -void flb_ra_parser_dump(struct flb_ra_parser *rp) -{ - struct mk_list *head; - struct flb_ra_key *key; - struct flb_ra_subentry *entry; - - key = rp->key; - if (rp->type == FLB_RA_PARSER_STRING) { - printf("type : STRING\n"); - printf("string : '%s'\n", key->name); - } - if (rp->type == FLB_RA_PARSER_REGEX_ID) { - printf("type : REGEX_ID\n"); - printf("integer : '%i'\n", rp->id); - } - if (rp->type == FLB_RA_PARSER_TAG) { - printf("type : TAG\n"); - } - if (rp->type == FLB_RA_PARSER_TAG_PART) { - printf("type : TAG[%i]\n", rp->id); - } - else if (rp->type == FLB_RA_PARSER_KEYMAP) { - printf("type : KEYMAP\n"); - if (rp->key) { - printf("key name : %s\n", key->name); - mk_list_foreach(head, key->subkeys) { - entry = mk_list_entry(head, struct flb_ra_subentry, _head); - if (entry->type == FLB_RA_PARSER_STRING) { - printf(" - subkey : %s\n", entry->str); - } - else if (entry->type == FLB_RA_PARSER_ARRAY_ID) { - printf(" - array id: %i\n", entry->array_id); - } - } - } - } -} - -static void ra_parser_subentry_destroy_all(struct mk_list *list) -{ - struct mk_list *tmp; - struct mk_list *head; - struct flb_ra_subentry *entry; - - mk_list_foreach_safe(head, tmp, list) { - entry = mk_list_entry(head, struct flb_ra_subentry, _head); - mk_list_del(&entry->_head); - if (entry->type == FLB_RA_PARSER_STRING) { - flb_sds_destroy(entry->str); - } - flb_free(entry); - } -} - -int flb_ra_parser_subentry_add_string(struct flb_ra_parser *rp, char *key) -{ - struct flb_ra_subentry *entry; - - entry = flb_malloc(sizeof(struct flb_ra_subentry)); - if (!entry) { - flb_errno(); - return -1; - } - - entry->type = FLB_RA_PARSER_STRING; - entry->str = flb_sds_create(key); - if (!entry->str) { - flb_errno(); - flb_free(entry); - return -1; - } - mk_list_add(&entry->_head, rp->slist); - - return 0; -} - -int flb_ra_parser_subentry_add_array_id(struct flb_ra_parser *rp, int id) -{ - struct flb_ra_subentry *entry; - - entry = flb_malloc(sizeof(struct flb_ra_subentry)); - if (!entry) { - flb_errno(); - return -1; - } - - entry->type = FLB_RA_PARSER_ARRAY_ID; - entry->array_id = id; - mk_list_add(&entry->_head, rp->slist); - - return 0; -} - - -struct flb_ra_key *flb_ra_parser_key_add(struct flb_ra_parser *rp, char *key) -{ - struct flb_ra_key *k; - - k = flb_malloc(sizeof(struct flb_ra_key)); - if (!k) { - flb_errno(); - return NULL; - } - - k->name = flb_sds_create(key); - if (!k->name) { - flb_errno(); - flb_free(k); - return NULL; - } - k->subkeys = NULL; - - return k; -} - -struct flb_ra_array *flb_ra_parser_array_add(struct flb_ra_parser *rp, int index) -{ - struct flb_ra_array *arr; - - if (index < 0) { - return NULL; - } - - arr = flb_malloc(sizeof(struct flb_ra_array)); - if (!arr) { - flb_errno(); - return NULL; - } - - arr->index = index; - arr->subkeys = NULL; - - return arr; -} - -struct flb_ra_key *flb_ra_parser_string_add(struct flb_ra_parser *rp, - char *str, int len) -{ - struct flb_ra_key *k; - - k = flb_malloc(sizeof(struct flb_ra_key)); - if (!k) { - flb_errno(); - return NULL; - } - - k->name = flb_sds_create_len(str, len); - if (!k->name) { - flb_errno(); - flb_free(k); - return NULL; - } - k->subkeys = NULL; - - return k; -} - -static struct flb_ra_parser *flb_ra_parser_create() -{ - struct flb_ra_parser *rp; - - rp = flb_calloc(1, sizeof(struct flb_ra_parser)); - if (!rp) { - flb_errno(); - return NULL; - } - rp->type = -1; - rp->key = NULL; - rp->slist = flb_malloc(sizeof(struct mk_list)); - if (!rp->slist) { - flb_errno(); - flb_free(rp); - return NULL; - } - mk_list_init(rp->slist); - - return rp; -} - -struct flb_ra_parser *flb_ra_parser_string_create(char *str, int len) -{ - struct flb_ra_parser *rp; - - rp = flb_ra_parser_create(); - if (!rp) { - flb_error("[record accessor] could not create string context"); - return NULL; - } - - rp->type = FLB_RA_PARSER_STRING; - rp->key = flb_malloc(sizeof(struct flb_ra_key)); - if (!rp->key) { - flb_errno(); - flb_ra_parser_destroy(rp); - return NULL; - } - rp->key->subkeys = NULL; - rp->key->name = flb_sds_create_len(str, len); - if (!rp->key->name) { - flb_ra_parser_destroy(rp); - return NULL; - } - - return rp; -} - -struct flb_ra_parser *flb_ra_parser_regex_id_create(int id) -{ - struct flb_ra_parser *rp; - - rp = flb_ra_parser_create(); - if (!rp) { - flb_error("[record accessor] could not create string context"); - return NULL; - } - - rp->type = FLB_RA_PARSER_REGEX_ID; - rp->id = id; - return rp; -} - -struct flb_ra_parser *flb_ra_parser_tag_create() -{ - struct flb_ra_parser *rp; - - rp = flb_ra_parser_create(); - if (!rp) { - flb_error("[record accessor] could not create tag context"); - return NULL; - } - - rp->type = FLB_RA_PARSER_TAG; - return rp; -} - -struct flb_ra_parser *flb_ra_parser_tag_part_create(int id) -{ - struct flb_ra_parser *rp; - - rp = flb_ra_parser_create(); - if (!rp) { - flb_error("[record accessor] could not create tag context"); - return NULL; - } - - rp->type = FLB_RA_PARSER_TAG_PART; - rp->id = id; - - return rp; -} - -struct flb_ra_parser *flb_ra_parser_meta_create(char *str, int len) -{ - int ret; - yyscan_t scanner; - YY_BUFFER_STATE buf; - flb_sds_t s; - struct flb_ra_parser *rp; - struct flb_ra_key *key; - - rp = flb_ra_parser_create(); - if (!rp) { - flb_error("[record accessor] could not create meta context"); - return NULL; - } - - /* Temporal buffer of string with fixed length */ - s = flb_sds_create_len(str, len); - if (!s) { - flb_errno(); - flb_ra_parser_destroy(rp); - return NULL; - } - - /* Flex/Bison work */ - flb_ra_lex_init(&scanner); - buf = flb_ra__scan_string(s, scanner); - - ret = flb_ra_parse(rp, s, scanner); - - /* release resources */ - flb_sds_destroy(s); - flb_ra__delete_buffer(buf, scanner); - flb_ra_lex_destroy(scanner); - - /* Finish structure mapping */ - if (rp->type == FLB_RA_PARSER_KEYMAP) { - if (rp->key) { - key = rp->key; - key->subkeys = rp->slist; - rp->slist = NULL; - } - } - - if (ret != 0) { - flb_ra_parser_destroy(rp); - return NULL; - } - - return rp; -} - -void flb_ra_parser_destroy(struct flb_ra_parser *rp) -{ - struct flb_ra_key *key; - - key = rp->key; - if (key) { - flb_sds_destroy(key->name); - if (key->subkeys) { - ra_parser_subentry_destroy_all(key->subkeys); - flb_free(key->subkeys); - } - flb_free(rp->key); - } - if (rp->slist) { - ra_parser_subentry_destroy_all(rp->slist); - flb_free(rp->slist); - } - flb_free(rp); -} diff --git a/fluent-bit/src/record_accessor/ra.l b/fluent-bit/src/record_accessor/ra.l deleted file mode 100644 index b35d6bd26..000000000 --- a/fluent-bit/src/record_accessor/ra.l +++ /dev/null @@ -1,69 +0,0 @@ -%option prefix="flb_ra_" -%option caseless -%{ -#include <stdio.h> -#include <stdbool.h> -#include <fluent-bit/flb_str.h> -#include <fluent-bit/flb_log.h> -#include <fluent-bit/record_accessor/flb_ra_parser.h> - -#include "ra_parser.h" - -static inline char *remove_dup_quotes(const char *s, size_t n) -{ - char *str; - int dups; - int i, j; - - dups = 0; - for (i = 0; i < n; i++) { - if (s[i] == '\'') { - dups++; - i++; - } - } - - str = (char *) flb_malloc(n - dups + 1); - if (!str) { - return NULL; - } - - j = 0; - for (i = 0; i < n; i++, j++) { - if (s[i] == '\'') { - str[j] = '\''; - i++; - } else { - str[j] = s[i]; - } - } - str[j] = '\0'; - - return str; -} - -%} - -%option 8bit reentrant bison-bridge -%option warn noyywrap nodefault -%option nounput -%option noinput - -%% - -[1-9][0-9]*|0 { yylval->integer = atoi(yytext); return INTEGER; } -\'([^']|'{2})*\' { yylval->string = remove_dup_quotes(yytext + 1, yyleng - 2); return STRING; } -[_A-Za-z][A-Za-z0-9_.\-/]* { yylval->string = flb_strdup(yytext); return IDENTIFIER; } - -"$" | -"[" | -"]" | -"." | -"," | -";" { return yytext[0]; } -\n -[ \t]+ /* ignore whitespace */; - -. flb_error("[record accessor] bad input character '%s' at line %d", yytext, yylineno); - -%% diff --git a/fluent-bit/src/record_accessor/ra.y b/fluent-bit/src/record_accessor/ra.y deleted file mode 100644 index 9c4e25b87..000000000 --- a/fluent-bit/src/record_accessor/ra.y +++ /dev/null @@ -1,99 +0,0 @@ -%define api.pure full -%name-prefix "flb_ra_" - -%parse-param { struct flb_ra_parser *rp }; -%parse-param { const char *str }; -%lex-param { void *scanner } -%parse-param { void *scanner } - -%{ -#include <stdio.h> -#include <stdlib.h> - -#include <fluent-bit/flb_info.h> -#include <fluent-bit/flb_mem.h> -#include <fluent-bit/flb_log.h> -#include <fluent-bit/flb_slist.h> -#include <fluent-bit/record_accessor/flb_ra_parser.h> - -#include "ra_parser.h" -#include "ra_lex.h" - -extern int flb_ra_lex(); - -void flb_ra_error(struct flb_ra_parser *rp, const char *query, void *scanner, - const char *str) -{ - flb_error("[record accessor] %s at '%s'", str, query); -} - -%} /* EOF C code */ - - -/* Known Tokens (refer to sql.l) */ - -/* Keywords */ -%token IDENTIFIER STRING INTEGER - -%define parse.error verbose - -/* Union and field types */ -%union -{ - int integer; - float fval; - char *string; - struct flb_sp_cmd *cmd; - struct flb_exp *expression; -} - -%type <string> IDENTIFIER -%type <integer> INTEGER -%type <string> STRING -%type <string> record_key - -%destructor { flb_free ($$); } IDENTIFIER -%destructor { flb_free ($$); } STRING - -%% /* rules section */ - -statements: record_accessor - -/* Parse record accessor string: $key, $key['x'], $key['x'][...] */ -record_accessor: record_key - record_key: - '$' IDENTIFIER - { - void *key; - - rp->type = FLB_RA_PARSER_KEYMAP; - key = flb_ra_parser_key_add(rp, $2); - if (key) { - rp->key = key; - } - flb_free($2); - } - | - '$' IDENTIFIER record_subkey - { - void *key; - rp->type = FLB_RA_PARSER_KEYMAP; - key = flb_ra_parser_key_add(rp, $2); - if (key) { - rp->key = key; - } - flb_free($2); - } - record_subkey: record_subkey record_subkey_index | record_subkey_index - record_subkey_index: - '[' STRING ']' - { - flb_ra_parser_subentry_add_string(rp, $2); - flb_free($2); - } - | - '[' INTEGER ']' - { - flb_ra_parser_subentry_add_array_id(rp, $2); - } - ; diff --git a/fluent-bit/src/stream_processor/CMakeLists.txt b/fluent-bit/src/stream_processor/CMakeLists.txt deleted file mode 100644 index de2c2fe38..000000000 --- a/fluent-bit/src/stream_processor/CMakeLists.txt +++ /dev/null @@ -1,21 +0,0 @@ -project(stream-processor C) - -include_directories(${CMAKE_CURRENT_SOURCE_DIR}) - -add_subdirectory(parser) - -set(src - flb_sp.c - flb_sp_key.c - flb_sp_func_time.c - flb_sp_func_record.c - flb_sp_stream.c - flb_sp_snapshot.c - flb_sp_window.c - flb_sp_groupby.c - flb_sp_aggregate_func.c - ) - -add_library(flb-sp STATIC ${src}) -target_link_libraries(flb-sp rbtree) -target_link_libraries(flb-sp flb-sp-parser) diff --git a/fluent-bit/src/stream_processor/README.md b/fluent-bit/src/stream_processor/README.md deleted file mode 100644 index d39a51ef5..000000000 --- a/fluent-bit/src/stream_processor/README.md +++ /dev/null @@ -1,84 +0,0 @@ -## SQL Statement Syntax - -The following is the SQL statement syntax supported by Fluent Bit stream processor in EBNF form. For readability, we assume the conventional definition for integer, float and string values. A single quote in a constant string literal has to be escaped with an extra one. For instance, the string representation of `O'Keefe` in the query will be `'O''Keefe'`. - -```xml -<sql_stmt> := <create> | <select> -<create> := CREATE STREAM <id> AS <select> | CREATE STREAM <id> WITH (<properties>) AS <select> -<properties> := <property> | <property>, <properties> -<property> := <id> = '<id>' -<select> := SELECT <keys> FROM <source> [WHERE <condition>] - [WINDOW TUMBLING (<integer> SECOND) | WINDOW HOPPING (<integer> SECOND, ADVANCE BY <integer> SECOND)] - [GROUP BY <record_keys>] -<keys> := '*' | <record_keys> -<record_keys> := <record_key> | <record_key>, <record_keys> -<record_key> := <exp> | <exp> AS <id> -<exp> := <key> | <fun> -<fun> := AVG(<key>) | SUM(<key>) | COUNT(<key>) | COUNT(*) | MIN(<key>) | MAX(<key>) | TIMESERIES_FORECAST(<key>, <integer>) -<source> := STREAM:<id> | TAG:<id> -<condition> := <key> | <value> | <key> <relation> <value> | (<condition>) - | NOT <condition> | <condition> AND <condition> | <condition> OR <condition> - | @record.contains(<key>) | <id> IS NULL | <id> IS NOT NULL -<key> := <id> | <id><subkey-idx> -<subkey-idx> := [<id>] | <subkey-idx>[<id>] -<relation> := = | != | <> | < | <= | > | >= -<id> := <letter> <characters> -<characters> := <letter> | <digit> | _ | <characters> <characters> -<value> := true | false | <integer> | <float> | '<string>' -``` - -In addition to the common aggregation functions, Stream Processor provides the timeseries function `TIMESERIES_FORECAST`, which uses [simple linear regression algorithm](<https://en.wikipedia.org/wiki/Simple_linear_regression) to predict the value of a (dependent) variable in future. - -### Timeseries Functions - -| name | description | -| ------------------------- | ------------------------------------------------------ | -| TIMESERIES_FORECAST(x, t) | forecasts the value of x at current time + t seconds | - -### Time Functions - -| name | description | example | -| ---------------- | ------------------------------------------------- | ------------------- | -| NOW() | adds system time using format: %Y-%m-%d %H:%M:%S | 2019-03-09 21:36:05 | -| UNIX_TIMESTAMP() | add current Unix timestamp | 1552196165 | - -### Record Functions - -| name | description | example | -| ------------- | ------------------------------------------------------------ | ----------------- | -| RECORD_TAG() | append Tag string associated to the record | samples | -| RECORD_TIME() | append record Timestamp in _double_ format: seconds.nanoseconds | 1552196165.705683 | - -## Type of windows - -FluentBit stream processor has implemented two time-based windows: hopping window and tumbling window. - -### Hopping window - -In hopping window (also known as sliding window), records are stored in a time window of the interval in seconds defined as the parameter. The `ADVANCE BY` parameter determines the time the window slides forward. Aggregation functions are computed over the records inside a window, and reported right before window moves. - -For example. the hopping window `WINDOW HOPPING (10 SECOND, ADVANCE BY 2 SECOND)` behaves like this: - -``` -[ x x x x x ... x x x x x ] -<--------- 10 sec --------> - [ x x x x x ... x x x x x ] -<- 2 sec -><--------- 10 sec --------> - [ x x x x x ... x x x x x ] - <- 2 sec -><--------- 10 sec --------> -``` - -### Tumbling window - -A tumbling window is similar to a hopping window where `ADVANCE BY` value is the same as the window size. That means the new window doesn't include any record from the previous one. - -For example. the tumbling window `WINDOW TUMBLING (10 SECOND)` works like this: - -``` -[ x x x x x ... x x x x x ] -<--------- 10 sec --------> - [ x x x x x ... x x x x x ] - <--------- 10 sec --------> - [ x x x x x ... x x x x x ] - <--------- 10 sec --------> -``` diff --git a/fluent-bit/src/stream_processor/flb_sp.c b/fluent-bit/src/stream_processor/flb_sp.c deleted file mode 100644 index 00eb2f18b..000000000 --- a/fluent-bit/src/stream_processor/flb_sp.c +++ /dev/null @@ -1,2157 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <fluent-bit/flb_info.h> -#include <fluent-bit/flb_log.h> -#include <fluent-bit/flb_sds.h> -#include <fluent-bit/flb_mem.h> -#include <fluent-bit/flb_slist.h> -#include <fluent-bit/flb_utils.h> -#include <fluent-bit/flb_time.h> -#include <fluent-bit/flb_input.h> -#include <fluent-bit/flb_pack.h> -#include <fluent-bit/flb_router.h> -#include <fluent-bit/flb_config_format.h> -#include <fluent-bit/stream_processor/flb_sp.h> -#include <fluent-bit/stream_processor/flb_sp_key.h> -#include <fluent-bit/stream_processor/flb_sp_stream.h> -#include <fluent-bit/stream_processor/flb_sp_snapshot.h> -#include <fluent-bit/stream_processor/flb_sp_parser.h> -#include <fluent-bit/stream_processor/flb_sp_func_time.h> -#include <fluent-bit/stream_processor/flb_sp_func_record.h> -#include <fluent-bit/stream_processor/flb_sp_aggregate_func.h> -#include <fluent-bit/stream_processor/flb_sp_window.h> -#include <fluent-bit/stream_processor/flb_sp_groupby.h> - -#include <stdlib.h> -#include <sys/types.h> -#include <sys/stat.h> -#ifndef _WIN32 -#include <unistd.h> -#endif - -/* don't do this at home */ -#define pack_uint16(buf, d) _msgpack_store16(buf, (uint16_t) d) -#define pack_uint32(buf, d) _msgpack_store32(buf, (uint32_t) d) - -/* String type to numerical conversion */ -#define FLB_STR_INT 1 -#define FLB_STR_FLOAT 2 - -/* Read and process file system configuration file */ -static int sp_config_file(struct flb_config *config, struct flb_sp *sp, - const char *file) -{ - int ret; - flb_sds_t name; - flb_sds_t exec; - char *cfg = NULL; - char tmp[PATH_MAX + 1]; - struct stat st; - struct mk_list *head; - struct flb_sp_task *task; - struct flb_cf *cf; - struct flb_cf_section *section; - -#ifndef FLB_HAVE_STATIC_CONF - ret = stat(file, &st); - if (ret == -1 && errno == ENOENT) { - /* Try to resolve the real path (if exists) */ - if (file[0] == '/') { - flb_error("[sp] cannot open configuration file: %s", file); - return -1; - } - - if (config->conf_path) { - snprintf(tmp, PATH_MAX, "%s%s", config->conf_path, file); - cfg = tmp; - } - } - else { - cfg = (char *) file; - } - - cf = flb_cf_create_from_file(NULL, cfg); -#else - cf = flb_config_static_open(file); -#endif - - if (!cf) { - return -1; - } - - /* Read all 'stream_task' sections */ - mk_list_foreach(head, &cf->sections) { - section = mk_list_entry(head, struct flb_cf_section, _head); - if (strcasecmp(section->name, "stream_task") != 0) { - continue; - } - - name = NULL; - exec = NULL; - - /* name */ - name = flb_cf_section_property_get_string(cf, section, "name"); - if (!name) { - flb_error("[sp] task 'name' not found in file '%s'", cfg); - goto fconf_error; - } - - /* exec */ - exec = flb_cf_section_property_get_string(cf, section, "exec"); - if (!exec) { - flb_error("[sp] task '%s' don't have an 'exec' command", name); - goto fconf_error; - } - - /* Register the task */ - task = flb_sp_task_create(sp, name, exec); - if (!task) { - goto fconf_error; - } - flb_sds_destroy(name); - flb_sds_destroy(exec); - name = NULL; - exec = NULL; - } - - flb_cf_destroy(cf); - return 0; - -fconf_error: - if (name) { - flb_sds_destroy(name); - } - if (exec) { - flb_sds_destroy(exec); - } - flb_cf_destroy(cf); - return -1; -} - -static int sp_task_to_instance(struct flb_sp_task *task, struct flb_sp *sp) -{ - struct mk_list *head; - struct flb_input_instance *in; - - if (task->cmd->source_type != FLB_SP_STREAM) { - return -1; - } - - mk_list_foreach(head, &sp->config->inputs) { - in = mk_list_entry(head, struct flb_input_instance, _head); - if (in->alias) { - if (strcasecmp(in->alias, task->cmd->source_name) == 0) { - task->source_instance = in; - return 0; - } - } - - if (strcasecmp(in->name, task->cmd->source_name) == 0) { - task->source_instance = in; - return 0; - } - } - - return -1; -} - -static void sp_info(struct flb_sp *sp) -{ - struct mk_list *head; - struct flb_sp_task *task; - - flb_info("[sp] stream processor started"); - - mk_list_foreach(head, &sp->tasks) { - task = mk_list_entry(head, struct flb_sp_task, _head); - flb_info("[sp] registered task: %s", task->name); - } -} - -int subkeys_compare(struct mk_list *subkeys1, struct mk_list *subkeys2) -{ - int i; - struct flb_slist_entry *entry1; - struct flb_slist_entry *entry2; - - if (!subkeys1 && !subkeys2) { - return 0; - } - - if (!subkeys1 || !subkeys2) { - return -1; - } - - if (mk_list_size(subkeys1) != mk_list_size(subkeys2)) { - return -1; - } - - entry1 = mk_list_entry_first(subkeys1, struct flb_slist_entry, _head); - entry2 = mk_list_entry_first(subkeys2, struct flb_slist_entry, _head); - - for (i = 0; i < mk_list_size(subkeys1); i++) { - if (flb_sds_cmp(entry1->str, entry2->str, flb_sds_len(entry2->str)) != 0) { - return -1; - } - - entry1 = mk_list_entry_next(&entry1->_head, struct flb_slist_entry, - _head, subkeys1); - entry2 = mk_list_entry_next(&entry2->_head, struct flb_slist_entry, - _head, subkeys2); - } - - return 0; -} - -static int sp_cmd_aggregated_keys(struct flb_sp_cmd *cmd) -{ - int aggr = 0; - int not_aggr = 0; - struct mk_list *head; - struct mk_list *head_gb; - struct flb_sp_cmd_key *key; - struct flb_sp_cmd_gb_key *gb_key; - - mk_list_foreach(head, &cmd->keys) { - key = mk_list_entry(head, struct flb_sp_cmd_key, _head); - if (key->time_func > 0 || key->record_func > 0) { - continue; - } - - if (key->aggr_func > 0) { - /* AVG, SUM, COUNT or timeseries functions */ - aggr++; - } - else { - mk_list_foreach(head_gb, &cmd->gb_keys) { - gb_key = mk_list_entry(head_gb, struct flb_sp_cmd_gb_key, _head); - - if (!key->name) { /* Key name is a wildcard '*' */ - break; - } - - if (flb_sds_cmp(key->name, gb_key->name, - flb_sds_len(gb_key->name)) == 0) { - if (subkeys_compare(key->subkeys, gb_key->subkeys) != 0) { - continue; - } - - not_aggr--; - - /* Map key selector with group-by */ - key->gb_key = gb_key; - break; - } - } - - not_aggr++; - } - } - - /* - * if some aggregated function is required, not aggregated keys are - * not allowed so we return an error (-1). - */ - if (aggr > 0 && not_aggr == 0) { - return aggr; - } - else if (aggr > 0 && not_aggr > 0) { - return -1; - } - - return 0; -} - -/* - * Convert a string to a numerical representation: - * - * - if output number is an integer, 'i' is set and returns FLB_STR_INT - * - if output number is a float, 'd' is set and returns FLB_STR_FLOAT - * - if no conversion is possible (not a number), returns -1 - */ -static int string_to_number(const char *str, int len, int64_t *i, double *d) -{ - int c; - int dots = 0; - char *end; - int64_t i_out; - double d_out; - - /* Detect if this is a floating point number */ - for (c = 0; c < len; c++) { - if (str[c] == '.') { - dots++; - } - } - - if (dots > 1) { - return -1; - } - else if (dots == 1) { - /* Floating point number */ - errno = 0; - d_out = strtold(str, &end); - - /* Check for various possible errors */ - if ((errno == ERANGE || (errno != 0 && d_out == 0))) { - return -1; - } - - if (end == str) { - return -1; - } - - *d = d_out; - return FLB_STR_FLOAT; - } - else { - /* Integer */ - errno = 0; - i_out = strtoll(str, &end, 10); - - /* Check for various possible errors */ - if ((errno == ERANGE || (errno != 0 && i_out == 0))) { - return -1; - } - - if (end == str) { - return -1; - } - - *i = i_out; - return FLB_STR_INT; - } - - return -1; -} - -/* - * Convert a msgpack object value to a number 'if possible'. The conversion - * result is either stored on 'i' for 64 bits integers or in 'd' for - * float/doubles. - * - * This function aims to take care of strings representing a value too. - */ -static int object_to_number(msgpack_object obj, int64_t *i, double *d, - int convert_str_to_num) -{ - int ret; - int64_t i_out; - double d_out; - char str_num[20]; - - if (obj.type == MSGPACK_OBJECT_POSITIVE_INTEGER || - obj.type == MSGPACK_OBJECT_NEGATIVE_INTEGER) { - *i = obj.via.i64; - return FLB_STR_INT; - } - else if (obj.type == MSGPACK_OBJECT_FLOAT32 || - obj.type == MSGPACK_OBJECT_FLOAT) { - *d = obj.via.f64; - return FLB_STR_FLOAT; - } - else if (obj.type == MSGPACK_OBJECT_STR && convert_str_to_num == FLB_TRUE) { - /* A numeric representation of a string should not exceed 19 chars */ - if (obj.via.str.size > 19) { - return -1; - } - - memcpy(str_num, obj.via.str.ptr, obj.via.str.size); - str_num[obj.via.str.size] = '\0'; - - ret = string_to_number(str_num, obj.via.str.size, - &i_out, &d_out); - if (ret == FLB_STR_FLOAT) { - *d = d_out; - return FLB_STR_FLOAT; - } - else if (ret == FLB_STR_INT) { - *i = i_out; - return FLB_STR_INT; - } - } - - return -1; -} - -int flb_sp_snapshot_create(struct flb_sp_task *task) -{ - struct flb_sp_cmd *cmd; - struct flb_sp_snapshot *snapshot; - - cmd = task->cmd; - - snapshot = (struct flb_sp_snapshot *) flb_calloc(1, sizeof(struct flb_sp_snapshot)); - if (!snapshot) { - flb_error("[sp] could not create snapshot '%s'", cmd->stream_name); - return -1; - } - - mk_list_init(&snapshot->pages); - snapshot->record_limit = cmd->limit; - - if (flb_sp_cmd_stream_prop_get(cmd, "seconds") != NULL) { - snapshot->time_limit = atoi(flb_sp_cmd_stream_prop_get(cmd, "seconds")); - } - - if (snapshot->time_limit == 0 && snapshot->record_limit == 0) { - flb_error("[sp] could not create snapshot '%s': size is not defined", - cmd->stream_name); - flb_sp_snapshot_destroy(snapshot); - return -1; - } - - task->snapshot = snapshot; - return 0; -} - -struct flb_sp_task *flb_sp_task_create(struct flb_sp *sp, const char *name, - const char *query) -{ - int fd; - int ret; - struct mk_event *event; - struct flb_sp_cmd *cmd; - struct flb_sp_task *task; - - /* - * Parse and validate the incoming exec query and create the 'command' - * context (this will be associated to the task in a later step - */ - cmd = flb_sp_cmd_create(query); - - if (!cmd) { - flb_error("[sp] invalid query on task '%s': '%s'", name, query); - return NULL; - } - - /* Check if we got an invalid type due an error/restriction */ - if (cmd->status == FLB_SP_ERROR) { - flb_error("[sp] invalid query on task '%s': '%s'", name, query); - flb_sp_cmd_destroy(cmd); - return NULL; - } - - /* Create the task context */ - task = flb_calloc(1, sizeof(struct flb_sp_task)); - if (!task) { - flb_errno(); - flb_sp_cmd_destroy(cmd); - return NULL; - } - task->name = flb_sds_create(name); - if (!task->name) { - flb_free(task); - flb_sp_cmd_destroy(cmd); - return NULL; - } - - task->query = flb_sds_create(query); - if (!task->query) { - flb_sds_destroy(task->name); - flb_free(task); - flb_sp_cmd_destroy(cmd); - return NULL; - } - - task->sp = sp; - task->cmd = cmd; - mk_list_add(&task->_head, &sp->tasks); - - /* - * Assume no aggregated keys exists, if so, a different strategy is - * required to process the records. - */ - task->aggregate_keys = FLB_FALSE; - - mk_list_init(&task->window.data); - mk_list_init(&task->window.aggregate_list); - rb_tree_new(&task->window.aggregate_tree, flb_sp_groupby_compare); - - mk_list_init(&task->window.hopping_slot); - - /* Check and validate aggregated keys */ - ret = sp_cmd_aggregated_keys(task->cmd); - if (ret == -1) { - flb_error("[sp] aggregated query cannot mix not aggregated keys: %s", - query); - flb_sp_task_destroy(task); - return NULL; - } - else if (ret > 0) { - task->aggregate_keys = FLB_TRUE; - - task->window.type = cmd->window.type; - - /* Register a timer event when task contains aggregation rules */ - if (task->window.type != FLB_SP_WINDOW_DEFAULT) { - /* Initialize event loop context */ - event = &task->window.event; - MK_EVENT_ZERO(event); - - /* Run every 'size' seconds */ - fd = mk_event_timeout_create(sp->config->evl, - cmd->window.size, (long) 0, - &task->window.event); - if (fd == -1) { - flb_error("[sp] registration for task %s failed", task->name); - flb_free(task); - return NULL; - } - task->window.fd = fd; - - if (task->window.type == FLB_SP_WINDOW_HOPPING) { - /* Initialize event loop context */ - event = &task->window.event_hop; - MK_EVENT_ZERO(event); - - /* Run every 'size' seconds */ - fd = mk_event_timeout_create(sp->config->evl, - cmd->window.advance_by, (long) 0, - &task->window.event_hop); - if (fd == -1) { - flb_error("[sp] registration for task %s failed", task->name); - flb_free(task); - return NULL; - } - task->window.advance_by = cmd->window.advance_by; - task->window.fd_hop = fd; - task->window.first_hop = true; - } - } - } - - /* Init snapshot page list */ - if (cmd->type == FLB_SP_CREATE_SNAPSHOT) { - if (flb_sp_snapshot_create(task) == -1) { - flb_sp_task_destroy(task); - return NULL; - } - } - - /* - * If the task involves a stream creation (CREATE STREAM abc..), create - * the stream. - */ - if (cmd->type == FLB_SP_CREATE_STREAM || - cmd->type == FLB_SP_CREATE_SNAPSHOT || - cmd->type == FLB_SP_FLUSH_SNAPSHOT) { - - ret = flb_sp_stream_create(cmd->stream_name, task, sp); - if (ret == -1) { - flb_error("[sp] could not create stream '%s'", cmd->stream_name); - flb_sp_task_destroy(task); - return NULL; - } - } - - /* - * Based in the command type, check if the source of data is a known - * stream so make a reference on this task for a quick comparisson and - * access it when processing data. - */ - sp_task_to_instance(task, sp); - return task; -} - -void groupby_nums_destroy(struct aggregate_num *groupby_nums, int size) -{ - int i; - - for (i = 0; i < size; i++) { - if (groupby_nums[i].type == FLB_SP_STRING) { - flb_sds_destroy(groupby_nums[i].string); - } - } - - flb_free(groupby_nums); -} - -/* - * Destroy aggregation node context: before to use this function make sure - * to unlink from the linked list. - */ -void flb_sp_aggregate_node_destroy(struct flb_sp_cmd *cmd, - struct aggregate_node *aggr_node) -{ - int i; - int key_id; - struct mk_list *head; - struct aggregate_num *num; - struct flb_sp_cmd_key *ckey; - - for (i = 0; i < aggr_node->nums_size; i++) { - num = &aggr_node->nums[i]; - if (num->type == FLB_SP_STRING) { - flb_sds_destroy(num->string); - } - } - - groupby_nums_destroy(aggr_node->groupby_nums, aggr_node->groupby_keys); - - key_id = 0; - mk_list_foreach(head, &cmd->keys) { - ckey = mk_list_entry(head, struct flb_sp_cmd_key, _head); - - if (!ckey->aggr_func) { - key_id++; - continue; - } - - aggregate_func_destroy[ckey->aggr_func - 1](aggr_node, key_id); - key_id++; - } - - flb_free(aggr_node->nums); - flb_free(aggr_node->aggregate_data); - flb_free(aggr_node); -} - -void flb_sp_window_destroy(struct flb_sp_cmd *cmd, - struct flb_sp_task_window *window) -{ - struct flb_sp_window_data *data; - struct aggregate_node *aggr_node; - struct flb_sp_hopping_slot *hs; - struct mk_list *head; - struct mk_list *tmp; - struct mk_list *head_hs; - struct mk_list *tmp_hs; - - mk_list_foreach_safe(head, tmp, &window->data) { - data = mk_list_entry(head, struct flb_sp_window_data, _head); - flb_free(data->buf_data); - mk_list_del(&data->_head); - flb_free(data); - } - - mk_list_foreach_safe(head, tmp, &window->aggregate_list) { - aggr_node = mk_list_entry(head, struct aggregate_node, _head); - mk_list_del(&aggr_node->_head); - flb_sp_aggregate_node_destroy(cmd, aggr_node); - } - - mk_list_foreach_safe(head, tmp, &window->hopping_slot) { - hs = mk_list_entry(head, struct flb_sp_hopping_slot, _head); - mk_list_foreach_safe(head_hs, tmp_hs, &hs->aggregate_list) { - aggr_node = mk_list_entry(head_hs, struct aggregate_node, _head); - mk_list_del(&aggr_node->_head); - flb_sp_aggregate_node_destroy(cmd, aggr_node); - } - rb_tree_destroy(&hs->aggregate_tree); - flb_free(hs); - } - - rb_tree_destroy(&window->aggregate_tree); -} - -void flb_sp_task_destroy(struct flb_sp_task *task) -{ - flb_sds_destroy(task->name); - flb_sds_destroy(task->query); - flb_sp_window_destroy(task->cmd, &task->window); - flb_sp_snapshot_destroy(task->snapshot); - mk_list_del(&task->_head); - - if (task->stream) { - flb_sp_stream_destroy(task->stream, task->sp); - } - - flb_sp_cmd_destroy(task->cmd); - flb_free(task); -} - -/* Create the stream processor context */ -struct flb_sp *flb_sp_create(struct flb_config *config) -{ - int i = 0; - int ret; - char buf[32]; - struct mk_list *head; - struct flb_sp *sp; - struct flb_slist_entry *e; - struct flb_sp_task *task; - - /* Allocate context */ - sp = flb_malloc(sizeof(struct flb_sp)); - if (!sp) { - flb_errno(); - return NULL; - } - sp->config = config; - mk_list_init(&sp->tasks); - - /* Check for pre-configured Tasks (command line) */ - mk_list_foreach(head, &config->stream_processor_tasks) { - e = mk_list_entry(head, struct flb_slist_entry, _head); - snprintf(buf, sizeof(buf) - 1, "flb-console:%i", i); - i++; - task = flb_sp_task_create(sp, buf, e->str); - if (!task) { - continue; - } - } - - /* Lookup configuration file if any */ - if (config->stream_processor_file) { - ret = sp_config_file(config, sp, config->stream_processor_file); - if (ret == -1) { - flb_error("[sp] could not initialize stream processor"); - flb_sp_destroy(sp); - return NULL; - } - } - - /* Write sp info to stdout */ - sp_info(sp); - - return sp; -} - -void free_value(struct flb_exp_val *v) -{ - if (!v) { - return; - } - - if (v->type == FLB_EXP_STRING) { - flb_sds_destroy(v->val.string); - } - - flb_free(v); -} - -static void itof_convert(struct flb_exp_val *val) -{ - if (val->type != FLB_EXP_INT) { - return; - } - - val->type = FLB_EXP_FLOAT; - val->val.f64 = (double) val->val.i64; -} - -/* Convert (string) expression to number */ -static void exp_string_to_number(struct flb_exp_val *val) -{ - int ret; - int len; - int64_t i = 0; - char *str; - double d = 0.0; - - len = flb_sds_len(val->val.string); - str = val->val.string; - - ret = string_to_number(str, len, &i, &d); - if (ret == -1) { - return; - } - - /* Assign to proper type */ - if (ret == FLB_STR_FLOAT) { - flb_sds_destroy(val->val.string); - val->type = FLB_EXP_FLOAT; - val->val.f64 = d; - } - else if (ret == FLB_STR_INT) { - flb_sds_destroy(val->val.string); - val->type = FLB_EXP_INT; - val->val.i64 = i; - } -} - -static void numerical_comp(struct flb_exp_val *left, - struct flb_exp_val *right, - struct flb_exp_val *result, int op) -{ - result->type = FLB_EXP_BOOL; - - if (left == NULL || right == NULL) { - result->val.boolean = false; - return; - } - - /* Check if left expression value is a number, if so, convert it */ - if (left->type == FLB_EXP_STRING && right->type != FLB_EXP_STRING) { - exp_string_to_number(left); - } - - if (left->type == FLB_EXP_INT && right->type == FLB_EXP_FLOAT) { - itof_convert(left); - } - else if (left->type == FLB_EXP_FLOAT && right->type == FLB_EXP_INT) { - itof_convert(right); - } - - switch (op) { - case FLB_EXP_EQ: - if (left->type == right->type) { - switch(left->type) { - case FLB_EXP_NULL: - result->val.boolean = true; - break; - case FLB_EXP_BOOL: - result->val.boolean = (left->val.boolean == right->val.boolean); - break; - case FLB_EXP_INT: - result->val.boolean = (left->val.i64 == right->val.i64); - break; - case FLB_EXP_FLOAT: - result->val.boolean = (left->val.f64 == right->val.f64); - break; - case FLB_EXP_STRING: - if (flb_sds_len(left->val.string) != - flb_sds_len(right->val.string)) { - result->val.boolean = false; - } - else if (strncmp(left->val.string, right->val.string, - flb_sds_len(left->val.string)) != 0) { - result->val.boolean = false; - } - else { - result->val.boolean = true; - } - break; - default: - result->val.boolean = false; - break; - } - } - else { - result->val.boolean = false; - } - break; - case FLB_EXP_LT: - if (left->type == right->type) { - switch(left->type) { - case FLB_EXP_INT: - result->val.boolean = (left->val.i64 < right->val.i64); - break; - case FLB_EXP_FLOAT: - result->val.boolean = (left->val.f64 < right->val.f64); - break; - case FLB_EXP_STRING: - if (strncmp(left->val.string, right->val.string, - flb_sds_len(left->val.string)) < 0) { - result->val.boolean = true; - } - else { - result->val.boolean = false; - } - break; - default: - result->val.boolean = false; - break; - } - } - else { - result->val.boolean = false; - } - break; - case FLB_EXP_LTE: - if (left->type == right->type) { - switch(left->type) { - case FLB_EXP_INT: - result->val.boolean = (left->val.i64 <= right->val.i64); - break; - case FLB_EXP_FLOAT: - result->val.boolean = (left->val.f64 <= right->val.f64); - break; - case FLB_EXP_STRING: - if (strncmp(left->val.string, right->val.string, - flb_sds_len(left->val.string)) <= 0) { - result->val.boolean = true; - } - else { - result->val.boolean = false; - } - break; - default: - result->val.boolean = false; - break; - } - } - else { - result->val.boolean = false; - } - break; - case FLB_EXP_GT: - if (left->type == right->type) { - switch(left->type) { - case FLB_EXP_INT: - result->val.boolean = (left->val.i64 > right->val.i64); - break; - case FLB_EXP_FLOAT: - result->val.boolean = (left->val.f64 > right->val.f64); - break; - case FLB_EXP_STRING: - if (strncmp(left->val.string, right->val.string, - flb_sds_len(left->val.string)) > 0) { - result->val.boolean = true; - } - else { - result->val.boolean = false; - } - break; - default: - result->val.boolean = false; - break; - } - } - else { - result->val.boolean = false; - } - break; - case FLB_EXP_GTE: - if (left->type == right->type) { - switch(left->type) { - case FLB_EXP_INT: - result->val.boolean = (left->val.i64 >= right->val.i64); - break; - case FLB_EXP_FLOAT: - result->val.boolean = (left->val.f64 >= right->val.f64); - break; - case FLB_EXP_STRING: - if (strncmp(left->val.string, right->val.string, - flb_sds_len(left->val.string)) >= 0) { - result->val.boolean = true; - } - else { - result->val.boolean = false; - } - break; - default: - result->val.boolean = false; - break; - } - } - else { - result->val.boolean = false; - } - break; - } -} - -static bool value_to_bool(struct flb_exp_val *val) { - bool result = FLB_FALSE; - - switch (val->type) { - case FLB_EXP_BOOL: - result = val->val.boolean; - break; - case FLB_EXP_INT: - result = val->val.i64 > 0; - break; - case FLB_EXP_FLOAT: - result = val->val.f64 > 0; - break; - case FLB_EXP_STRING: - result = true; - break; - } - - return result; -} - - -static void logical_operation(struct flb_exp_val *left, - struct flb_exp_val *right, - struct flb_exp_val *result, int op) -{ - bool lval; - bool rval; - - result->type = FLB_EXP_BOOL; - - /* Null is always interpreted as false in a logical operation */ - lval = left ? value_to_bool(left) : false; - rval = right ? value_to_bool(right) : false; - - switch (op) { - case FLB_EXP_NOT: - result->val.boolean = !lval; - break; - case FLB_EXP_AND: - result->val.boolean = lval & rval; - break; - case FLB_EXP_OR: - result->val.boolean = lval | rval; - break; - } -} - -static struct flb_exp_val *reduce_expression(struct flb_exp *expression, - const char *tag, int tag_len, - struct flb_time *tms, - msgpack_object *map) -{ - int operation; - flb_sds_t s; - flb_sds_t tmp_sds = NULL; - struct flb_exp_key *key; - struct flb_sp_value *sval; - struct flb_exp_val *ret, *left, *right; - struct flb_exp_val *result; - - if (!expression) { - return NULL; - } - - result = flb_calloc(1, sizeof(struct flb_exp_val)); - if (!result) { - flb_errno(); - return NULL; - } - - switch (expression->type) { - case FLB_EXP_NULL: - result->type = expression->type; - break; - case FLB_EXP_BOOL: - result->type = expression->type; - result->val.boolean = ((struct flb_exp_val *) expression)->val.boolean; - break; - case FLB_EXP_INT: - result->type = expression->type; - result->val.i64 = ((struct flb_exp_val *) expression)->val.i64; - break; - case FLB_EXP_FLOAT: - result->type = expression->type; - result->val.f64 = ((struct flb_exp_val *) expression)->val.f64; - break; - case FLB_EXP_STRING: - s = ((struct flb_exp_val *) expression)->val.string; - result->type = expression->type; - result->val.string = flb_sds_create_size(flb_sds_len(s)); - tmp_sds = flb_sds_copy(result->val.string, s, flb_sds_len(s)); - if (tmp_sds != result->val.string) { - result->val.string = tmp_sds; - } - break; - case FLB_EXP_KEY: - key = (struct flb_exp_key *) expression; - sval = flb_sp_key_to_value(key->name, *map, key->subkeys); - if (sval) { - result->type = sval->type; - result->val = sval->val; - flb_free(sval); - return result; - } - else { - flb_free(result); - return NULL; - } - break; - case FLB_EXP_FUNC: - /* we don't need result */ - flb_free(result); - ret = reduce_expression(((struct flb_exp_func *) expression)->param, - tag, tag_len, tms, map); - result = ((struct flb_exp_func *) expression)->cb_func(tag, tag_len, - tms, ret); - free_value(ret); - break; - case FLB_LOGICAL_OP: - left = reduce_expression(expression->left, - tag, tag_len, tms, map); - right = reduce_expression(expression->right, - tag, tag_len, tms, map); - - operation = ((struct flb_exp_op *) expression)->operation; - - switch (operation) { - case FLB_EXP_PAR: - if (left == NULL) { /* Null is always interpreted as false in a - logical operation */ - result->type = FLB_EXP_BOOL; - result->val.boolean = false; - } - else { /* Left and right sides of a logical operation reduce to - boolean values */ - result->type = FLB_EXP_BOOL; - result->val.boolean = left->val.boolean; - } - break; - case FLB_EXP_EQ: - case FLB_EXP_LT: - case FLB_EXP_LTE: - case FLB_EXP_GT: - case FLB_EXP_GTE: - numerical_comp(left, right, result, operation); - break; - case FLB_EXP_NOT: - case FLB_EXP_AND: - case FLB_EXP_OR: - logical_operation(left, right, result, operation); - break; - } - free_value(left); - free_value(right); - } - return result; -} - - -void package_results(const char *tag, int tag_len, - char **out_buf, size_t *out_size, - struct flb_sp_task *task) -{ - int i; - int len; - int map_entries; - msgpack_sbuffer mp_sbuf; - msgpack_packer mp_pck; - struct aggregate_num *num; - struct flb_time tm; - struct flb_sp_cmd_key *ckey; - struct flb_sp_cmd *cmd = task->cmd; - struct mk_list *head; - struct aggregate_node *aggr_node; - struct flb_sp_cmd_gb_key *gb_key = NULL; - - map_entries = mk_list_size(&cmd->keys); - - msgpack_sbuffer_init(&mp_sbuf); - msgpack_packer_init(&mp_pck, &mp_sbuf, msgpack_sbuffer_write); - - mk_list_foreach(head, &task->window.aggregate_list) { - aggr_node = mk_list_entry(head, struct aggregate_node, _head); - - /* set outgoing array + map and it fixed size */ - msgpack_pack_array(&mp_pck, 2); - - flb_time_get(&tm); - flb_time_append_to_msgpack(&tm, &mp_pck, 0); - msgpack_pack_map(&mp_pck, map_entries); - - /* Packaging results */ - ckey = mk_list_entry_first(&cmd->keys, struct flb_sp_cmd_key, _head); - for (i = 0; i < map_entries; i++) { - num = &aggr_node->nums[i]; - - /* Check if there is a defined function */ - if (ckey->time_func > 0) { - flb_sp_func_time(&mp_pck, ckey); - goto next; - } - else if (ckey->record_func > 0) { - flb_sp_func_record(tag, tag_len, &tm, &mp_pck, ckey); - goto next; - } - - /* Pack key */ - if (ckey->alias) { - msgpack_pack_str(&mp_pck, flb_sds_len(ckey->alias)); - msgpack_pack_str_body(&mp_pck, - ckey->alias, - flb_sds_len(ckey->alias)); - } - else { - len = 0; - char *c_name; - if (!ckey->name) { - c_name = "*"; - } - else { - c_name = ckey->name; - } - - msgpack_pack_str(&mp_pck, len); - msgpack_pack_str_body(&mp_pck, c_name, len); - } - - /* - * If a group_by key is mapped as a source of this key, - * change the 'num' reference to obtain the proper information - * for the grouped key value. - */ - if (ckey->gb_key != NULL) { - gb_key = ckey->gb_key; - if (aggr_node->groupby_keys > 0) { - num = &aggr_node->groupby_nums[gb_key->id]; - } - } - - /* Pack value */ - switch (ckey->aggr_func) { - case FLB_SP_NOP: - if (num->type == FLB_SP_NUM_I64) { - msgpack_pack_int64(&mp_pck, num->i64); - } - else if (num->type == FLB_SP_NUM_F64) { - msgpack_pack_float(&mp_pck, num->f64); - } - else if (num->type == FLB_SP_STRING) { - msgpack_pack_str(&mp_pck, - flb_sds_len(num->string)); - msgpack_pack_str_body(&mp_pck, - num->string, - flb_sds_len(num->string)); - } - else if (num->type == FLB_SP_BOOLEAN) { - if (num->boolean) { - msgpack_pack_true(&mp_pck); - } - else { - msgpack_pack_false(&mp_pck); - } - } - break; - default: - aggregate_func_calc[ckey->aggr_func - 1](aggr_node, ckey, &mp_pck, i); - break; - } - -next: - ckey = mk_list_entry_next(&ckey->_head, struct flb_sp_cmd_key, - _head, &cmd->keys); - } - } - - *out_buf = mp_sbuf.data; - *out_size = mp_sbuf.size; -} - -static struct aggregate_node * sp_process_aggregate_data(struct flb_sp_task *task, - msgpack_object map, - int convert_str_to_num) -{ - int i; - int ret; - int map_size; - int key_id; - int map_entries; - int gb_entries; - int values_found; - int64_t ival; - double dval; - struct flb_sp_value *sval; - struct aggregate_num *gb_nums; - struct aggregate_node *aggr_node; - struct flb_sp_cmd *cmd; - struct flb_sp_cmd_gb_key *gb_key; - struct mk_list *head; - struct rb_tree_node *rb_result; - msgpack_object key; - - aggr_node = NULL; - cmd = task->cmd; - map_size = map.via.map.size; - values_found = 0; - - /* Number of expected output entries in the map */ - map_entries = mk_list_size(&cmd->keys); - gb_entries = mk_list_size(&cmd->gb_keys); - - if (gb_entries > 0) { - gb_nums = flb_calloc(1, sizeof(struct aggregate_num) * gb_entries); - if (!gb_nums) { - return NULL; - } - - /* extract GROUP BY values */ - for (i = 0; i < map_size; i++) { /* extract group-by values */ - key = map.via.map.ptr[i].key; - - key_id = 0; - mk_list_foreach(head, &cmd->gb_keys) { - gb_key = mk_list_entry(head, struct flb_sp_cmd_gb_key, - _head); - if (flb_sds_cmp(gb_key->name, key.via.str.ptr, - key.via.str.size) != 0) { - key_id++; - continue; - } - - sval = flb_sp_key_to_value(gb_key->name, map, gb_key->subkeys); - if (!sval) { - /* If evaluation fails/sub-key doesn't exist */ - key_id++; - continue; - } - - values_found++; - - /* Convert string to number if that is possible */ - ret = object_to_number(sval->o, &ival, &dval, convert_str_to_num); - if (ret == -1) { - if (sval->o.type == MSGPACK_OBJECT_STR) { - gb_nums[key_id].type = FLB_SP_STRING; - gb_nums[key_id].string = - flb_sds_create_len(sval->o.via.str.ptr, - sval->o.via.str.size); - } - else if (sval->o.type == MSGPACK_OBJECT_BOOLEAN) { - gb_nums[key_id].type = FLB_SP_NUM_I64; - gb_nums[key_id].i64 = sval->o.via.boolean; - } - } - else if (ret == FLB_STR_INT) { - gb_nums[key_id].type = FLB_SP_NUM_I64; - gb_nums[key_id].i64 = ival; - } - else if (ret == FLB_STR_FLOAT) { - gb_nums[key_id].type = FLB_SP_NUM_F64; - gb_nums[key_id].f64 = dval; - } - - key_id++; - flb_sp_key_value_destroy(sval); - } - } - - /* if some GROUP BY keys are not found in the record */ - if (values_found < gb_entries) { - groupby_nums_destroy(gb_nums, gb_entries); - return NULL; - } - - aggr_node = (struct aggregate_node *) flb_calloc(1, sizeof(struct aggregate_node)); - if (!aggr_node) { - flb_errno(); - groupby_nums_destroy(gb_nums, gb_entries); - return NULL; - } - - aggr_node->groupby_keys = gb_entries; - aggr_node->groupby_nums = gb_nums; - - rb_tree_find_or_insert(&task->window.aggregate_tree, aggr_node, &aggr_node->_rb_head, &rb_result); - if (&aggr_node->_rb_head != rb_result) { - /* We don't need aggr_node anymore */ - flb_sp_aggregate_node_destroy(cmd, aggr_node); - - aggr_node = container_of(rb_result, struct aggregate_node, _rb_head); - container_of(rb_result, struct aggregate_node, _rb_head)->records++; - } - else { - aggr_node->nums = flb_calloc(1, sizeof(struct aggregate_num) * map_entries); - if (!aggr_node->nums) { - flb_sp_aggregate_node_destroy(cmd, aggr_node); - return NULL; - } - aggr_node->records = 1; - aggr_node->nums_size = map_entries; - aggr_node->aggregate_data = (struct aggregate_data **) flb_calloc(1, sizeof(struct aggregate_data *) * map_entries); - mk_list_add(&aggr_node->_head, &task->window.aggregate_list); - } - } - else { /* If query doesn't have GROUP BY */ - if (!mk_list_size(&task->window.aggregate_list)) { - aggr_node = flb_calloc(1, sizeof(struct aggregate_node)); - if (!aggr_node) { - flb_errno(); - return NULL; - } - aggr_node->nums = flb_calloc(1, sizeof(struct aggregate_num) * map_entries); - if (!aggr_node->nums) { - flb_sp_aggregate_node_destroy(cmd, aggr_node); - return NULL; - } - - aggr_node->nums_size = map_entries; - aggr_node->records = 1; - aggr_node->aggregate_data = (struct aggregate_data **) flb_calloc(1, sizeof(struct aggregate_data *) * map_entries); - mk_list_add(&aggr_node->_head, &task->window.aggregate_list); - } - else { - aggr_node = mk_list_entry_first(&task->window.aggregate_list, struct aggregate_node, _head); - aggr_node->records++; - } - } - - return aggr_node; -} - -/* - * Process data, task and it defined command involves the call of aggregation - * functions (AVG, SUM, COUNT, MIN, MAX). - */ -int sp_process_data_aggr(const char *buf_data, size_t buf_size, - const char *tag, int tag_len, - struct flb_sp_task *task, - struct flb_sp *sp, - int convert_str_to_num) -{ - int i; - int ok; - int ret; - int map_size; - int key_id; - size_t off; - int64_t ival; - double dval; - msgpack_object root; - msgpack_object map; - msgpack_unpacked result; - msgpack_object key; - msgpack_object *obj; - struct aggregate_num *nums = NULL; - struct mk_list *head; - struct flb_time tms; - struct flb_sp_cmd *cmd = task->cmd; - struct flb_sp_cmd_key *ckey; - struct flb_sp_value *sval; - struct flb_exp_val *condition; - struct aggregate_node *aggr_node; - - /* Number of expected output entries in the map */ - off = 0; - - /* vars initialization */ - ok = MSGPACK_UNPACK_SUCCESS; - msgpack_unpacked_init(&result); - - /* Iterate incoming records */ - while (msgpack_unpack_next(&result, buf_data, buf_size, &off) == ok) { - root = result.data; - - /* extract timestamp */ - flb_time_pop_from_msgpack(&tms, &result, &obj); - - /* get the map data and it size (number of items) */ - map = root.via.array.ptr[1]; - map_size = map.via.map.size; - - /* Evaluate condition */ - if (cmd->condition) { - condition = reduce_expression(cmd->condition, - tag, tag_len, &tms, &map); - if (!condition) { - continue; - } - else if (!condition->val.boolean) { - flb_free(condition); - continue; - } - else { - flb_free(condition); - } - } - - aggr_node = sp_process_aggregate_data(task, map, convert_str_to_num); - if (!aggr_node) - { - continue; - } - - task->window.records++; - - nums = aggr_node->nums; - - /* Iterate each map key and see if it matches any command key */ - for (i = 0; i < map_size; i++) { - key = map.via.map.ptr[i].key; - - if (key.type != MSGPACK_OBJECT_STR) { - continue; - } - - - /* - * Iterate each command key. Note that since the command key - * can have different aggregation functions to the same key - * we should compare all of them. - */ - key_id = 0; - mk_list_foreach(head, &cmd->keys) { - ckey = mk_list_entry(head, struct flb_sp_cmd_key, _head); - - if (!ckey->name) { - key_id++; - continue; - } - - if (flb_sds_cmp(ckey->name, key.via.str.ptr, - key.via.str.size) != 0) { - key_id++; - continue; - } - - /* convert the value if it string */ - sval = flb_sp_key_to_value(ckey->name, map, ckey->subkeys); - if (!sval) { - key_id++; - continue; - } - - /* - * Convert value to a numeric representation only if key has an - * assigned aggregation function - */ - ival = 0; - dval = 0.0; - if (ckey->aggr_func != FLB_SP_NOP) { - ret = object_to_number(sval->o, &ival, &dval, convert_str_to_num); - if (ret == -1) { - /* Value cannot be represented as a number */ - key_id++; - flb_sp_key_value_destroy(sval); - continue; - } - - /* - * If a floating pointer number exists, we use the same data - * type for the output. - */ - if (dval != 0.0 && nums[key_id].type == FLB_SP_NUM_I64) { - nums[key_id].type = FLB_SP_NUM_F64; - nums[key_id].f64 = (double) nums[key_id].i64; - } - - aggregate_func_add[ckey->aggr_func - 1](aggr_node, ckey, key_id, &tms, ival, dval); - } - else { - if (sval->o.type == MSGPACK_OBJECT_BOOLEAN) { - nums[key_id].type = FLB_SP_BOOLEAN; - nums[key_id].boolean = sval->o.via.boolean; - } - if (sval->o.type == MSGPACK_OBJECT_POSITIVE_INTEGER || - sval->o.type == MSGPACK_OBJECT_NEGATIVE_INTEGER) { - nums[key_id].type = FLB_SP_NUM_I64; - nums[key_id].i64 = sval->o.via.i64; - } - else if (sval->o.type == MSGPACK_OBJECT_FLOAT32 || - sval->o.type == MSGPACK_OBJECT_FLOAT) { - nums[key_id].type = FLB_SP_NUM_F64; - nums[key_id].f64 = sval->o.via.f64; - } - else if (sval->o.type == MSGPACK_OBJECT_STR) { - nums[key_id].type = FLB_SP_STRING; - if (nums[key_id].string == NULL) { - nums[key_id].string = - flb_sds_create_len(sval->o.via.str.ptr, - sval->o.via.str.size); - } - } - } - - key_id++; - flb_sp_key_value_destroy(sval); - } - } - } - - msgpack_unpacked_destroy(&result); - return task->window.records; -} - -/* - * Data processing (no aggregation functions) - */ -int sp_process_data(const char *tag, int tag_len, - const char *buf_data, size_t buf_size, - char **out_buf, size_t *out_size, - struct flb_sp_task *task, - struct flb_sp *sp) -{ - int i; - int ok; - int ret; - int map_size; - int map_entries; - int records; - uint8_t h; - off_t map_off; - off_t no_data; - size_t off; - size_t off_copy; - size_t snapshot_out_size; - char *tmp; - char *snapshot_out_buffer; - msgpack_object root; - msgpack_object *obj; - msgpack_object key; - msgpack_object val; - msgpack_unpacked result; - msgpack_sbuffer mp_sbuf; - msgpack_packer mp_pck; - msgpack_object map; - struct flb_time tms; - struct mk_list *head; - struct flb_sp_cmd *cmd; - struct flb_sp_cmd_key *cmd_key; - struct flb_exp_val *condition; - struct flb_sp_value *sval; - - /* Vars initialization */ - off = 0; - off_copy = off; - records = 0; - cmd = task->cmd; - ok = MSGPACK_UNPACK_SUCCESS; - msgpack_unpacked_init(&result); - msgpack_sbuffer_init(&mp_sbuf); - msgpack_packer_init(&mp_pck, &mp_sbuf, msgpack_sbuffer_write); - - snapshot_out_size = 0; - snapshot_out_buffer = NULL; - - /* Iterate incoming records */ - while (msgpack_unpack_next(&result, buf_data, buf_size, &off) == ok) { - root = result.data; - - /* extract timestamp */ - flb_time_pop_from_msgpack(&tms, &result, &obj); - - /* Store the buffer if the stream is a snapshot */ - if (cmd->type == FLB_SP_CREATE_SNAPSHOT) { - flb_sp_snapshot_update(task, buf_data + off_copy, off - off_copy, &tms); - off_copy = off; - continue; - } - - /* get the map data and it size (number of items) */ - map = root.via.array.ptr[1]; - map_size = map.via.map.size; - - /* Evaluate condition */ - if (cmd->condition) { - condition = reduce_expression(cmd->condition, - tag, tag_len, &tms, &map); - if (!condition) { - continue; - } - else if (!condition->val.boolean) { - flb_free(condition); - continue; - } - else { - flb_free(condition); - } - } - - records++; - - /* Flush the snapshot if condition holds */ - if (cmd->type == FLB_SP_FLUSH_SNAPSHOT) { - if (flb_sp_snapshot_flush(sp, task, &snapshot_out_buffer, - &snapshot_out_size) == -1) { - msgpack_unpacked_destroy(&result); - msgpack_sbuffer_destroy(&mp_sbuf); - return -1; - } - continue; - } - - - /* - * If for some reason the Task keys did not insert any data, we will - * need to discard any changes and reset the buffer position, let's - * keep the memory size for that purpose. - */ - no_data = mp_sbuf.size; - - /* Pack main array */ - msgpack_pack_array(&mp_pck, 2); - msgpack_pack_object(&mp_pck, root.via.array.ptr[0]); - - /* - * Save the current size/position of the buffer since this is - * where the Map header will be stored. - */ - map_off = mp_sbuf.size; - - /* - * In the new record register the same number of items, if due to - * fields selection the number is lower, we perform an adjustment - */ - msgpack_pack_map(&mp_pck, map_size); - - /* Counter for new entries added to the outgoing map */ - map_entries = 0; - - /* Iterate key selection */ - mk_list_foreach(head, &cmd->keys) { - cmd_key = mk_list_entry(head, struct flb_sp_cmd_key, _head); - if (cmd_key->time_func > 0) { - /* Process time function */ - ret = flb_sp_func_time(&mp_pck, cmd_key); - if (ret > 0) { - map_entries += ret; - } - continue; - } - else if (cmd_key->record_func > 0) { - ret = flb_sp_func_record(tag, tag_len, &tms, &mp_pck, cmd_key); - if (ret > 0) { - map_entries += ret; - } - continue; - } - - /* Lookup selection key in the incoming map */ - for (i = 0; i < map_size; i++) { - key = map.via.map.ptr[i].key; - val = map.via.map.ptr[i].val; - - if (key.type != MSGPACK_OBJECT_STR) { - continue; - } - - /* Wildcard selection: * */ - if (cmd_key->name == NULL) { - msgpack_pack_object(&mp_pck, key); - msgpack_pack_object(&mp_pck, val); - map_entries++; - continue; - } - - /* Compare lengths */ - if (flb_sds_cmp(cmd_key->name, - key.via.str.ptr, key.via.str.size) != 0) { - continue; - } - - /* - * Package key name: - * - * Check if the command ask for an alias 'key AS abc' - */ - if (cmd_key->alias) { - msgpack_pack_str(&mp_pck, - flb_sds_len(cmd_key->alias)); - msgpack_pack_str_body(&mp_pck, - cmd_key->alias, - flb_sds_len(cmd_key->alias)); - } - else { - msgpack_pack_object(&mp_pck, key); - } - - /* Package value */ - sval = flb_sp_key_to_value(cmd_key->name, map, - cmd_key->subkeys); - if (sval) { - msgpack_pack_object(&mp_pck, sval->o); - flb_sp_key_value_destroy(sval); - } - - map_entries++; - } - } - - /* Final Map size adjustment */ - if (map_entries == 0) { - mp_sbuf.size = no_data; - } - else { - /* - * The fields were packed, now we need to adjust the map size - * to set the proper number of fields appended to the record. - */ - tmp = mp_sbuf.data + map_off; - h = tmp[0]; - if (h >> 4 == 0x8) { - *tmp = (uint8_t) 0x8 << 4 | ((uint8_t) map_entries); - } - else if (h == 0xde) { - tmp++; - pack_uint16(tmp, map_entries); - } - else if (h == 0xdf) { - tmp++; - pack_uint32(tmp, map_entries); - } - } - } - - msgpack_unpacked_destroy(&result); - - if (records == 0) { - msgpack_sbuffer_destroy(&mp_sbuf); - return 0; - } - - /* Use snapshot out buffer if it is flush stream */ - if (cmd->type == FLB_SP_FLUSH_SNAPSHOT) { - if (snapshot_out_size == 0) { - msgpack_sbuffer_destroy(&mp_sbuf); - flb_free(snapshot_out_buffer); - return 0; - } - else { - *out_buf = snapshot_out_buffer; - *out_size = snapshot_out_size; - return records; - } - } - - /* set outgoing results */ - *out_buf = mp_sbuf.data; - *out_size = mp_sbuf.size; - - return records; -} - -int sp_process_hopping_slot(const char *tag, int tag_len, - struct flb_sp_task *task) -{ - int i; - int key_id; - int map_entries; - int gb_entries; - struct flb_sp_cmd *cmd = task->cmd; - struct mk_list *head; - struct mk_list *head_hs; - struct aggregate_node *aggr_node; - struct aggregate_node *aggr_node_hs; - struct aggregate_node *aggr_node_prev; - struct flb_sp_hopping_slot *hs; - struct flb_sp_hopping_slot *hs_; - struct rb_tree_node *rb_result; - struct flb_sp_cmd_key *ckey; - rb_result_t result; - - map_entries = mk_list_size(&cmd->keys); - gb_entries = mk_list_size(&cmd->gb_keys); - - /* Initialize a hoping slot */ - hs = flb_calloc(1, sizeof(struct flb_sp_hopping_slot)); - if (!hs) { - flb_errno(); - return -1; - } - - mk_list_init(&hs->aggregate_list); - rb_tree_new(&hs->aggregate_tree, flb_sp_groupby_compare); - - /* Loop over aggregation nodes on window */ - mk_list_foreach(head, &task->window.aggregate_list) { - /* Window aggregation node */ - aggr_node = mk_list_entry(head, struct aggregate_node, _head); - - /* Create a hopping slot aggregation node */ - aggr_node_hs = flb_calloc(1, sizeof(struct aggregate_node)); - if (!aggr_node_hs) { - flb_errno(); - flb_free(hs); - return -1; - } - - aggr_node_hs->nums = malloc(sizeof(struct aggregate_node) * map_entries); - if (!aggr_node_hs->nums) { - flb_errno(); - flb_free(hs); - flb_free(aggr_node_hs); - return -1; - } - - memcpy(aggr_node_hs->nums, aggr_node->nums, sizeof(struct aggregate_num) * map_entries); - aggr_node_hs->records = aggr_node->records; - - /* Clone aggregate data */ - key_id = 0; - mk_list_foreach(head_hs, &cmd->keys) { - ckey = mk_list_entry(head_hs, struct flb_sp_cmd_key, _head); - - if (ckey->aggr_func) { - if (!aggr_node_hs->aggregate_data) { - aggr_node_hs->aggregate_data = (struct aggregate_data **) - flb_calloc(1, sizeof(struct aggregate_data *) * map_entries); - if (!aggr_node_hs->aggregate_data) { - flb_errno(); - flb_free(hs); - flb_free(aggr_node_hs->nums); - flb_free(aggr_node_hs); - return -1; - } - } - - if (aggregate_func_clone[ckey->aggr_func - 1](aggr_node_hs, aggr_node, ckey, key_id) == -1) { - flb_errno(); - flb_free(aggr_node_hs->nums); - flb_free(aggr_node_hs->aggregate_data); - flb_free(aggr_node_hs); - flb_free(hs); - return -1; - } - } - - key_id++; - } - - /* Traverse over previous slots to calculate values/record numbers */ - mk_list_foreach(head_hs, &task->window.hopping_slot) { - hs_ = mk_list_entry(head_hs, struct flb_sp_hopping_slot, _head); - result = rb_tree_find(&hs_->aggregate_tree, aggr_node, &rb_result); - /* If corresponding aggregation node exists in previous hopping slot, - * calculate aggregation values - */ - if (result == RB_OK) { - aggr_node_prev = mk_list_entry(rb_result, struct aggregate_node, - _rb_head); - aggr_node_hs->records -= aggr_node_prev->records; - - key_id = 0; - ckey = mk_list_entry_first(&cmd->keys, struct flb_sp_cmd_key, - _head); - for (i = 0; i < map_entries; i++) { - if (ckey->aggr_func) { - aggregate_func_remove[ckey->aggr_func - 1](aggr_node_hs, aggr_node_prev, i); - } - - ckey = mk_list_entry_next(&ckey->_head, struct flb_sp_cmd_key, - _head, &cmd->keys); - } - } - } - - if (aggr_node_hs->records > 0) { - aggr_node_hs->groupby_nums = - flb_calloc(1, sizeof(struct aggregate_node) * gb_entries); - if (gb_entries > 0 && !aggr_node_hs->groupby_nums) { - flb_errno(); - flb_free(hs); - flb_free(aggr_node_hs->nums); - flb_free(aggr_node_hs->aggregate_data); - flb_free(aggr_node_hs); - return -1; - } - - if (aggr_node_hs->groupby_nums != NULL) { - memcpy(aggr_node_hs->groupby_nums, aggr_node->groupby_nums, - sizeof(struct aggregate_num) * gb_entries); - } - - aggr_node_hs->nums_size = aggr_node->nums_size; - aggr_node_hs->groupby_keys = aggr_node->groupby_keys; - - rb_tree_insert(&hs->aggregate_tree, aggr_node_hs, &aggr_node_hs->_rb_head); - mk_list_add(&aggr_node_hs->_head, &hs->aggregate_list); - } - else { - flb_free(aggr_node_hs->nums); - flb_free(aggr_node_hs->aggregate_data); - flb_free(aggr_node_hs); - } - } - - hs->records = task->window.records; - mk_list_foreach(head_hs, &task->window.hopping_slot) { - hs_ = mk_list_entry(head_hs, struct flb_sp_hopping_slot, _head); - hs->records -= hs_->records; - } - - mk_list_add(&hs->_head, &task->window.hopping_slot); - - return 0; -} - -/* Iterate and find input chunks to process */ -int flb_sp_do(struct flb_sp *sp, struct flb_input_instance *in, - const char *tag, int tag_len, - const char *buf_data, size_t buf_size) - -{ - int ret; - size_t out_size; - char *out_buf; - struct mk_list *head; - struct flb_sp_task *task; - struct flb_sp_cmd *cmd; - - /* Lookup tasks that match the incoming instance data */ - mk_list_foreach(head, &sp->tasks) { - task = mk_list_entry(head, struct flb_sp_task, _head); - cmd = task->cmd; - - if (cmd->source_type == FLB_SP_STREAM) { - if (task->source_instance != in) { - continue; - } - } - else if (cmd->source_type == FLB_SP_TAG) { - ret = flb_router_match(tag, tag_len, cmd->source_name, NULL); - if (ret == FLB_FALSE) { - continue; - } - } - - /* We found a task that matches the stream rule */ - if (task->aggregate_keys == FLB_TRUE) { - ret = sp_process_data_aggr(buf_data, buf_size, - tag, tag_len, - task, sp, in->config->stream_processor_str_conv); - - if (ret == -1) { - flb_error("[sp] error processing records for '%s'", - task->name); - continue; - } - - if (flb_sp_window_populate(task, buf_data, buf_size) == -1) { - flb_error("[sp] error populating window for '%s'", - task->name); - continue; - } - - if (task->window.type == FLB_SP_WINDOW_DEFAULT) { - package_results(tag, tag_len, &out_buf, &out_size, task); - flb_sp_window_prune(task); - } - } - else { - ret = sp_process_data(tag, tag_len, - buf_data, buf_size, - &out_buf, &out_size, - task, sp); - - if (ret == -1) { - flb_error("[sp] error processing records for '%s'", - task->name); - continue; - } - } - - if (ret == 0) { - /* no records */ - continue; - } - - /* - * This task involves append data to a stream, which - * means: register the output of the query as data - * generated by an input instance plugin. - */ - if (task->aggregate_keys != FLB_TRUE || - task->window.type == FLB_SP_WINDOW_DEFAULT) { - /* - * Add to stream processing stream if there is no - * aggregation function. Otherwise, write it at timer event - */ - if (task->stream) { - flb_sp_stream_append_data(out_buf, out_size, task->stream); - } - else { - flb_pack_print(out_buf, out_size); - flb_free(out_buf); - } - } - } - - return -1; -} - -int flb_sp_fd_event(int fd, struct flb_sp *sp) -{ - bool update_timer_event; - char *out_buf; - char *tag = NULL; - int tag_len = 0; - int fd_timeout = 0; - size_t out_size; - struct mk_list *tmp; - struct mk_list *head; - struct flb_sp_task *task; - struct flb_input_instance *in = NULL; - - /* Lookup Tasks that matches the incoming event */ - mk_list_foreach_safe(head, tmp, &sp->tasks) { - task = mk_list_entry(head, struct flb_sp_task, _head); - - if (fd == task->window.fd) { - update_timer_event = task->window.type == FLB_SP_WINDOW_HOPPING && - task->window.first_hop; - - in = task->source_instance; - if (in) { - if (in->tag && in->tag_len > 0) { - tag = in->tag; - tag_len = in->tag_len; - } - else { - tag = in->name; - tag_len = strlen(in->name); - } - } - else { - in = NULL; - } - - if (task->window.records > 0) { - /* find input tag from task source */ - package_results(tag, tag_len, &out_buf, &out_size, task); - if (task->stream) { - flb_sp_stream_append_data(out_buf, out_size, task->stream); - } - else { - flb_pack_print(out_buf, out_size); - flb_free(out_buf); - } - - } - - flb_sp_window_prune(task); - - flb_utils_timer_consume(fd); - - if (update_timer_event && in) { - task->window.first_hop = false; - mk_event_timeout_destroy(in->config->evl, &task->window.event); - mk_event_closesocket(fd); - - fd_timeout = mk_event_timeout_create(in->config->evl, - task->window.advance_by, (long) 0, - &task->window.event); - if (fd_timeout == -1) { - flb_error("[sp] registration for task (updating timer event) %s failed", task->name); - return -1; - } - task->window.fd = fd_timeout; - } - - break; - } - else if (fd == task->window.fd_hop) { - in = task->source_instance; - if (in) { - if (in->tag && in->tag_len > 0) { - tag = in->tag; - tag_len = in->tag_len; - } - else { - tag = in->name; - tag_len = strlen(in->name); - } - } - sp_process_hopping_slot(tag, tag_len, task); - flb_utils_timer_consume(fd); - } - } - return 0; -} - -/* Destroy stream processor context */ -void flb_sp_destroy(struct flb_sp *sp) -{ - struct mk_list *tmp; - struct mk_list *head; - struct flb_sp_task *task; - - /* destroy tasks */ - mk_list_foreach_safe(head, tmp, &sp->tasks) { - task = mk_list_entry(head, struct flb_sp_task, _head); - flb_sp_task_destroy(task); - } - - flb_free(sp); -} diff --git a/fluent-bit/src/stream_processor/flb_sp_aggregate_func.c b/fluent-bit/src/stream_processor/flb_sp_aggregate_func.c deleted file mode 100644 index 624be878c..000000000 --- a/fluent-bit/src/stream_processor/flb_sp_aggregate_func.c +++ /dev/null @@ -1,364 +0,0 @@ -#include <fluent-bit/stream_processor/flb_sp.h> -#include <fluent-bit/stream_processor/flb_sp_parser.h> -#include <fluent-bit/stream_processor/flb_sp_aggregate_func.h> - -char aggregate_func_string[AGGREGATE_FUNCTIONS][sizeof("TIMESERIES_FORECAST") + 1] = { - "AVG", - "SUM", - "COUNT", - "MIN", - "MAX", - "TIMESERIES_FORECAST" -}; - -int aggregate_func_clone_nop(struct aggregate_node *aggr_node, - struct aggregate_node *aggr_node_prev, - struct flb_sp_cmd_key *ckey, - int key_id) { - return 0; -} - -int aggregate_func_clone_timeseries_forecast(struct aggregate_node *aggr_node_clone, - struct aggregate_node *aggr_node, - struct flb_sp_cmd_key *ckey, - int key_id) { - struct timeseries_forecast *forecast_clone; - struct timeseries_forecast *forecast; - - forecast_clone = (struct timeseries_forecast *) aggr_node_clone->aggregate_data[key_id]; - if (!forecast_clone) { - forecast_clone = (struct timeseries_forecast *) flb_calloc(1, sizeof(struct timeseries_forecast)); - if (!forecast_clone) { - return -1; - } - - forecast_clone->future_time = ckey->constant; - aggr_node_clone->aggregate_data[key_id] = (struct aggregate_data *) forecast_clone; - } - - forecast = (struct timeseries_forecast *) aggr_node->aggregate_data[key_id]; - - forecast_clone->sigma_x = forecast->sigma_x; - forecast_clone->sigma_y = forecast->sigma_y; - forecast_clone->sigma_xy = forecast->sigma_xy; - forecast_clone->sigma_x2 = forecast->sigma_x2; - - return 0; -} - -/* Summarize a value into the temporary array considering data type */ -void aggregate_func_add_sum(struct aggregate_node *aggr_node, - struct flb_sp_cmd_key *ckey, - int key_id, - struct flb_time *tms, - int64_t ival, double dval) { - if (aggr_node->nums[key_id].type == FLB_SP_NUM_I64) { - aggr_node->nums[key_id].i64 += ival; - aggr_node->nums[key_id].ops++; - } - else if (aggr_node->nums[key_id].type == FLB_SP_NUM_F64) { - if (dval != 0.0) { - aggr_node->nums[key_id].f64 += dval; - } - else { - aggr_node->nums[key_id].f64 += (double) ival; - } - aggr_node->nums[key_id].ops++; - } -} - -void aggregate_func_add_count(struct aggregate_node *aggr_node, - struct flb_sp_cmd_key *ckey, - int key_id, - struct flb_time *tms, - int64_t ival, double dval) { -} - -/* Calculate the minimum value considering data type */ -void aggregate_func_add_min(struct aggregate_node *aggr_node, - struct flb_sp_cmd_key *ckey, - int key_id, - struct flb_time *tms, - int64_t ival, double dval) { - - if (aggr_node->nums[key_id].type == FLB_SP_NUM_I64) { - if (aggr_node->nums[key_id].ops == 0) { - aggr_node->nums[key_id].i64 = ival; - aggr_node->nums[key_id].ops++; - } - else { - if (aggr_node->nums[key_id].i64 > ival) { - aggr_node->nums[key_id].i64 = ival; - aggr_node->nums[key_id].ops++; - } - } - } - else if (aggr_node->nums[key_id].type == FLB_SP_NUM_F64) { - if (dval != 0.0) { - if (aggr_node->nums[key_id].ops == 0) { - aggr_node->nums[key_id].f64 = dval; - aggr_node->nums[key_id].ops++; - } - else { - if (aggr_node->nums[key_id].f64 > dval) { - aggr_node->nums[key_id].f64 = dval; - aggr_node->nums[key_id].ops++; - } - } - } - else { - if (aggr_node->nums[key_id].ops == 0) { - aggr_node->nums[key_id].f64 = (double) ival; - aggr_node->nums[key_id].ops++; - } - else { - if (aggr_node->nums[key_id].f64 > (double) ival) { - aggr_node->nums[key_id].f64 = ival; - aggr_node->nums[key_id].ops++; - } - } - } - } -} - -/* Calculate the maximum value considering data type */ -void aggregate_func_add_max(struct aggregate_node *aggr_node, - struct flb_sp_cmd_key *ckey, - int key_id, - struct flb_time *tms, - int64_t ival, double dval) { - if (aggr_node->nums[key_id].type == FLB_SP_NUM_I64) { - if (aggr_node->nums[key_id].ops == 0) { - aggr_node->nums[key_id].i64 = ival; - aggr_node->nums[key_id].ops++; - } - else { - if (aggr_node->nums[key_id].i64 < ival) { - aggr_node->nums[key_id].i64 = ival; - aggr_node->nums[key_id].ops++; - } - } - } - else if (aggr_node->nums[key_id].type == FLB_SP_NUM_F64) { - if (dval != 0.0) { - if (aggr_node->nums[key_id].ops == 0) { - aggr_node->nums[key_id].f64 = dval; - aggr_node->nums[key_id].ops++; - } - else { - if (aggr_node->nums[key_id].f64 < dval) { - aggr_node->nums[key_id].f64 = dval; - aggr_node->nums[key_id].ops++; - } - } - } - else { - if (aggr_node->nums[key_id].ops == 0) { - aggr_node->nums[key_id].f64 = (double) ival; - aggr_node->nums[key_id].ops++; - } - else { - if (aggr_node->nums[key_id].f64 < (double) ival) { - aggr_node->nums[key_id].f64 = (double) ival; - aggr_node->nums[key_id].ops++; - } - } - } - } -} - -void aggregate_func_calc_avg(struct aggregate_node *aggr_node, - struct flb_sp_cmd_key *ckey, - msgpack_packer *mp_pck, - int key_id) { - double dval = 0.0; - /* average = sum(values) / records */ - if (aggr_node->nums[key_id].type == FLB_SP_NUM_I64) { - dval = (double) aggr_node->nums[key_id].i64 / aggr_node->records; - } - else if (aggr_node->nums[key_id].type == FLB_SP_NUM_F64) { - dval = (double) aggr_node->nums[key_id].f64 / aggr_node->records; - } - - msgpack_pack_float(mp_pck, dval); -} - -void aggregate_func_calc_sum(struct aggregate_node *aggr_node, - struct flb_sp_cmd_key *ckey, - msgpack_packer *mp_pck, - int key_id) { - /* pack result stored in nums[key_id] */ - if (aggr_node->nums[key_id].type == FLB_SP_NUM_I64) { - msgpack_pack_int64(mp_pck, aggr_node->nums[key_id].i64); - } - else if (aggr_node->nums[key_id].type == FLB_SP_NUM_F64) { - msgpack_pack_float(mp_pck, aggr_node->nums[key_id].f64); - } -} - -void aggregate_func_calc_count(struct aggregate_node *aggr_node, - struct flb_sp_cmd_key *ckey, - msgpack_packer *mp_pck, - int key_id) { - /* number of records in total */ - msgpack_pack_int64(mp_pck, aggr_node->records); -} - -void aggregate_func_remove_sum(struct aggregate_node *aggr_node, - struct aggregate_node *aggr_node_prev, - int key_id) { - if (aggr_node->nums[key_id].type == FLB_SP_NUM_I64) { - aggr_node->nums[key_id].i64 -= aggr_node_prev->nums[key_id].i64; - } - else if (aggr_node->nums[key_id].type == FLB_SP_NUM_F64) { - aggr_node->nums[key_id].f64 -= aggr_node_prev->nums[key_id].f64; - } -} - -void aggregate_func_remove_nop(struct aggregate_node *aggr_node, - struct aggregate_node *aggr_node_prev, - int key_id) { -} - -void aggregate_func_add_timeseries_forecast(struct aggregate_node *aggr_node, - struct flb_sp_cmd_key *ckey, - int key_id, - struct flb_time *tms, - int64_t ival, double dval) -{ - double x; - double y; - struct timeseries_forecast *forecast; - - forecast = (struct timeseries_forecast *) aggr_node->aggregate_data[key_id]; - if (!forecast) { - forecast = (struct timeseries_forecast *) flb_calloc(1, sizeof(struct timeseries_forecast)); - /* fixme: return if error */ - - forecast->future_time = ckey->constant; - aggr_node->aggregate_data[key_id] = (struct aggregate_data *) forecast; - } - - if (!forecast->offset) { - forecast->offset = flb_time_to_double(tms); - } - - x = flb_time_to_double(tms) - forecast->offset; - - forecast->latest_x = x; - - if (ival) { - y = (double) ival; - } - else { - y = dval; - } - - forecast->sigma_x += x; - forecast->sigma_y += y; - - forecast->sigma_xy += x * y; - forecast->sigma_x2 += x * x; -} - -void aggregate_func_calc_timeseries_forecast(struct aggregate_node *aggr_node, - struct flb_sp_cmd_key *ckey, - msgpack_packer *mp_pck, - int key_id) -{ - double mean_x; - double mean_y; - double var_x; - double cov_xy; - double result; - /* y = b0 + b1 * x */ - double b0; - double b1; - struct timeseries_forecast *forecast; - - forecast = (struct timeseries_forecast *) aggr_node->aggregate_data[key_id]; - - mean_x = forecast->sigma_x / aggr_node->records; - mean_y = forecast->sigma_y / aggr_node->records; - cov_xy = (forecast->sigma_xy / (double) aggr_node->records) - mean_x * mean_y; - var_x = (forecast->sigma_x2 / aggr_node->records) - mean_x * mean_x; - - b1 = cov_xy / var_x; - b0 = mean_y - b1 * mean_x; - - result = b0 + b1 * (forecast->future_time + forecast->latest_x); - - msgpack_pack_float(mp_pck, result); -} - -void aggregate_func_remove_timeseries_forecast(struct aggregate_node *aggr_node, - struct aggregate_node *aggr_node_prev, - int key_id) -{ - struct timeseries_forecast *forecast_w; - struct timeseries_forecast *forecast_h; - - forecast_w = (struct timeseries_forecast *) aggr_node->aggregate_data[key_id]; - forecast_h = (struct timeseries_forecast *) aggr_node_prev->aggregate_data[key_id]; - - forecast_w->sigma_x -= forecast_h->sigma_x; - forecast_w->sigma_y -= forecast_h->sigma_y; - forecast_w->sigma_xy -= forecast_h->sigma_xy; - forecast_w->sigma_x2 -= forecast_h->sigma_x2; -} - -void aggregate_func_destroy_sum(struct aggregate_node *aggr_node, - int key_id) -{ -} - -void aggregate_func_destroy_timeseries_forecast(struct aggregate_node *aggr_node, - int key_id) -{ - flb_free(aggr_node->aggregate_data[key_id]); -} - -aggregate_function_clone aggregate_func_clone[AGGREGATE_FUNCTIONS] = { - aggregate_func_clone_nop, - aggregate_func_clone_nop, - aggregate_func_clone_nop, - aggregate_func_clone_nop, - aggregate_func_clone_nop, - aggregate_func_clone_timeseries_forecast, -}; - -aggregate_function_add aggregate_func_add[AGGREGATE_FUNCTIONS] = { - aggregate_func_add_sum, - aggregate_func_add_sum, - aggregate_func_add_count, - aggregate_func_add_min, - aggregate_func_add_max, - aggregate_func_add_timeseries_forecast, -}; - -aggregate_function_calc aggregate_func_calc[AGGREGATE_FUNCTIONS] = { - aggregate_func_calc_avg, - aggregate_func_calc_sum, - aggregate_func_calc_count, - aggregate_func_calc_sum, - aggregate_func_calc_sum, - aggregate_func_calc_timeseries_forecast, -}; - -aggregate_function_remove aggregate_func_remove[AGGREGATE_FUNCTIONS] = { - aggregate_func_remove_sum, - aggregate_func_remove_sum, - aggregate_func_remove_nop, - aggregate_func_remove_nop, - aggregate_func_remove_nop, - aggregate_func_remove_timeseries_forecast, -}; - -aggregate_function_destroy aggregate_func_destroy[AGGREGATE_FUNCTIONS] = { - aggregate_func_destroy_sum, - aggregate_func_destroy_sum, - aggregate_func_destroy_sum, - aggregate_func_destroy_sum, - aggregate_func_destroy_sum, - aggregate_func_destroy_timeseries_forecast, -}; diff --git a/fluent-bit/src/stream_processor/flb_sp_func_record.c b/fluent-bit/src/stream_processor/flb_sp_func_record.c deleted file mode 100644 index 26625a1ec..000000000 --- a/fluent-bit/src/stream_processor/flb_sp_func_record.c +++ /dev/null @@ -1,77 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <fluent-bit/flb_info.h> -#include <fluent-bit/flb_pack.h> -#include <fluent-bit/stream_processor/flb_sp.h> -#include <fluent-bit/stream_processor/flb_sp_parser.h> - -static inline void pack_key(msgpack_packer *mp_pck, - struct flb_sp_cmd_key *cmd_key, - const char *name, int len) -{ - if (cmd_key->alias) { - msgpack_pack_str(mp_pck, flb_sds_len(cmd_key->alias)); - msgpack_pack_str_body(mp_pck, cmd_key->alias, - flb_sds_len(cmd_key->alias)); - } - else { - msgpack_pack_str(mp_pck, len); - msgpack_pack_str_body(mp_pck, name, len); - } -} - -static int func_tag(const char *tag, int tag_len, - msgpack_packer *mp_pck, struct flb_sp_cmd_key *cmd_key) -{ - pack_key(mp_pck, cmd_key, "RECORD_TAG()", 12); - msgpack_pack_str(mp_pck, tag_len); - msgpack_pack_str_body(mp_pck, tag, tag_len); - - return 1; -} - -static int func_time(struct flb_time *tms, msgpack_packer *mp_pck, - struct flb_sp_cmd_key *cmd_key) -{ - double t; - - t = flb_time_to_double(tms); - pack_key(mp_pck, cmd_key, "RECORD_TIME()", 13); - msgpack_pack_double(mp_pck, t); - - return 1; -} - -/* - * Wrapper to handle record functions, returns the number of entries added - * to the map. - */ -int flb_sp_func_record(const char *tag, int tag_len, struct flb_time *tms, - msgpack_packer *mp_pck, struct flb_sp_cmd_key *cmd_key) -{ - switch (cmd_key->record_func) { - case FLB_SP_RECORD_TAG: - return func_tag(tag, tag_len, mp_pck, cmd_key); - case FLB_SP_RECORD_TIME: - return func_time(tms, mp_pck, cmd_key); - }; - - return 0; -} diff --git a/fluent-bit/src/stream_processor/flb_sp_func_time.c b/fluent-bit/src/stream_processor/flb_sp_func_time.c deleted file mode 100644 index 3e75eb775..000000000 --- a/fluent-bit/src/stream_processor/flb_sp_func_time.c +++ /dev/null @@ -1,95 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <fluent-bit/flb_info.h> -#include <fluent-bit/flb_pack.h> -#include <fluent-bit/stream_processor/flb_sp.h> -#include <fluent-bit/stream_processor/flb_sp_parser.h> - -static inline void pack_key(msgpack_packer *mp_pck, - struct flb_sp_cmd_key *cmd_key, - const char *name, int len) -{ - if (cmd_key->alias) { - msgpack_pack_str(mp_pck, flb_sds_len(cmd_key->alias)); - msgpack_pack_str_body(mp_pck, cmd_key->alias, - flb_sds_len(cmd_key->alias)); - } - else { - msgpack_pack_str(mp_pck, len); - msgpack_pack_str_body(mp_pck, name, len); - } -} - -static int func_now(msgpack_packer *mp_pck, struct flb_sp_cmd_key *cmd_key) -{ - size_t len; - time_t now; - char buf[32]; - struct tm *local; - - local = flb_malloc(sizeof(struct tm)); - if (!local) { - flb_errno(); - return 0; - } - - /* Get current system time */ - now = time(NULL); - localtime_r(&now, local); - - /* Format string value */ - len = strftime(buf, sizeof(buf) - 1, "%Y-%m-%d %H:%M:%S", local); - flb_free(local); - - pack_key(mp_pck, cmd_key, "NOW()", 5); - msgpack_pack_str(mp_pck, len); - msgpack_pack_str_body(mp_pck, buf, len); - - return 1; -} - -static int func_unix_timestamp(msgpack_packer *mp_pck, - struct flb_sp_cmd_key *cmd_key) -{ - time_t now; - - /* Get unix timestamp */ - now = time(NULL); - - pack_key(mp_pck, cmd_key, "UNIX_TIMESTAMP()", 16); - msgpack_pack_uint64(mp_pck, now); - return 1; -} - -/* - * Wrapper to handle time functions, returns the number of entries added - * to the map. - */ -int flb_sp_func_time(msgpack_packer *mp_pck, struct flb_sp_cmd_key *cmd_key) -{ - switch (cmd_key->time_func) { - case FLB_SP_NOW: - return func_now(mp_pck, cmd_key); - case FLB_SP_UNIX_TIMESTAMP: - return func_unix_timestamp(mp_pck, cmd_key); - }; - - return 0; -} diff --git a/fluent-bit/src/stream_processor/flb_sp_groupby.c b/fluent-bit/src/stream_processor/flb_sp_groupby.c deleted file mode 100644 index 9cd72c230..000000000 --- a/fluent-bit/src/stream_processor/flb_sp_groupby.c +++ /dev/null @@ -1,82 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <fluent-bit/flb_info.h> -#include <fluent-bit/stream_processor/flb_sp.h> - -int flb_sp_groupby_compare(const void *lhs, const void *rhs) -{ - int i; - int strcmp_result; - struct aggregate_node *left = (struct aggregate_node *) lhs; - struct aggregate_node *right = (struct aggregate_node *) rhs; - struct aggregate_num *lval; - struct aggregate_num *rval; - - for (i = 0; i < left->groupby_keys; i++) { - lval = &left->groupby_nums[i]; - rval = &right->groupby_nums[i]; - - /* Convert integer to double if a float value appears on one side */ - if (lval->type == FLB_SP_NUM_I64 && rval->type == FLB_SP_NUM_F64) { - lval->type = FLB_SP_NUM_F64; - lval->f64 = (double) lval->i64; - } - else if (lval->type == FLB_SP_NUM_F64 && rval->type == FLB_SP_NUM_I64) { - rval->type = FLB_SP_NUM_F64; - rval->f64 = (double) rval->i64; - } - - /* Comparison */ - if (lval->type == FLB_SP_BOOLEAN && rval->type == FLB_SP_BOOLEAN) { - if (lval->boolean != rval->boolean) { - return 1; - } - } - else if (lval->type == FLB_SP_NUM_I64 && rval->type == FLB_SP_NUM_I64) { - if (lval->i64 > rval->i64) { - return 1; - } - - if (lval->i64 < rval->i64) { - return -1; - } - } - else if (lval->type == FLB_SP_NUM_F64 && rval->type == FLB_SP_NUM_F64) { - if (lval->f64 > rval->f64) { - return 1; - } - - if (lval->f64 < rval->f64) { - return -1; - } - } - else if (lval->type == FLB_SP_STRING && rval->type == FLB_SP_STRING) { - strcmp_result = strcmp((const char *) lval->string, (const char *) rval->string); - if (strcmp_result != 0) { - return strcmp_result; - } - } - else { /* Sides have different types */ - return -1; - } - } - - return 0; -} diff --git a/fluent-bit/src/stream_processor/flb_sp_key.c b/fluent-bit/src/stream_processor/flb_sp_key.c deleted file mode 100644 index 945621843..000000000 --- a/fluent-bit/src/stream_processor/flb_sp_key.c +++ /dev/null @@ -1,231 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <fluent-bit/flb_info.h> -#include <fluent-bit/flb_log.h> -#include <fluent-bit/flb_sds.h> -#include <fluent-bit/flb_mem.h> -#include <fluent-bit/flb_slist.h> - -#include <fluent-bit/stream_processor/flb_sp.h> -#include <fluent-bit/stream_processor/flb_sp_parser.h> - -void flb_sp_key_value_print(struct flb_sp_value *v) -{ - if (v->type == FLB_EXP_BOOL) { - if (v->val.boolean) { - printf("true"); - } - else { - printf("false"); - } - } - else if (v->type == FLB_EXP_INT) { - printf("%" PRId64, v->val.i64); - } - else if (v->type == FLB_EXP_FLOAT) { - printf("%f", v->val.f64); - } - else if (v->type == FLB_EXP_STRING) { - printf("%s", v->val.string); - } - else if (v->type == FLB_EXP_NULL) { - printf("NULL"); - } -} - -/* Map msgpack object intp flb_sp_value representation */ -static int msgpack_object_to_sp_value(msgpack_object o, - struct flb_sp_value *result) -{ - result->o = o; - - /* Compose result with found value */ - if (o.type == MSGPACK_OBJECT_BOOLEAN) { - result->type = FLB_EXP_BOOL; - result->val.boolean = o.via.boolean; - return 0; - } - else if (o.type == MSGPACK_OBJECT_POSITIVE_INTEGER || - o.type == MSGPACK_OBJECT_NEGATIVE_INTEGER) { - result->type = FLB_EXP_INT; - result->val.i64 = o.via.i64; - return 0; - } - else if (o.type == MSGPACK_OBJECT_FLOAT32 || - o.type == MSGPACK_OBJECT_FLOAT) { - result->type = FLB_EXP_FLOAT; - result->val.f64 = o.via.f64; - return 0; - } - else if (o.type == MSGPACK_OBJECT_STR) { - result->type = FLB_EXP_STRING; - result->val.string = flb_sds_create_len((char *) o.via.str.ptr, - o.via.str.size); - return 0; - } - else if (o.type == MSGPACK_OBJECT_MAP) { - /* return boolean 'true', just denoting the existence of the key */ - result->type = FLB_EXP_BOOL; - result->val.boolean = true; - return 0; - } - else if (o.type == MSGPACK_OBJECT_NIL) { - result->type = FLB_EXP_NULL; - return 0; - } - - return -1; -} - -/* Lookup perfect match of sub-keys and map content */ -static int subkey_to_value(msgpack_object *map, struct mk_list *subkeys, - struct flb_sp_value *result) -{ - int i = 0; - int ret; - int levels; - int matched = 0; - msgpack_object *key_found = NULL; - msgpack_object key; - msgpack_object val; - msgpack_object cur_map; - struct mk_list *head; - struct flb_slist_entry *entry; - - /* Expected number of map levels in the map */ - levels = mk_list_size(subkeys); - - cur_map = *map; - - mk_list_foreach(head, subkeys) { - /* Key expected key entry */ - entry = mk_list_entry(head, struct flb_slist_entry, _head); - - if (cur_map.type != MSGPACK_OBJECT_MAP) { - break; - } - - /* Get map entry that matches entry name */ - for (i = 0; i < cur_map.via.map.size; i++) { - key = cur_map.via.map.ptr[i].key; - val = cur_map.via.map.ptr[i].val; - - /* A bit obvious, but it's better to validate data type */ - if (key.type != MSGPACK_OBJECT_STR) { - continue; - } - - /* Compare strings by length and content */ - if (flb_sds_cmp(entry->str, - (char *) key.via.str.ptr, - key.via.str.size) != 0) { - key_found = NULL; - continue; - } - - key_found = &key; - cur_map = val; - matched++; - break; - } - - if (levels == matched) { - break; - } - } - - /* No matches */ - if (!key_found || (matched > 0 && levels != matched)) { - return -1; - } - - ret = msgpack_object_to_sp_value(val, result); - if (ret == -1) { - //flb_error("[sp key] cannot process key value"); - return -1; - } - - return 0; -} - -struct flb_sp_value *flb_sp_key_to_value(flb_sds_t ckey, - msgpack_object map, - struct mk_list *subkeys) -{ - int i; - int ret; - int map_size; - msgpack_object key; - msgpack_object val; - struct flb_sp_value *result; - - map_size = map.via.map.size; - for (i = 0; i < map_size; i++) { - key = map.via.map.ptr[i].key; - val = map.via.map.ptr[i].val; - - /* Compare by length and by key name */ - if (flb_sds_cmp(ckey, key.via.str.ptr, key.via.str.size) != 0) { - continue; - } - - result = flb_calloc(1, sizeof(struct flb_sp_value)); - if (!result) { - flb_errno(); - return NULL; - } - result->o = val; - - if (val.type == MSGPACK_OBJECT_MAP && subkeys != NULL) { - ret = subkey_to_value(&val, subkeys, result); - if (ret == 0) { - return result; - } - else { - flb_free(result); - return NULL; - } - } - else { - ret = msgpack_object_to_sp_value(val, result); - if (ret == -1) { - flb_error("[sp key] cannot process key value"); - flb_free(result); - return NULL; - } - } - - return result; - } - - /* - * NULL return means: failed memory allocation, an invalid value, - * or non-existing key. - */ - return NULL; -} - -void flb_sp_key_value_destroy(struct flb_sp_value *v) -{ - if (v->type == FLB_EXP_STRING) { - flb_sds_destroy(v->val.string); - } - flb_free(v); -} diff --git a/fluent-bit/src/stream_processor/flb_sp_snapshot.c b/fluent-bit/src/stream_processor/flb_sp_snapshot.c deleted file mode 100644 index edef823b4..000000000 --- a/fluent-bit/src/stream_processor/flb_sp_snapshot.c +++ /dev/null @@ -1,277 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <fluent-bit/flb_mem.h> -#include <fluent-bit/stream_processor/flb_sp.h> -#include <fluent-bit/stream_processor/flb_sp_parser.h> -#include <fluent-bit/stream_processor/flb_sp_snapshot.h> - -static struct flb_sp_snapshot_page *snapshot_page_create() -{ - struct flb_sp_snapshot_page *page; - - page = (struct flb_sp_snapshot_page *) - flb_calloc(1, sizeof(struct flb_sp_snapshot_page)); - if (!page) { - flb_errno(); - return NULL; - } - - page->snapshot_page = (char *) flb_malloc(SNAPSHOT_PAGE_SIZE); - if (!page->snapshot_page) { - flb_errno(); - flb_free(page); - return NULL; - } - - return page; -} - -static int snapshot_cleanup(struct flb_sp_snapshot *snapshot, struct flb_time *tms) -{ - int ok; - size_t off; - size_t off_copy; - msgpack_unpacked result; - msgpack_object *obj; - struct flb_time tms0; - struct flb_sp_snapshot_page *page; - - ok = MSGPACK_UNPACK_SUCCESS; - off = 0; - - while (mk_list_is_empty(&snapshot->pages) != 0) { - page = mk_list_entry_first(&snapshot->pages, struct flb_sp_snapshot_page, - _head); - off = page->start_pos; - off_copy = off; - - msgpack_unpacked_init(&result); - - while (msgpack_unpack_next(&result, page->snapshot_page, page->end_pos, - &off) == ok) { - - if (snapshot->record_limit > 0 && - snapshot->records > snapshot->record_limit) { - page->start_pos = off; - snapshot->records--; - snapshot->size = snapshot->size - (off - off_copy); - off_copy = off; - - continue; - } - - /* extract timestamp */ - flb_time_pop_from_msgpack(&tms0, &result, &obj); - - if (snapshot->time_limit > 0 && - tms->tm.tv_sec - tms0.tm.tv_sec > snapshot->time_limit) { - page->start_pos = off; - snapshot->records--; - snapshot->size = snapshot->size - (off - off_copy); - off_copy = off; - - continue; - } - - break; - } - - msgpack_unpacked_destroy(&result); - - /* If page is empty, free the page and move to the next one */ - if (page->start_pos != page->end_pos) { - break; - } - - mk_list_del(&page->_head); - flb_free(page->snapshot_page); - flb_free(page); - } - - return 0; -} - -static bool snapshot_page_is_full(struct flb_sp_snapshot_page *page, size_t buf_size) -{ - return SNAPSHOT_PAGE_SIZE - page->end_pos < buf_size; -} - -char *flb_sp_snapshot_name_from_flush(flb_sds_t name) -{ - return name + sizeof("__flush_") - 1; -} - -int flb_sp_snapshot_update(struct flb_sp_task *task, const char *buf_data, - size_t buf_size, struct flb_time *tms) -{ - int ok; - size_t off = 0; - struct flb_time tm; - struct flb_sp_snapshot *snapshot; - struct flb_sp_snapshot_page *page; - msgpack_unpacked result; - msgpack_object *obj; - - ok = MSGPACK_UNPACK_SUCCESS; - msgpack_unpacked_init(&result); - - if (buf_size <= 0) { - return -1; - } - - snapshot = (struct flb_sp_snapshot *) task->snapshot; - - /* Create a snapshot pgae if the list is empty */ - if (mk_list_is_empty(&snapshot->pages) == 0) { - page = snapshot_page_create(); - if (!page) { - flb_errno(); - return -1; - } - - mk_list_add(&page->_head, &snapshot->pages); - } - else { - page = mk_list_entry_last(&snapshot->pages, struct flb_sp_snapshot_page, _head); - - if (snapshot_page_is_full(page, buf_size)) { - page = snapshot_page_create(); - if (!page) { - flb_errno(); - return -1; - } - - mk_list_add(&page->_head, &snapshot->pages); - } - } - - memcpy(page->snapshot_page + page->end_pos, buf_data, buf_size); - page->end_pos = page->end_pos + buf_size; - - /* Get the last timestamp */ - while (msgpack_unpack_next(&result, page->snapshot_page, - page->end_pos - page->start_pos, &off) == ok) { - flb_time_pop_from_msgpack(&tm, &result, &obj); - } - - msgpack_unpacked_destroy(&result); - - snapshot->records++; - snapshot->size = snapshot->size + buf_size; - - /* Remove records from snapshot pages based on time/length window */ - snapshot_cleanup(snapshot, tms); - - return 0; -} - -int flb_sp_snapshot_flush(struct flb_sp *sp, struct flb_sp_task *task, - char **out_buf_data, size_t *out_buf_size) -{ - size_t off; - size_t page_size; - char *snapshot_name; - char *out_buf_data_tmp; - struct flb_sp_cmd *cmd; - struct mk_list *tmp; - struct mk_list *head; - struct mk_list *snapshot_head; - struct flb_sp_task *snapshot_task; - struct flb_sp_snapshot *snapshot; - struct flb_sp_snapshot_page *page; - - off = 0; - cmd = task->cmd; - snapshot_name = flb_sp_snapshot_name_from_flush(cmd->stream_name); - - /* Lookup Tasks that matches the incoming instance data */ - mk_list_foreach(head, &sp->tasks) { - snapshot_task = mk_list_entry(head, struct flb_sp_task, _head); - cmd = snapshot_task->cmd; - - if (cmd->type == FLB_SP_CREATE_SNAPSHOT && - flb_sds_cmp(cmd->stream_name, snapshot_name, - strlen(snapshot_name)) == 0) { - - snapshot = (struct flb_sp_snapshot *) snapshot_task->snapshot; - - if (snapshot->size == 0) { - break; - } - - if (*out_buf_data == NULL) { - *out_buf_data = (char *) flb_malloc(snapshot->size); - if (!*out_buf_data) { - flb_errno(); - return -1; - } - *out_buf_size = snapshot->size; - } - else { - out_buf_data_tmp = (char *) flb_realloc(*out_buf_data, - *out_buf_size + snapshot->size); - if (!out_buf_data_tmp) { - flb_errno(); - return -1; - } - *out_buf_data = out_buf_data_tmp; - *out_buf_size = *out_buf_size + snapshot->size; - } - - mk_list_foreach_safe(snapshot_head, tmp, &snapshot->pages) { - page = mk_list_entry_first(&snapshot->pages, - struct flb_sp_snapshot_page, _head); - page_size = page->end_pos - page->start_pos; - memcpy(*out_buf_data + off, - page->snapshot_page + page->start_pos, page_size); - off = off + page_size; - - /* Remove page from list */ - mk_list_del(&page->_head); - flb_free(page->snapshot_page); - flb_free(page); - } - - mk_list_init(&snapshot->pages); - - snapshot->records = 0; - snapshot->size = 0; - } - } - - return 0; -} - -void flb_sp_snapshot_destroy(struct flb_sp_snapshot *snapshot) -{ - struct mk_list *head; - struct mk_list *tmp; - struct flb_sp_snapshot_page *page; - - if (snapshot != NULL) { - mk_list_foreach_safe(head, tmp, &snapshot->pages) { - page = mk_list_entry(head, struct flb_sp_snapshot_page, _head); - mk_list_del(&page->_head); - flb_free(page->snapshot_page); - flb_free(page); - } - flb_free(snapshot); - } -} diff --git a/fluent-bit/src/stream_processor/flb_sp_stream.c b/fluent-bit/src/stream_processor/flb_sp_stream.c deleted file mode 100644 index b4f8a37a2..000000000 --- a/fluent-bit/src/stream_processor/flb_sp_stream.c +++ /dev/null @@ -1,168 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <fluent-bit/flb_info.h> -#include <fluent-bit/flb_log.h> -#include <fluent-bit/flb_mem.h> -#include <fluent-bit/flb_input.h> -#include <fluent-bit/flb_metrics.h> -#include <fluent-bit/flb_storage.h> -#include <fluent-bit/flb_utils.h> -#include <fluent-bit/stream_processor/flb_sp.h> -#include <fluent-bit/stream_processor/flb_sp_parser.h> -#include <fluent-bit/stream_processor/flb_sp_stream.h> - -/* Function defined in plugins/in_stream_processor/sp.c */ -int in_stream_processor_add_chunk(const char *buf_data, size_t buf_size, - struct flb_input_instance *in); - -int flb_sp_stream_create(const char *name, struct flb_sp_task *task, - struct flb_sp *sp) -{ - int ret; - const char *tmp; - struct flb_input_instance *in; - struct flb_sp_stream *stream; - - /* The name must be different than an input plugin instance name or alias */ - ret = flb_input_name_exists(name, sp->config); - if (ret == FLB_TRUE) { - flb_error("[sp] stream name '%s' already exists", name); - return -1; - } - - /* Create stream context for 'stream processor' */ - stream = flb_calloc(1, sizeof(struct flb_sp_stream)); - if (!stream) { - flb_errno(); - return -1; - } - stream->name = flb_sds_create(name); - if (!stream->name) { - flb_free(stream); - return -1; - } - - /* - * Register an input plugin instance using 'in_stream_processor', that one - * is used as the parent plugin to ingest data back into Fluent Bit - * data pipeline. - */ - in = flb_input_new(sp->config, "stream_processor", NULL, FLB_FALSE); - if (!in) { - flb_error("[sp] cannot create instance of in_stream_processor"); - flb_free(stream); - return -1; - } - - /* Set an alias, otherwise the stream will be called stream_processor.N */ - ret = flb_input_set_property(in, "alias", name); - if (ret == -1) { - flb_warn("[sp] cannot set stream name, using fallback name %s", - in->name); - } - - /* - * Lookup for Stream properties: at this point we only care about a - * possible Tag defined in the query, e.g: - * - * CREATE STREAM data WITH(tag='mydata') as SELECT * FROM STREAM:apache; - */ - tmp = flb_sp_cmd_stream_prop_get(task->cmd, "tag"); - if (tmp) { - /* - * Duplicate value in a new variable since input instance property - * will be released upon plugin exit. - */ - stream->tag = flb_sds_create(tmp); - if (!stream->tag) { - flb_error("[sp] cannot set Tag property"); - flb_sp_stream_destroy(stream, sp); - return -1; - } - - /* Tag property is just an assignation, cannot fail */ - flb_input_set_property(in, "tag", stream->tag); - } - - /* - * Check if the new stream is 'routable' or not - */ - tmp = flb_sp_cmd_stream_prop_get(task->cmd, "routable"); - if (tmp) { - stream->routable = flb_utils_bool(tmp); - flb_input_set_property(in, "routable", tmp); - } - - /* - * Set storage type - */ - tmp = flb_sp_cmd_stream_prop_get(task->cmd, "storage.type"); - if (tmp) { - flb_input_set_property(in, "storage.type", tmp); - } - - /* Initialize instance */ - ret = flb_input_instance_init(in, sp->config); - if (ret == -1) { - flb_error("[sp] cannot initialize instance of in_stream_processor"); - flb_input_instance_exit(in, sp->config); - flb_input_instance_destroy(in); - } - stream->in = in; - - /* Initialize plugin collector (event callback) */ - flb_input_collector_start(0, in); - -#ifdef FLB_HAVE_METRICS - /* Override Metrics title */ - ret = flb_metrics_title(name, in->metrics); - if (ret == -1) { - flb_warn("[sp] cannot set metrics title, using fallback name %s", - in->name); - } -#endif - - /* Storage context */ - ret = flb_storage_input_create(sp->config->cio, in); - if (ret == -1) { - flb_error("[sp] cannot initialize storage for stream '%s'", - name); - flb_sp_stream_destroy(stream, sp); - return -1; - } - - task->stream = stream; - return 0; -} - -int flb_sp_stream_append_data(const char *buf_data, size_t buf_size, - struct flb_sp_stream *stream) -{ - return in_stream_processor_add_chunk(buf_data, buf_size, stream->in); -} - -void flb_sp_stream_destroy(struct flb_sp_stream *stream, struct flb_sp *sp) -{ - flb_sds_destroy(stream->name); - flb_sds_destroy(stream->tag); - flb_input_instance_exit(stream->in, sp->config); - flb_input_instance_destroy(stream->in); - flb_free(stream); -} diff --git a/fluent-bit/src/stream_processor/flb_sp_window.c b/fluent-bit/src/stream_processor/flb_sp_window.c deleted file mode 100644 index 3937b4273..000000000 --- a/fluent-bit/src/stream_processor/flb_sp_window.c +++ /dev/null @@ -1,122 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <fluent-bit/stream_processor/flb_sp.h> -#include <fluent-bit/stream_processor/flb_sp_window.h> -#include <fluent-bit/stream_processor/flb_sp_parser.h> -#include <fluent-bit/stream_processor/flb_sp_groupby.h> -#include <fluent-bit/stream_processor/flb_sp_aggregate_func.h> - -void flb_sp_window_prune(struct flb_sp_task *task) -{ - int i; - int map_entries; - rb_result_t result; - struct aggregate_node *aggr_node; - struct aggregate_node *aggr_node_hs; - struct mk_list *tmp; - struct mk_list *head; - struct flb_sp_hopping_slot *hs; - struct rb_tree_node *rb_result; - struct flb_sp_cmd_key *ckey; - struct flb_sp_cmd *cmd = task->cmd; - - switch (task->window.type) { - case FLB_SP_WINDOW_DEFAULT: - case FLB_SP_WINDOW_TUMBLING: - if (task->window.records > 0) { - mk_list_foreach_safe(head, tmp, &task->window.aggregate_list) { - aggr_node = mk_list_entry(head, struct aggregate_node, _head); - mk_list_del(&aggr_node->_head); - flb_sp_aggregate_node_destroy(cmd, aggr_node); - } - - rb_tree_destroy(&task->window.aggregate_tree); - mk_list_init(&task->window.aggregate_list); - rb_tree_new(&task->window.aggregate_tree, flb_sp_groupby_compare); - task->window.records = 0; - } - break; - case FLB_SP_WINDOW_HOPPING: - if (mk_list_size(&task->window.hopping_slot) == 0) { - return; - } - - hs = mk_list_entry_first(&task->window.hopping_slot, - struct flb_sp_hopping_slot, _head); - mk_list_foreach_safe(head, tmp, &task->window.aggregate_list) { - aggr_node = mk_list_entry(head, struct aggregate_node, _head); - result = rb_tree_find(&hs->aggregate_tree, aggr_node, &rb_result); - if (result == RB_OK) { - aggr_node_hs = mk_list_entry(rb_result, struct aggregate_node, _rb_head); - if (aggr_node_hs->records == aggr_node->records) { - rb_tree_remove(&task->window.aggregate_tree, &aggr_node->_rb_head); - mk_list_del(&aggr_node->_head); - // Destroy aggregation node - flb_sp_aggregate_node_destroy(cmd, aggr_node); - } - else { - aggr_node->records -= aggr_node_hs->records; - map_entries = mk_list_size(&cmd->keys); - - ckey = mk_list_entry_first(&cmd->keys, - struct flb_sp_cmd_key, _head); - for (i = 0; i < map_entries; i++) { - if (ckey->aggr_func) { - aggregate_func_remove[ckey->aggr_func - 1](aggr_node, aggr_node_hs, i); - } - - ckey = mk_list_entry_next(&ckey->_head, struct flb_sp_cmd_key, - _head, &cmd->keys); - } - } - } - } - task->window.records -= hs->records; - - /* Destroy hopping slot */ - mk_list_foreach_safe(head, tmp, &hs->aggregate_list) { - aggr_node_hs = mk_list_entry(head, struct aggregate_node, _head); - mk_list_del(&aggr_node_hs->_head); - flb_sp_aggregate_node_destroy(cmd, aggr_node_hs); - } - rb_tree_destroy(&hs->aggregate_tree); - mk_list_del(&hs->_head); - flb_free(hs); - - break; - } -} - -int flb_sp_window_populate(struct flb_sp_task *task, const char *buf_data, - size_t buf_size) -{ - switch (task->window.type) { - case FLB_SP_WINDOW_DEFAULT: - case FLB_SP_WINDOW_TUMBLING: - case FLB_SP_WINDOW_HOPPING: - break; - default: - flb_error("[sp] error populating window for '%s': window type unknown", - task->name); - return -1; - } - - return 0; -} diff --git a/fluent-bit/src/stream_processor/parser/CMakeLists.txt b/fluent-bit/src/stream_processor/parser/CMakeLists.txt deleted file mode 100644 index d14b02bbd..000000000 --- a/fluent-bit/src/stream_processor/parser/CMakeLists.txt +++ /dev/null @@ -1,31 +0,0 @@ -flex_target(lexer sql.l "${CMAKE_CURRENT_BINARY_DIR}/sql_lex.c" - DEFINES_FILE "${CMAKE_CURRENT_BINARY_DIR}/sql_lex.h" - ) -bison_target(parser sql.y "${CMAKE_CURRENT_BINARY_DIR}/sql_parser.c") - -set(sources - flb_sp_parser.c - ) - -if(CMAKE_SYSTEM_NAME MATCHES "Windows") - FLB_DEFINITION(YY_NO_UNISTD_H) - message(STATUS "Specifying YY_NO_UNISTD_H") -endif() - -include_directories( - ${CMAKE_CURRENT_SOURCE_DIR} - ${CMAKE_CURRENT_BINARY_DIR} - ) - -add_library(flb-sp-parser STATIC - ${sources} - "${CMAKE_CURRENT_BINARY_DIR}/sql_lex.c" - "${CMAKE_CURRENT_BINARY_DIR}/sql_parser.c" - ) - -add_flex_bison_dependency(lexer parser) -add_dependencies(flb-sp-parser onigmo-static) - -if(FLB_JEMALLOC) - target_link_libraries(flb-sp-parser libjemalloc) -endif() diff --git a/fluent-bit/src/stream_processor/parser/flb_sp_parser.c b/fluent-bit/src/stream_processor/parser/flb_sp_parser.c deleted file mode 100644 index 429d0b4c1..000000000 --- a/fluent-bit/src/stream_processor/parser/flb_sp_parser.c +++ /dev/null @@ -1,851 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <stdio.h> -#include <stdlib.h> -#include <string.h> - -#include <fluent-bit/flb_info.h> -#include <fluent-bit/flb_log.h> -#include <fluent-bit/flb_mem.h> -#include <fluent-bit/flb_str.h> -#include <fluent-bit/flb_sds.h> -#include <fluent-bit/flb_slist.h> -#include <fluent-bit/stream_processor/flb_sp_parser.h> -#include <fluent-bit/stream_processor/flb_sp_aggregate_func.h> -#include <fluent-bit/stream_processor/flb_sp_record_func.h> - -#include "sql_parser.h" -#include "sql_lex.h" - -static int swap_tmp_subkeys(struct mk_list **target, struct flb_sp_cmd *cmd) -{ - /* Map context keys into this command key structure */ - *target = cmd->tmp_subkeys; - - cmd->tmp_subkeys = flb_malloc(sizeof(struct mk_list)); - if (!cmd->tmp_subkeys) { - flb_errno(); - cmd->tmp_subkeys = *target; - cmd->status = FLB_SP_ERROR; - return -1; - } - - flb_slist_create(cmd->tmp_subkeys); - return 0; -} - -void flb_sp_cmd_destroy(struct flb_sp_cmd *cmd) -{ - struct mk_list *head; - struct mk_list *tmp; - struct flb_sp_cmd_key *key; - struct flb_sp_cmd_gb_key *gb_key; - struct flb_sp_cmd_prop *prop; - - /* remove keys */ - mk_list_foreach_safe(head, tmp, &cmd->keys) { - key = mk_list_entry(head, struct flb_sp_cmd_key, _head); - mk_list_del(&key->_head); - flb_sp_cmd_key_del(key); - } - - /* remove groupby keys */ - mk_list_foreach_safe(head, tmp, &cmd->gb_keys) { - gb_key = mk_list_entry(head, struct flb_sp_cmd_gb_key, _head); - mk_list_del(&gb_key->_head); - flb_sp_cmd_gb_key_del(gb_key); - } - - /* stream */ - if (cmd->stream_name) { - mk_list_foreach_safe(head, tmp, &cmd->stream_props) { - prop = mk_list_entry(head, struct flb_sp_cmd_prop, _head); - mk_list_del(&prop->_head); - flb_sp_cmd_stream_prop_del(prop); - } - flb_sds_destroy(cmd->stream_name); - } - flb_sds_destroy(cmd->source_name); - - if (mk_list_size(&cmd->cond_list) > 0) { - flb_sp_cmd_condition_del(cmd); - } - - if (cmd->tmp_subkeys) { - flb_slist_destroy(cmd->tmp_subkeys); - flb_free(cmd->tmp_subkeys); - } - - flb_free(cmd); -} - -void flb_sp_cmd_key_del(struct flb_sp_cmd_key *key) -{ - if (key->name) { - flb_sds_destroy(key->name); - } - if (key->alias) { - flb_sds_destroy(key->alias); - } - if (key->subkeys) { - flb_slist_destroy(key->subkeys); - flb_free(key->subkeys); - } - flb_free(key); -} - -void flb_sp_cmd_gb_key_del(struct flb_sp_cmd_gb_key *key) -{ - if (key->name) { - flb_sds_destroy(key->name); - } - if (key->subkeys) { - flb_slist_destroy(key->subkeys); - flb_free(key->subkeys); - } - flb_free(key); -} - -struct flb_sp_cmd_key *flb_sp_key_create(struct flb_sp_cmd *cmd, int func, - const char *key_name, - const char *key_alias) -{ - char tmp_alias[256]; - int s; - int ret; - int len; - int aggr_func = 0; - int time_func = 0; - int record_func = 0; - char *tmp; - struct mk_list *head; - struct flb_sp_cmd_key *key; - struct flb_slist_entry *entry; - - /* aggregation function ? */ - if (func >= FLB_SP_AVG && func <= FLB_SP_FORECAST) { - aggr_func = func; - } - else if (func >= FLB_SP_NOW && func <= FLB_SP_UNIX_TIMESTAMP) { - /* Time function */ - time_func = func; - } - else if (func >= FLB_SP_RECORD_TAG && func <= FLB_SP_RECORD_TIME) { - /* Record function */ - record_func = func; - } - - key = flb_calloc(1, sizeof(struct flb_sp_cmd_key)); - if (!key) { - flb_errno(); - cmd->status = FLB_SP_ERROR; - return NULL; - } - key->gb_key = NULL; - key->subkeys = NULL; - - /* key name and aliases works when the selection is not a wildcard */ - if (key_name) { - key->name = flb_sds_create(key_name); - if (!key->name) { - flb_sp_cmd_key_del(key); - cmd->status = FLB_SP_ERROR; - return NULL; - } - } - else { - /* - * Wildcard key only allowed on: - * - no aggregation mode (left side / first entry) - * - aggregation using COUNT(*) - */ - if (mk_list_size(&cmd->keys) > 0 && aggr_func == 0 && - record_func == 0 && time_func == 0) { - flb_sp_cmd_key_del(key); - cmd->status = FLB_SP_ERROR; - return NULL; - } - } - - if (key_alias) { - key->alias = flb_sds_create(key_alias); - if (!key->alias) { - flb_sp_cmd_key_del(key); - cmd->status = FLB_SP_ERROR; - return NULL; - } - } - - /* Aggregation function */ - if (aggr_func > 0) { - key->aggr_func = aggr_func; - } - else if (time_func > 0) { - key->time_func = time_func; - } - else if (record_func > 0) { - key->record_func = record_func; - } - - /* Lookup for any subkeys in the temporary list */ - if (mk_list_size(cmd->tmp_subkeys) > 0) { - ret = swap_tmp_subkeys(&key->subkeys, cmd); - if (ret == -1) { - flb_sp_cmd_key_del(key); - cmd->status = FLB_SP_ERROR; - return NULL; - } - - /* Compose a name key that include listed sub keys */ - if (!key->alias) { - s = flb_sds_len(key->name) + (16 * mk_list_size(key->subkeys)); - key->alias = flb_sds_create_size(s); - if (!key->alias) { - flb_sp_cmd_key_del(key); - return NULL; - } - - tmp = flb_sds_cat(key->alias, key->name, flb_sds_len(key->name)); - if (tmp != key->alias) { - key->alias = tmp; - } - - mk_list_foreach(head, key->subkeys) { - entry = mk_list_entry(head, struct flb_slist_entry, _head); - - /* prefix */ - tmp = flb_sds_cat(key->alias, "['", 2); - if (tmp) { - key->alias = tmp; - } - else { - flb_sp_cmd_key_del(key); - return NULL; - } - - /* selected key name */ - tmp = flb_sds_cat(key->alias, - entry->str, flb_sds_len(entry->str)); - if (tmp) { - key->alias = tmp; - } - else { - flb_sp_cmd_key_del(key); - return NULL; - } - - /* suffix */ - tmp = flb_sds_cat(key->alias, "']", 2); - if (tmp) { - key->alias = tmp; - } - else { - flb_sp_cmd_key_del(key); - return NULL; - } - } - - if (aggr_func) { - len = snprintf(tmp_alias, sizeof(tmp_alias) - 1, "%s(%s)", - aggregate_func_string[aggr_func - 1], key->alias); - - tmp = flb_sds_copy(key->alias, tmp_alias, len); - if (tmp) { - key->alias = tmp; - } - else { - flb_sp_cmd_key_del(key); - return NULL; - } - } - } - } - else if (aggr_func && !key->alias) { - if (key->name) { - len = snprintf(tmp_alias, sizeof(tmp_alias) - 1, "%s(%s)", - aggregate_func_string[aggr_func - 1], key->name); - } else { - len = snprintf(tmp_alias, sizeof(tmp_alias) - 1, "%s(*)", - aggregate_func_string[aggr_func - 1]); - } - - key->alias = flb_sds_create_len(tmp_alias, len); - if (!key->alias) { - flb_sp_cmd_key_del(key); - return NULL; - } - } - - return key; -} - -int flb_sp_cmd_key_add(struct flb_sp_cmd *cmd, int func, const char *key_name) -{ - struct flb_sp_cmd_key *key; - - key = flb_sp_key_create(cmd, func, key_name, cmd->alias); - - if (!key) { - return -1; - } - - mk_list_add(&key->_head, &cmd->keys); - - /* free key alias and set cmd->alias to null */ - if (cmd->alias) { - flb_free(cmd->alias); - cmd->alias = NULL; - } - - return 0; -} - -void flb_sp_cmd_alias_add(struct flb_sp_cmd *cmd, const char *key_alias) -{ - cmd->alias = (char *) key_alias; -} - -int flb_sp_cmd_source(struct flb_sp_cmd *cmd, int type, const char *source) -{ - cmd->source_type = type; - cmd->source_name = flb_sds_create(source); - if (!cmd->source_name) { - flb_errno(); - return -1; - } - - return 0; -} - -void flb_sp_cmd_dump(struct flb_sp_cmd *cmd) -{ - struct mk_list *head; - struct mk_list *tmp; - struct flb_sp_cmd_key *key; - - /* Lookup keys */ - printf("== KEYS ==\n"); - mk_list_foreach_safe(head, tmp, &cmd->keys) { - key = mk_list_entry(head, struct flb_sp_cmd_key, _head); - printf("- '%s'\n", key->name); - } - printf("== SOURCE ==\n"); - if (cmd->source_type == FLB_SP_STREAM) { - printf("stream => "); - } - else if (cmd->source_type == FLB_SP_TAG) { - printf("tag match => "); - } - - printf("'%s'\n", cmd->source_name); -} - -struct flb_sp_cmd *flb_sp_cmd_create(const char *sql) -{ - int ret; - yyscan_t scanner; - YY_BUFFER_STATE buf; - struct flb_sp_cmd *cmd; - - /* create context */ - cmd = flb_calloc(1, sizeof(struct flb_sp_cmd)); - if (!cmd) { - flb_errno(); - return NULL; - } - cmd->status = FLB_SP_OK; - cmd->type = FLB_SP_SELECT; - - mk_list_init(&cmd->stream_props); - mk_list_init(&cmd->keys); - - /* Condition linked list (we use them to free resources) */ - mk_list_init(&cmd->cond_list); - mk_list_init(&cmd->gb_keys); - - /* Allocate temporary list and initialize */ - cmd->tmp_subkeys = flb_malloc(sizeof(struct mk_list)); - if (!cmd->tmp_subkeys) { - flb_errno(); - flb_free(cmd); - return NULL; - } - flb_slist_create(cmd->tmp_subkeys); - - /* Flex/Bison work */ - flb_sp_lex_init(&scanner); - buf = flb_sp__scan_string(sql, scanner); - - ret = flb_sp_parse(cmd, sql, scanner); - - flb_sp__delete_buffer(buf, scanner); - flb_sp_lex_destroy(scanner); - - if (ret != 0) { - flb_sp_cmd_destroy(cmd); - return NULL; - } - - return cmd; -} - -int flb_sp_cmd_stream_new(struct flb_sp_cmd *cmd, const char *stream_name) -{ - cmd->stream_name = flb_sds_create(stream_name); - if (!cmd->stream_name) { - return -1; - } - - cmd->type = FLB_SP_CREATE_STREAM; - return 0; -} - -int flb_sp_cmd_snapshot_new(struct flb_sp_cmd *cmd, const char *snapshot_name) -{ - const char *tmp; - - cmd->stream_name = flb_sds_create(snapshot_name); - if (!cmd->stream_name) { - return -1; - } - - tmp = flb_sp_cmd_stream_prop_get(cmd, "tag"); - if (!tmp) { - cmd->status = FLB_SP_ERROR; - flb_error("[sp] tag for snapshot is required. Add WITH(tag = <TAG>) to the snapshot %s", - snapshot_name); - return -1; - } - - cmd->type = FLB_SP_CREATE_SNAPSHOT; - - return 0; -} - -int flb_sp_cmd_snapshot_flush_new(struct flb_sp_cmd *cmd, const char *snapshot_name) -{ - cmd->stream_name = flb_sds_cat(flb_sds_create("__flush_"), - snapshot_name, strlen(snapshot_name)); - - if (!cmd->stream_name) { - return -1; - } - - cmd->type = FLB_SP_FLUSH_SNAPSHOT; - - return 0; -} - -int flb_sp_cmd_stream_prop_add(struct flb_sp_cmd *cmd, const char *key, const char *val) -{ - struct flb_sp_cmd_prop *prop; - - prop = flb_malloc(sizeof(struct flb_sp_cmd_prop)); - if (!prop) { - flb_errno(); - return -1; - } - - prop->key = flb_sds_create(key); - if (!prop->key) { - flb_free(prop); - return -1; - } - - prop->val = flb_sds_create(val); - if (!prop->val) { - flb_free(prop->key); - flb_free(prop); - return -1; - } - - mk_list_add(&prop->_head, &cmd->stream_props); - return 0; -} - -void flb_sp_cmd_stream_prop_del(struct flb_sp_cmd_prop *prop) -{ - if (prop->key) { - flb_sds_destroy(prop->key); - } - if (prop->val) { - flb_sds_destroy(prop->val); - } - flb_free(prop); -} - -const char *flb_sp_cmd_stream_prop_get(struct flb_sp_cmd *cmd, const char *key) -{ - int len; - struct mk_list *head; - struct flb_sp_cmd_prop *prop; - - if (!key) { - return NULL; - } - len = strlen(key); - - mk_list_foreach(head, &cmd->stream_props) { - prop = mk_list_entry(head, struct flb_sp_cmd_prop, _head); - if (flb_sds_len(prop->key) != len) { - continue; - } - - if (strcmp(prop->key, key) == 0) { - return prop->val; - } - } - - return NULL; -} - -/* WINDOW functions */ - -int flb_sp_cmd_window(struct flb_sp_cmd *cmd, - int window_type, int size, int time_unit, - int advance_by_size, int advance_by_time_unit) -{ - cmd->window.type = window_type; - - switch (time_unit) { - case FLB_SP_TIME_SECOND: - cmd->window.size = (time_t) size; - break; - case FLB_SP_TIME_MINUTE: - cmd->window.size = (time_t) size * 60; - break; - case FLB_SP_TIME_HOUR: - cmd->window.size = (time_t) size * 3600; - break; - } - - if (window_type == FLB_SP_WINDOW_HOPPING) { - switch (advance_by_time_unit) { - case FLB_SP_TIME_SECOND: - cmd->window.advance_by = (time_t) advance_by_size; - break; - case FLB_SP_TIME_MINUTE: - cmd->window.advance_by = (time_t) advance_by_size * 60; - break; - case FLB_SP_TIME_HOUR: - cmd->window.advance_by = (time_t) advance_by_size * 3600; - break; - } - - if (cmd->window.advance_by >= cmd->window.size) { - return -1; - } - } - - return 0; -} - -/* WHERE <condition> functions */ - -struct flb_exp *flb_sp_cmd_operation(struct flb_sp_cmd *cmd, - struct flb_exp *e1, struct flb_exp *e2, - int operation) -{ - struct flb_exp_op *expression; - - expression = flb_malloc(sizeof(struct flb_exp_op)); - if (!expression) { - flb_errno(); - return NULL; - } - - expression->type = FLB_LOGICAL_OP; - expression->left = e1; - expression->right = e2; - expression->operation = operation; - mk_list_add(&expression->_head, &cmd->cond_list); - - return (struct flb_exp *) expression; -} - -struct flb_exp *flb_sp_cmd_comparison(struct flb_sp_cmd *cmd, - struct flb_exp *key, struct flb_exp *val, - int operation) -{ - struct flb_exp_op *expression; - - expression = flb_malloc(sizeof(struct flb_exp_op)); - if (!expression) { - flb_errno(); - return NULL; - } - - expression->type = FLB_LOGICAL_OP; - expression->left = (struct flb_exp *) key; - expression->right = (struct flb_exp *) val; - expression->operation = operation; - mk_list_add(&expression->_head, &cmd->cond_list); - - return (struct flb_exp *) expression; -} - -struct flb_exp *flb_sp_cmd_condition_key(struct flb_sp_cmd *cmd, - const char *identifier) -{ - int ret; - struct flb_exp_key *key; - - key = flb_calloc(1, sizeof(struct flb_exp_key)); - if (!key) { - flb_errno(); - return NULL; - } - - key->type = FLB_EXP_KEY; - key->name = flb_sds_create(identifier); - mk_list_add(&key->_head, &cmd->cond_list); - - if (mk_list_size(cmd->tmp_subkeys) > 0) { - ret = swap_tmp_subkeys(&key->subkeys, cmd); - if (ret == -1) { - flb_sds_destroy(key->name); - mk_list_del(&key->_head); - flb_free(key); - return NULL; - } - } - - return (struct flb_exp *) key; -} - -struct flb_exp *flb_sp_cmd_condition_integer(struct flb_sp_cmd *cmd, - int integer) -{ - struct flb_exp_val *val; - - val = flb_malloc(sizeof(struct flb_exp_val)); - if (!val) { - flb_errno(); - return NULL; - } - - val->type = FLB_EXP_INT; - val->val.i64 = integer; - mk_list_add(&val->_head, &cmd->cond_list); - - return (struct flb_exp *) val; -} - -struct flb_exp *flb_sp_cmd_condition_float(struct flb_sp_cmd *cmd, float fval) -{ - struct flb_exp_val *val; - - val = flb_malloc(sizeof(struct flb_exp_val)); - if (!val) { - flb_errno(); - return NULL; - } - - val->type = FLB_EXP_FLOAT; - val->val.f64 = fval; - mk_list_add(&val->_head, &cmd->cond_list); - - return (struct flb_exp *) val; -} - -struct flb_exp *flb_sp_cmd_condition_string(struct flb_sp_cmd *cmd, - const char *string) -{ - struct flb_exp_val *val; - - val = flb_malloc(sizeof(struct flb_exp_val)); - if (!val) { - flb_errno(); - return NULL; - } - - val->type = FLB_EXP_STRING; - val->val.string = flb_sds_create(string); - mk_list_add(&val->_head, &cmd->cond_list); - - return (struct flb_exp *) val; -} - -struct flb_exp *flb_sp_cmd_condition_boolean(struct flb_sp_cmd *cmd, - bool boolean) -{ - struct flb_exp_val *val; - - val = flb_malloc(sizeof(struct flb_exp_val)); - if (!val) { - flb_errno(); - return NULL; - } - - val->type = FLB_EXP_BOOL; - val->val.boolean = boolean; - mk_list_add(&val->_head, &cmd->cond_list); - - return (struct flb_exp *) val; -} - -struct flb_exp *flb_sp_cmd_condition_null(struct flb_sp_cmd *cmd) -{ - struct flb_exp_val *val; - - val = flb_malloc(sizeof(struct flb_exp_val)); - if (!val) { - flb_errno(); - return NULL; - } - - val->type = FLB_EXP_NULL; - mk_list_add(&val->_head, &cmd->cond_list); - - return (struct flb_exp *) val; -} - -struct flb_exp *flb_sp_record_function_add(struct flb_sp_cmd *cmd, - char *name, struct flb_exp *param) -{ - char *rf_name; - int i; - struct flb_exp_func *func; - - for (i = 0; i < RECORD_FUNCTIONS_SIZE; i++) - { - rf_name = record_functions[i]; - if (strncmp(rf_name, name, strlen(rf_name)) == 0) - { - func = flb_calloc(1, sizeof(struct flb_exp_func)); - if (!func) { - flb_errno(); - return NULL; - } - - func->type = FLB_EXP_FUNC; - func->name = flb_sds_create(name); - func->cb_func = record_functions_ptr[i]; - func->param = param; - - mk_list_add(&func->_head, &cmd->cond_list); - - return (struct flb_exp *) func; - } - } - - return NULL; -} - -void flb_sp_cmd_condition_add(struct flb_sp_cmd *cmd, struct flb_exp *e) - -{ - cmd->condition = e; -} - -int flb_sp_cmd_gb_key_add(struct flb_sp_cmd *cmd, const char *key) -{ - int ret; - struct flb_sp_cmd_gb_key *gb_key; - - gb_key = flb_calloc(1, sizeof(struct flb_sp_cmd_gb_key)); - if (!gb_key) { - flb_errno(); - return -1; - } - - gb_key->name = flb_sds_create(key); - if (!gb_key->name) { - flb_free(gb_key); - return -1; - } - - gb_key->id = mk_list_size(&cmd->gb_keys); - mk_list_add(&gb_key->_head, &cmd->gb_keys); - - /* Lookup for any subkeys in the temporary list */ - if (mk_list_size(cmd->tmp_subkeys) > 0) { - ret = swap_tmp_subkeys(&gb_key->subkeys, cmd); - if (ret == -1) { - flb_sds_destroy(gb_key->name); - mk_list_del(&gb_key->_head); - flb_free(gb_key); - return -1; - } - } - - return 0; -} - -void flb_sp_cmd_condition_del(struct flb_sp_cmd *cmd) -{ - struct mk_list *tmp; - struct mk_list *head; - struct flb_exp *exp; - struct flb_exp_key *key; - struct flb_exp_val *val; - struct flb_exp_func *func; - - mk_list_foreach_safe(head, tmp, &cmd->cond_list) { - exp = mk_list_entry(head, struct flb_exp, _head); - if (exp->type == FLB_EXP_KEY) { - key = (struct flb_exp_key *) exp; - flb_sds_destroy(key->name); - if (key->subkeys) { - flb_slist_destroy(key->subkeys); - flb_free(key->subkeys); - } - } - else if (exp->type == FLB_EXP_STRING) { - val = (struct flb_exp_val *) exp; - flb_sds_destroy(val->val.string); - } - else if (exp->type == FLB_EXP_FUNC) { - func = (struct flb_exp_func *) exp; - flb_sds_destroy(func->name); - } - - mk_list_del(&exp->_head); - flb_free(exp); - } -} - -void flb_sp_cmd_limit_add(struct flb_sp_cmd *cmd, int limit) -{ - cmd->limit = limit; -} - -int flb_sp_cmd_timeseries_forecast(struct flb_sp_cmd *cmd, int func, const char *key_name, int seconds) -{ - struct flb_sp_cmd_key *key; - - key = flb_sp_key_create(cmd, func, key_name, cmd->alias); - - if (!key) { - return -1; - } - - mk_list_add(&key->_head, &cmd->keys); - - key->constant = seconds; - - /* free key alias and set cmd->alias to null */ - if (cmd->alias) { - flb_free(cmd->alias); - cmd->alias = NULL; - } - - return 0; -} diff --git a/fluent-bit/src/stream_processor/parser/sql.l b/fluent-bit/src/stream_processor/parser/sql.l deleted file mode 100644 index 91e5398e1..000000000 --- a/fluent-bit/src/stream_processor/parser/sql.l +++ /dev/null @@ -1,190 +0,0 @@ -%option prefix="flb_sp_" -%option caseless -%option 8bit reentrant bison-bridge -%option warn noyywrap nodefault -%option nounput -%option noinput - - -%{ -#include <stdio.h> -#include <stdbool.h> -#include <ctype.h> -#include <fluent-bit/flb_str.h> -#include <fluent-bit/flb_log.h> -#include "sql_parser.h" -#include <fluent-bit/stream_processor/flb_sp_parser.h> - -static inline char *remove_dup_qoutes(const char *s, size_t n) -{ - char *str; - int dups; - int i, j; - - dups = 0; - for (i = 0; i < n; i++) { - if (s[i] == '\'') { - dups++; - i++; - } - } - - str = (char *) flb_malloc(n - dups + 1); - if (!str) { - return NULL; - } - - j = 0; - for (i = 0; i < n; i++, j++) { - if (s[i] == '\'') { - str[j] = '\''; - i++; - } else { - str[j] = s[i]; - } - } - str[j] = '\0'; - - return str; -} - -char* to_upper(char* token, size_t len) -{ - int i; - char* token_; - - token_ = flb_malloc(len * sizeof(char) + 1); - - for (i = 0; i < len; i++) { - token_[i] = toupper(token[i]); - } - - token_[len] = '\0'; - return token_; -} - -int func_to_code(char* name, size_t len) -{ - int code; - char* name_; - - name_ = to_upper(name, len); - code = -1; - - if (!strcmp(name_, "AVG")) { - code = FLB_SP_AVG; - } else if (!strcmp(name_, "SUM")) { - code = FLB_SP_SUM; - } else if (!strcmp(name_, "COUNT")) { - code = FLB_SP_COUNT; - } else if (!strcmp(name_, "MIN")) { - code = FLB_SP_MIN; - } else if (!strcmp(name_, "MAX")) { - code = FLB_SP_MAX; - } else if (!strcmp(name_, "TIMESERIES_FORECAST")) { - code = FLB_SP_FORECAST; - } else if (!strcmp(name_, "NOW")) { - code = FLB_SP_NOW; - } else if (!strcmp(name_, "UNIX_TIMESTAMP")) { - code = FLB_SP_UNIX_TIMESTAMP; - } else if (!strcmp(name_, "RECORD_TAG")) { - code = FLB_SP_RECORD_TAG; - } else if (!strcmp(name_, "RECORD_TIME")) { - code = FLB_SP_RECORD_TIME; - } - - flb_free(name_); - return code; -} - -%} - -%% - - /* SQL */ -CREATE return CREATE; -FLUSH return FLUSH; -STREAM return STREAM; -SNAPSHOT return SNAPSHOT; -WITH return WITH; -SELECT return SELECT; -AS return AS; -FROM return FROM; -STREAM: return FROM_STREAM; -TAG: return FROM_TAG; -WHERE return WHERE; -AND return AND; -OR return OR; -NOT return NOT; -WINDOW return WINDOW; -"GROUP BY" return GROUP_BY; -LIMIT return LIMIT; - -IS return IS; -NULL return NUL; - - /* Aggregation Functions */ -SUM {yylval->integer = func_to_code(yytext, yyleng); return SUM;} -AVG {yylval->integer = func_to_code(yytext, yyleng); return AVG;} -COUNT {yylval->integer = func_to_code(yytext, yyleng); return COUNT;} -MIN {yylval->integer = func_to_code(yytext, yyleng); return MIN;} -MAX {yylval->integer = func_to_code(yytext, yyleng); return MAX;} -TIMESERIES_FORECAST {yylval->integer = func_to_code(yytext, yyleng); return TIMESERIES_FORECAST;}; - - /* Record Functions */ -@RECORD return RECORD; -CONTAINS return CONTAINS; -TIME return TIME; - - - /* Window Types */ -TUMBLING return TUMBLING; -HOPPING return HOPPING; -"ADVANCE BY" return ADVANCE_BY; - - /* Time */ -HOUR return HOUR; -MINUTE return MINUTE; -SECOND return SECOND; - - /* Date / Time Functions */ -NOW {yylval->integer = func_to_code(yytext, yyleng); return NOW;} -UNIX_TIMESTAMP {yylval->integer = func_to_code(yytext, yyleng); return UNIX_TIMESTAMP;} - - /* Record information */ -RECORD_TAG {yylval->integer = func_to_code(yytext, yyleng); return RECORD_TAG;} -RECORD_TIME {yylval->integer = func_to_code(yytext, yyleng); return RECORD_TIME;} - -"true" { yylval->boolean = true; return BOOLTYPE; }; -"false" { yylval->boolean = false; return BOOLTYPE; }; - --?[1-9][0-9]*|0 { yylval->integer = atoi(yytext); return INTEGER; } -(-?[1-9][0-9]*|0)\.[0-9]+ { yylval->fval = atof(yytext); return FLOATING; } -\'([^']|'{2})*\' { yylval->string = remove_dup_qoutes(yytext + 1, yyleng - 2); return STRING; } - -[_A-Za-z][A-Za-z0-9_.]* { yylval->string = flb_strdup(yytext); return IDENTIFIER; } - -"*" | -"," | -"=" | -"(" | -")" | -"[" | -"]" | -"." | -";" { return yytext[0]; } - -"!=" return NEQ; -"<>" return NEQ; -"<" return LT; -"<=" return LTE; -">" return GT; -">=" return GTE; - -\' return QUOTE; -\n -[ \t]+ /* ignore whitespace */; - -. flb_error("[sp] bad input character '%s' at line %d", yytext, yylineno); - -%% diff --git a/fluent-bit/src/stream_processor/parser/sql.y b/fluent-bit/src/stream_processor/parser/sql.y deleted file mode 100644 index 866f95cc0..000000000 --- a/fluent-bit/src/stream_processor/parser/sql.y +++ /dev/null @@ -1,437 +0,0 @@ -%name-prefix="flb_sp_" // replace with %define api.prefix {flb_sp_} -%define api.pure full -%define parse.error verbose -%parse-param { struct flb_sp_cmd *cmd }; -%parse-param { const char *query }; -%lex-param { void *scanner } -%parse-param { void *scanner } - -%{ // definition section (prologue) -#include <stdio.h> -#include <stdlib.h> -#include <ctype.h> - -#include <fluent-bit/flb_mem.h> -#include <fluent-bit/flb_slist.h> -#include <fluent-bit/stream_processor/flb_sp_parser.h> - -#include "sql_parser.h" -#include "sql_lex.h" - -extern int yylex(); - -void yyerror(struct flb_sp_cmd *cmd, const char *query, void *scanner, const char *str) -{ - flb_error("[sp] %s at '%s'", str, query); -} - -%} /* EOF C code */ - -/* Bison declarations */ -/* Known Tokens (refer to sql.l) */ - -/* Keywords */ -%token IDENTIFIER QUOTE - -/* Basic keywords for statements */ -%token CREATE STREAM SNAPSHOT FLUSH WITH SELECT AS FROM FROM_STREAM FROM_TAG -%token WHERE WINDOW GROUP_BY LIMIT - -/* Null keywords */ -%token IS NUL - -/* Aggregation functions */ -%token AVG SUM COUNT MAX MIN TIMESERIES_FORECAST - -/* Record functions */ -%token RECORD CONTAINS TIME - -/* Time functions */ -%token NOW UNIX_TIMESTAMP - - /* Record functions */ -%token RECORD_TAG RECORD_TIME - -/* Value types */ -%token INTEGER FLOATING STRING BOOLTYPE - -/* Logical operation tokens */ -%token AND OR NOT NEQ LT LTE GT GTE - -/* Time tokens */ -%token HOUR MINUTE SECOND - -/* Window tokens */ -%token TUMBLING HOPPING ADVANCE_BY - -/* Union and field types */ -%union -{ - bool boolean; - int integer; - float fval; - char *string; - struct flb_sp_cmd *cmd; - struct flb_exp *expression; -} - -%type <boolean> BOOLTYPE -%type <integer> INTEGER -%type <fval> FLOATING -%type <string> IDENTIFIER -%type <string> STRING -%type <string> record_keys -%type <string> record_key -%type <string> prop_key -%type <string> prop_val -%type <expression> condition -%type <expression> comparison -%type <expression> key -%type <expression> record_func -%type <expression> value -%type <expression> null -%type <integer> time - -%type <integer> time_record_func -%type <integer> NOW UNIX_TIMESTAMP RECORD_TAG RECORD_TIME - -%type <integer> aggregate_func -%type <integer> COUNT AVG SUM MAX MIN TIMESERIES_FORECAST - - -%destructor { flb_free ($$); } IDENTIFIER - -%% /* rules section */ - -statements: create | select - -/* Parser for 'CREATE STREAM' statement */ -create: - CREATE STREAM IDENTIFIER AS select - { - flb_sp_cmd_stream_new(cmd, $3); - flb_free($3); - } - | - CREATE STREAM IDENTIFIER WITH '(' properties ')' AS select - { - flb_sp_cmd_stream_new(cmd, $3); - flb_free($3); - } - | - CREATE SNAPSHOT IDENTIFIER AS SELECT '*' FROM source limit ';' - { - flb_sp_cmd_snapshot_new(cmd, $3); - flb_free($3); - } - | - CREATE SNAPSHOT IDENTIFIER WITH '(' properties ')' AS SELECT '*' FROM source limit ';' - { - flb_sp_cmd_snapshot_new(cmd, $3); - flb_free($3); - } - | - FLUSH SNAPSHOT IDENTIFIER AS SELECT '*' FROM source where ';' - { - flb_sp_cmd_snapshot_flush_new(cmd, $3); - flb_free($3); - } - | - FLUSH SNAPSHOT IDENTIFIER WITH '(' properties ')' AS SELECT '*' FROM source where ';' - { - flb_sp_cmd_snapshot_flush_new(cmd, $3); - flb_free($3); - } - properties: property - | - properties ',' property - property: prop_key '=' prop_val - { - flb_sp_cmd_stream_prop_add(cmd, $1, $3); - flb_free($1); - flb_free($3); - } - prop_key: IDENTIFIER - prop_val: STRING - -/* Parser for 'SELECT' statement */ -select: SELECT keys FROM source window where groupby limit ';' - { - cmd->type = FLB_SP_SELECT; - } - keys: record_keys - record_keys: record_key - | - record_keys ',' record_key - record_key: '*' - { - flb_sp_cmd_key_add(cmd, -1, NULL); - } - | - IDENTIFIER key_alias - { - flb_sp_cmd_key_add(cmd, -1, $1); - flb_free($1); - } - | - IDENTIFIER record_subkey key_alias - { - flb_sp_cmd_key_add(cmd, -1, $1); - flb_free($1); - } - | - COUNT '(' '*' ')' key_alias - { - flb_sp_cmd_key_add(cmd, $1, NULL); - } - | - COUNT '(' IDENTIFIER ')' key_alias - { - flb_sp_cmd_key_add(cmd, $1, $3); - flb_free($3); - } - | - COUNT '(' IDENTIFIER record_subkey ')' key_alias - { - flb_sp_cmd_key_add(cmd, $1, $3); - flb_free($3); - } - | - aggregate_func '(' IDENTIFIER ')' key_alias - { - flb_sp_cmd_key_add(cmd, $1, $3); - flb_free($3); - } - | - aggregate_func '(' IDENTIFIER record_subkey ')' key_alias - { - flb_sp_cmd_key_add(cmd, $1, $3); - flb_free($3); - } - | - TIMESERIES_FORECAST '(' IDENTIFIER ',' INTEGER ')' key_alias - { - flb_sp_cmd_timeseries_forecast(cmd, $1, $3, $5); - flb_free($3); - } - | - time_record_func '(' ')' key_alias - { - flb_sp_cmd_key_add(cmd, $1, NULL); - } - aggregate_func: - AVG | SUM | MAX | MIN - time_record_func: - NOW | UNIX_TIMESTAMP | RECORD_TAG | RECORD_TIME - key_alias: - %empty - | - AS IDENTIFIER - { - flb_sp_cmd_alias_add(cmd, $2); - } - record_subkey: '[' STRING ']' - { - flb_slist_add(cmd->tmp_subkeys, $2); - flb_free($2); - } - | - record_subkey record_subkey - source: FROM_STREAM IDENTIFIER - { - flb_sp_cmd_source(cmd, FLB_SP_STREAM, $2); - flb_free($2); - } - | - FROM_TAG STRING - { - flb_sp_cmd_source(cmd, FLB_SP_TAG, $2); - flb_free($2); - } - window: %empty - | - WINDOW window_spec - where: %empty - | - WHERE condition - { - flb_sp_cmd_condition_add(cmd, $2); - } - groupby: %empty - | - GROUP_BY gb_keys - limit: %empty - | - LIMIT INTEGER - { - flb_sp_cmd_limit_add(cmd, $2); - } - window_spec: - TUMBLING '(' INTEGER time ')' - { - flb_sp_cmd_window(cmd, FLB_SP_WINDOW_TUMBLING, $3, $4, 0, 0); - } - | - HOPPING '(' INTEGER time ',' ADVANCE_BY INTEGER time ')' - { - flb_sp_cmd_window(cmd, FLB_SP_WINDOW_HOPPING, $3, $4, $7, $8); - } - condition: comparison - | - key - { - $$ = flb_sp_cmd_operation(cmd, $1, NULL, FLB_EXP_OR); - } - | - value - { - $$ = flb_sp_cmd_operation(cmd, NULL, $1, FLB_EXP_OR); - } - | - '(' condition ')' - { - $$ = flb_sp_cmd_operation(cmd, $2, NULL, FLB_EXP_PAR); - } - | - NOT condition - { - $$ = flb_sp_cmd_operation(cmd, $2, NULL, FLB_EXP_NOT); - } - | - condition AND condition - { - $$ = flb_sp_cmd_operation(cmd, $1, $3, FLB_EXP_AND); - } - | - condition OR condition - { - $$ = flb_sp_cmd_operation(cmd, $1, $3, FLB_EXP_OR); - } - comparison: - key IS null - { - $$ = flb_sp_cmd_comparison(cmd, $1, $3, FLB_EXP_EQ); - } - | - key IS NOT null - { - $$ = flb_sp_cmd_operation(cmd, - flb_sp_cmd_comparison(cmd, $1, $4, FLB_EXP_EQ), - NULL, FLB_EXP_NOT); - } - | - record_func - { - $$ = flb_sp_cmd_comparison(cmd, - $1, - flb_sp_cmd_condition_boolean(cmd, true), - FLB_EXP_EQ); - } - | - record_func '=' value - { - $$ = flb_sp_cmd_comparison(cmd, $1, $3, FLB_EXP_EQ); - } - | - record_func NEQ value - { - $$ = flb_sp_cmd_operation(cmd, - flb_sp_cmd_comparison(cmd, $1, $3, FLB_EXP_EQ), - NULL, FLB_EXP_NOT) - ; - } - | - record_func LT value - { - $$ = flb_sp_cmd_comparison(cmd, $1, $3, FLB_EXP_LT); - } - | - record_func LTE value - { - $$ = flb_sp_cmd_comparison(cmd, $1, $3, FLB_EXP_LTE); - } - | - record_func GT value - { - $$ = flb_sp_cmd_comparison(cmd, $1, $3, FLB_EXP_GT); - } - | - record_func GTE value - { - $$ = flb_sp_cmd_comparison(cmd, $1, $3, FLB_EXP_GTE); - } - record_func: key /* Similar to an identity function */ - | - RECORD '.' CONTAINS '(' key ')' - { - $$ = flb_sp_record_function_add(cmd, "contains", $5); - } - | - RECORD '.' TIME '(' ')' - { - $$ = flb_sp_record_function_add(cmd, "time", NULL); - } - key: IDENTIFIER - { - $$ = flb_sp_cmd_condition_key(cmd, $1); - flb_free($1); - } - | - IDENTIFIER record_subkey - { - $$ = flb_sp_cmd_condition_key(cmd, $1); - flb_free($1); - } - value: INTEGER - { - $$ = flb_sp_cmd_condition_integer(cmd, $1); - } - | - FLOATING - { - $$ = flb_sp_cmd_condition_float(cmd, $1); - } - | - STRING - { - $$ = flb_sp_cmd_condition_string(cmd, $1); - flb_free($1); - } - | - BOOLTYPE - { - $$ = flb_sp_cmd_condition_boolean(cmd, $1); - } - null: NUL - { - $$ = flb_sp_cmd_condition_null(cmd); - } - time: SECOND - { - $$ = FLB_SP_TIME_SECOND; - } - | - MINUTE - { - $$ = FLB_SP_TIME_MINUTE; - } - | - HOUR - { - $$ = FLB_SP_TIME_HOUR; - } - gb_keys: gb_key - | - gb_key ',' gb_keys - gb_key: IDENTIFIER - { - flb_sp_cmd_gb_key_add(cmd, $1); - flb_free($1); - } - | - IDENTIFIER record_subkey - { - flb_sp_cmd_gb_key_add(cmd, $1); - flb_free($1); - } - ; diff --git a/fluent-bit/src/tls/flb_tls.c b/fluent-bit/src/tls/flb_tls.c deleted file mode 100644 index 8fc711ab3..000000000 --- a/fluent-bit/src/tls/flb_tls.c +++ /dev/null @@ -1,665 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2019-2020 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <fluent-bit/flb_info.h> -#include <fluent-bit/flb_time.h> -#include <fluent-bit/flb_socket.h> - -#include "openssl.c" - -/* Config map for Upstream networking setup */ -struct flb_config_map tls_configmap[] = { - { - FLB_CONFIG_MAP_BOOL, "tls", "off", - 0, FLB_FALSE, 0, - "Enable or disable TLS/SSL support", - }, - { - FLB_CONFIG_MAP_BOOL, "tls.verify", "on", - 0, FLB_FALSE, 0, - "Force certificate validation", - }, - { - FLB_CONFIG_MAP_INT, "tls.debug", "1", - 0, FLB_FALSE, 0, - "Set TLS debug verbosity level. It accept the following " - "values: 0 (No debug), 1 (Error), 2 (State change), 3 " - "(Informational) and 4 Verbose" - }, - { - FLB_CONFIG_MAP_STR, "tls.ca_file", NULL, - 0, FLB_FALSE, 0, - "Absolute path to CA certificate file" - }, - { - FLB_CONFIG_MAP_STR, "tls.ca_path", NULL, - 0, FLB_FALSE, 0, - "Absolute path to scan for certificate files" - }, - { - FLB_CONFIG_MAP_STR, "tls.crt_file", NULL, - 0, FLB_FALSE, 0, - "Absolute path to Certificate file" - }, - { - FLB_CONFIG_MAP_STR, "tls.key_file", NULL, - 0, FLB_FALSE, 0, - "Absolute path to private Key file" - }, - { - FLB_CONFIG_MAP_STR, "tls.key_passwd", NULL, - 0, FLB_FALSE, 0, - "Optional password for tls.key_file file" - }, - - { - FLB_CONFIG_MAP_STR, "tls.vhost", NULL, - 0, FLB_FALSE, 0, - "Hostname to be used for TLS SNI extension" - }, - - /* EOF */ - {0} -}; - -struct mk_list *flb_tls_get_config_map(struct flb_config *config) -{ - struct mk_list *config_map; - - config_map = flb_config_map_create(config, tls_configmap); - return config_map; -} - - -static inline void io_tls_backup_event(struct flb_connection *connection, - struct mk_event *backup) -{ - if (connection != NULL && backup != NULL) { - memcpy(backup, &connection->event, sizeof(struct mk_event)); - } -} - -static inline void io_tls_restore_event(struct flb_connection *connection, - struct mk_event *backup) -{ - int result; - - if (connection != NULL && backup != NULL) { - if (MK_EVENT_IS_REGISTERED((&connection->event))) { - result = mk_event_del(connection->evl, &connection->event); - - assert(result == 0); - } - - if (MK_EVENT_IS_REGISTERED(backup)) { - connection->event.priority = backup->priority; - connection->event.handler = backup->handler; - - result = mk_event_add(connection->evl, - connection->fd, - backup->type, - backup->mask, - &connection->event); - - assert(result == 0); - } - } -} - - -static inline int io_tls_event_switch(struct flb_tls_session *session, - int mask) -{ - struct mk_event_loop *event_loop; - struct mk_event *event; - int ret; - - event = &session->connection->event; - event_loop = session->connection->evl; - - if ((event->mask & mask) == 0) { - ret = mk_event_add(event_loop, - event->fd, - FLB_ENGINE_EV_THREAD, - mask, event); - - event->priority = FLB_ENGINE_PRIORITY_CONNECT; - - if (ret == -1) { - flb_error("[io_tls] error changing mask to %i", mask); - - return -1; - } - } - - return 0; -} - -int flb_tls_load_system_certificates(struct flb_tls *tls) -{ - return load_system_certificates(tls->ctx); -} - -struct flb_tls *flb_tls_create(int mode, - int verify, - int debug, - const char *vhost, - const char *ca_path, - const char *ca_file, - const char *crt_file, - const char *key_file, - const char *key_passwd) -{ - void *backend; - struct flb_tls *tls; - - /* Assuming the TLS role based on the connection direction is wrong - * but it's something we'll accept for the moment. - */ - - backend = tls_context_create(verify, debug, mode, - vhost, ca_path, ca_file, - crt_file, key_file, key_passwd); - if (!backend) { - flb_error("[tls] could not create TLS backend"); - return NULL; - } - - tls = flb_calloc(1, sizeof(struct flb_tls)); - if (!tls) { - flb_errno(); - tls_context_destroy(backend); - return NULL; - } - - tls->verify = verify; - tls->debug = debug; - tls->mode = mode; - - if (vhost != NULL) { - tls->vhost = flb_strdup(vhost); - } - tls->ctx = backend; - - tls->api = &tls_openssl; - - return tls; -} - -int flb_tls_init() -{ - return tls_init(); -} - -int flb_tls_destroy(struct flb_tls *tls) -{ - if (tls->ctx) { - tls->api->context_destroy(tls->ctx); - } - - if (tls->vhost != NULL) { - flb_free(tls->vhost); - } - - flb_free(tls); - - return 0; -} - -int flb_tls_net_read(struct flb_tls_session *session, void *buf, size_t len) -{ - time_t timeout_timestamp; - time_t current_timestamp; - struct flb_tls *tls; - int ret; - - tls = session->tls; - - if (session->connection->net->io_timeout > 0) { - timeout_timestamp = time(NULL) + session->connection->net->io_timeout; - } - else { - timeout_timestamp = 0; - } - - retry_read: - ret = tls->api->net_read(session, buf, len); - - current_timestamp = time(NULL); - - if (ret == FLB_TLS_WANT_READ) { - if (timeout_timestamp > 0 && - timeout_timestamp <= current_timestamp) { - return ret; - } - - goto retry_read; - } - else if (ret == FLB_TLS_WANT_WRITE) { - goto retry_read; - } - else if (ret < 0) { - return -1; - } - else if (ret == 0) { - return -1; - } - - return ret; -} - -int flb_tls_net_read_async(struct flb_coro *co, - struct flb_tls_session *session, - void *buf, size_t len) -{ - int event_restore_needed; - struct mk_event event_backup; - struct flb_tls *tls; - int ret; - - tls = session->tls; - - event_restore_needed = FLB_FALSE; - - io_tls_backup_event(session->connection, &event_backup); - - retry_read: - ret = tls->api->net_read(session, buf, len); - - if (ret == FLB_TLS_WANT_READ) { - event_restore_needed = FLB_TRUE; - - session->connection->coroutine = co; - - io_tls_event_switch(session, MK_EVENT_READ); - flb_coro_yield(co, FLB_FALSE); - - goto retry_read; - } - else if (ret == FLB_TLS_WANT_WRITE) { - event_restore_needed = FLB_TRUE; - - session->connection->coroutine = co; - - io_tls_event_switch(session, MK_EVENT_WRITE); - flb_coro_yield(co, FLB_FALSE); - - goto retry_read; - } - else - { - /* We want this field to hold NULL at all times unless we are explicitly - * waiting to be resumed. - */ - session->connection->coroutine = NULL; - - if (ret <= 0) { - ret = -1; - } - } - - if (event_restore_needed) { - /* If we enter here it means we registered this connection - * in the event loop, in which case we need to unregister it - * and restore the original registration if there was one. - * - * We do it conditionally because in those cases in which - * send succeeds on the first try we don't touch the event - * and it wouldn't make sense to unregister and register for - * the same event. - */ - - io_tls_restore_event(session->connection, &event_backup); - } - - return ret; -} - -int flb_tls_net_write(struct flb_tls_session *session, - const void *data, size_t len, size_t *out_len) -{ - size_t total; - int ret; - struct flb_tls *tls; - - total = 0; - tls = session->tls; - -retry_write: - ret = tls->api->net_write(session, - (unsigned char *) data + total, - len - total); - - if (ret == FLB_TLS_WANT_WRITE) { - goto retry_write; - } - else if (ret == FLB_TLS_WANT_READ) { - goto retry_write; - } - else if (ret < 0) { - *out_len = total; - - return -1; - } - - /* Update counter and check if we need to continue writing */ - total += ret; - - if (total < len) { - goto retry_write; - } - - *out_len = total; - - return ret; -} - -int flb_tls_net_write_async(struct flb_coro *co, - struct flb_tls_session *session, - const void *data, size_t len, size_t *out_len) -{ - int event_restore_needed; - struct mk_event event_backup; - size_t total; - int ret; - struct flb_tls *tls; - - total = 0; - tls = session->tls; - - event_restore_needed = FLB_FALSE; - - io_tls_backup_event(session->connection, &event_backup); - -retry_write: - session->connection->coroutine = co; - - ret = tls->api->net_write(session, - (unsigned char *) data + total, - len - total); - - if (ret == FLB_TLS_WANT_WRITE) { - event_restore_needed = FLB_TRUE; - - io_tls_event_switch(session, MK_EVENT_WRITE); - - flb_coro_yield(co, FLB_FALSE); - - goto retry_write; - } - else if (ret == FLB_TLS_WANT_READ) { - event_restore_needed = FLB_TRUE; - - io_tls_event_switch(session, MK_EVENT_READ); - - flb_coro_yield(co, FLB_FALSE); - - goto retry_write; - } - else if (ret < 0) { - /* We want this field to hold NULL at all times unless we are explicitly - * waiting to be resumed. - */ - - session->connection->coroutine = NULL; - *out_len = total; - - io_tls_restore_event(session->connection, &event_backup); - - return -1; - } - - /* Update counter and check if we need to continue writing */ - total += ret; - - if (total < len) { - io_tls_event_switch(session, MK_EVENT_WRITE); - - flb_coro_yield(co, FLB_FALSE); - - goto retry_write; - } - - /* We want this field to hold NULL at all times unless we are explicitly - * waiting to be resumed. - */ - - session->connection->coroutine = NULL; - - *out_len = total; - - if (event_restore_needed) { - /* If we enter here it means we registered this connection - * in the event loop, in which case we need to unregister it - * and restore the original registration if there was one. - * - * We do it conditionally because in those cases in which - * send succeeds on the first try we don't touch the event - * and it wouldn't make sense to unregister and register for - * the same event. - */ - - io_tls_restore_event(session->connection, &event_backup); - } - - return total; -} - -int flb_tls_client_session_create(struct flb_tls *tls, - struct flb_connection *u_conn, - struct flb_coro *co) -{ - return flb_tls_session_create(tls, u_conn, co); -} - -int flb_tls_server_session_create(struct flb_tls *tls, - struct flb_connection *connection, - struct flb_coro *co) -{ - return flb_tls_session_create(tls, connection, co); -} - -/* Create a TLS session (server) */ -int flb_tls_session_create(struct flb_tls *tls, - struct flb_connection *connection, - struct flb_coro *co) -{ - int event_restore_needed; - struct mk_event event_backup; - struct flb_tls_session *session; - int result; - char *vhost; - int flag; - - session = flb_calloc(1, sizeof(struct flb_tls_session)); - - if (session == NULL) { - return -1; - } - - vhost = NULL; - - if (connection->type == FLB_UPSTREAM_CONNECTION) { - if (connection->upstream->proxied_host != NULL) { - vhost = flb_rtrim(connection->upstream->proxied_host, '.'); - } - else { - if (tls->vhost == NULL) { - vhost = flb_rtrim(connection->upstream->tcp_host, '.'); - } - } - } - - /* Create TLS session */ - session->ptr = tls->api->session_create(tls, connection->fd); - - if (session == NULL) { - flb_error("[tls] could not create TLS session for %s", - flb_connection_get_remote_address(connection)); - - return -1; - } - - session->tls = tls; - session->connection = connection; - - result = 0; - - event_restore_needed = FLB_FALSE; - - io_tls_backup_event(session->connection, &event_backup); - - retry_handshake: - result = tls->api->net_handshake(tls, vhost, session->ptr); - - if (result != 0) { - if (result != FLB_TLS_WANT_READ && result != FLB_TLS_WANT_WRITE) { - result = -1; - - goto cleanup; - } - - flag = 0; - - if (result == FLB_TLS_WANT_WRITE) { - flag = MK_EVENT_WRITE; - } - else if (result == FLB_TLS_WANT_READ) { - flag = MK_EVENT_READ; - } - - /* - * If there are no coroutine thread context (th == NULL) it means this - * TLS handshake is happening from a blocking code. Just sleep a bit - * and retry. - * - * In the other case for an async socket 'th' is NOT NULL so the code - * is under a coroutine context and it can yield. - */ - if (co == NULL) { - flb_trace("[io_tls] server handshake connection #%i in process to %s", - connection->fd, - flb_connection_get_remote_address(connection)); - - /* Connect timeout */ - if (connection->net->connect_timeout > 0 && - connection->ts_connect_timeout > 0 && - connection->ts_connect_timeout <= time(NULL)) { - flb_error("[io_tls] handshake connection #%i to %s timed out after " - "%i seconds", - connection->fd, - flb_connection_get_remote_address(connection), - connection->net->connect_timeout); - - result = -1; - - goto cleanup; - } - - flb_time_msleep(500); - - goto retry_handshake; - } - - event_restore_needed = FLB_TRUE; - - /* - * FIXME: if we need multiple reads we are invoking the same - * system call multiple times. - */ - - result = mk_event_add(connection->evl, - connection->fd, - FLB_ENGINE_EV_THREAD, - flag, - &connection->event); - - connection->event.priority = FLB_ENGINE_PRIORITY_CONNECT; - - if (result == -1) { - goto cleanup; - } - - connection->coroutine = co; - - flb_coro_yield(co, FLB_FALSE); - - /* We want this field to hold NULL at all times unless we are explicitly - * waiting to be resumed. - */ - - connection->coroutine = NULL; - - /* This check's purpose is to abort when a timeout is detected. - */ - if (connection->net_error == -1) { - goto retry_handshake; - } - else { - result = -1; - } - } - -cleanup: - if (event_restore_needed) { - /* If we enter here it means we registered this connection - * in the event loop, in which case we need to unregister it - * and restore the original registration if there was one. - * - * We do it conditionally because in those cases in which - * send succeeds on the first try we don't touch the event - * and it wouldn't make sense to unregister and register for - * the same event. - */ - - io_tls_restore_event(session->connection, &event_backup); - } - - if (result != 0) { - flb_tls_session_destroy(session); - } - else { - connection->tls_session = session; - } - - if (vhost != NULL) { - flb_free(vhost); - } - - return result; -} - -int flb_tls_session_destroy(struct flb_tls_session *session) -{ - int ret; - - session->connection->tls_session = NULL; - - if (session->ptr != NULL) { - ret = session->tls->api->session_destroy(session->ptr); - - if (ret == -1) { - return -1; - } - - flb_free(session); - } - - return 0; -} diff --git a/fluent-bit/src/tls/openssl.c b/fluent-bit/src/tls/openssl.c deleted file mode 100644 index 0d5d60bb6..000000000 --- a/fluent-bit/src/tls/openssl.c +++ /dev/null @@ -1,616 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <fluent-bit/flb_info.h> -#include <fluent-bit/flb_compat.h> -#include <fluent-bit/tls/flb_tls.h> -#include <fluent-bit/tls/flb_tls_info.h> - -#include <openssl/ssl.h> -#include <openssl/err.h> -#include <openssl/opensslv.h> - -/* - * OPENSSL_VERSION_NUMBER has the following semantics - * - * 0x010100000L M = major F = fix S = status - * MMNNFFPPS N = minor P = patch - */ -#define OPENSSL_1_1_0 0x010100000L - -/* OpenSSL library context */ -struct tls_context { - int debug_level; - SSL_CTX *ctx; - int mode; - pthread_mutex_t mutex; -}; - -struct tls_session { - SSL *ssl; - int fd; - int continuation_flag; - struct tls_context *parent; /* parent struct tls_context ref */ -}; - - -static int tls_init(void) -{ -/* - * Explicit initialization is needed for older versions of - * OpenSSL (before v1.1.0). - * - * https://wiki.openssl.org/index.php/Library_Initialization - */ -#if OPENSSL_VERSION_NUMBER < OPENSSL_1_1_0 - OPENSSL_add_all_algorithms_noconf(); - SSL_load_error_strings(); - SSL_library_init(); -#endif - return 0; -} - -static void tls_info_callback(const SSL *s, int where, int ret) -{ - int w; - int fd; - const char *str; - - fd = SSL_get_fd(s); - w = where & ~SSL_ST_MASK; - if (w & SSL_ST_CONNECT) { - str = "SSL_connect"; - } - else if (w & SSL_ST_ACCEPT) { - str = "SSL_accept"; - } - else { - str = "undefined"; - } - - if (where & SSL_CB_LOOP) { - flb_debug("[tls] connection #%i %s: %s", - fd, str, SSL_state_string_long(s)); - } - else if (where & SSL_CB_ALERT) { - str = (where & SSL_CB_READ) ? "read" : "write"; - flb_debug("[tls] connection #%i SSL3 alert %s:%s:%s", - fd, str, - SSL_alert_type_string_long(ret), - SSL_alert_desc_string_long(ret)); - } - else if (where & SSL_CB_EXIT) { - if (ret == 0) { - flb_error("[tls] connection #%i %s: failed in %s", - fd, str, SSL_state_string_long(s)); - } - else if (ret < 0) { - ret = SSL_get_error(s, ret); - if (ret == SSL_ERROR_WANT_WRITE) { - flb_debug("[tls] connection #%i WANT_WRITE", fd); - } - else if (ret == SSL_ERROR_WANT_READ) { - flb_debug("[tls] connection #%i WANT_READ", fd); - } - else { - flb_error("[tls] connection #%i %s: error in %s", - fd, str, SSL_state_string_long(s)); - } - } - } -} - -static void tls_context_destroy(void *ctx_backend) -{ - struct tls_context *ctx = ctx_backend; - - pthread_mutex_lock(&ctx->mutex); - SSL_CTX_free(ctx->ctx); - pthread_mutex_unlock(&ctx->mutex); - - flb_free(ctx); -} - -#ifdef _MSC_VER -static int windows_load_system_certificates(struct tls_context *ctx) -{ - int ret; - HANDLE win_store; - PCCERT_CONTEXT win_cert = NULL; - const unsigned char *win_cert_data; - X509_STORE *ossl_store = SSL_CTX_get_cert_store(ctx->ctx); - X509 *ossl_cert; - - win_store = CertOpenSystemStoreA(0, "Root"); - if (win_store == NULL) { - flb_error("[tls] Cannot open cert store: %i", GetLastError()); - return -1; - } - - while (win_cert = CertEnumCertificatesInStore(win_store, win_cert)) { - if (win_cert->dwCertEncodingType & X509_ASN_ENCODING) { - /* - * Decode the certificate into X509 struct. - * - * The additional pointer variable is necessary per OpenSSL docs because the - * pointer is incremented by d2i_X509. - */ - win_cert_data = win_cert->pbCertEncoded; - ossl_cert = d2i_X509(NULL, &win_cert_data, win_cert->cbCertEncoded); - if (!ossl_cert) { - flb_debug("[tls] Cannot parse a certificate. skipping..."); - continue; - } - - /* Add X509 struct to the openssl cert store */ - ret = X509_STORE_add_cert(ossl_store, ossl_cert); - if (!ret) { - flb_warn("[tls] Failed to add a certificate to the store: %lu: %s", - ERR_get_error(), ERR_error_string(ERR_get_error(), NULL)); - } - X509_free(ossl_cert); - } - } - - if (!CertCloseStore(win_store, 0)) { - flb_error("[tls] Cannot close cert store: %i", GetLastError()); - return -1; - } - return 0; -} -#endif - -static int load_system_certificates(struct tls_context *ctx) -{ - int ret; - const char *ca_file = FLB_DEFAULT_SEARCH_CA_BUNDLE; - - /* For Windows use specific API to read the certs store */ -#ifdef _MSC_VER - return windows_load_system_certificates(ctx); -#endif - if (access(ca_file, R_OK) != 0) { - ca_file = NULL; - } - - ret = SSL_CTX_load_verify_locations(ctx->ctx, ca_file, FLB_DEFAULT_CA_DIR); - - if (ret != 1) { - ERR_print_errors_fp(stderr); - } - return 0; -} - -static void *tls_context_create(int verify, - int debug, - int mode, - const char *vhost, - const char *ca_path, - const char *ca_file, - const char *crt_file, - const char *key_file, - const char *key_passwd) -{ - int ret; - SSL_CTX *ssl_ctx; - struct tls_context *ctx; - char err_buf[256]; - - /* - * Init library ? based in the documentation on OpenSSL >= 1.1.0 is not longer - * necessary since the library will initialize it self: - * - * https://wiki.openssl.org/index.php/Library_Initialization - */ - - /* Create OpenSSL context */ -#if OPENSSL_VERSION_NUMBER < OPENSSL_1_1_0 - /* - * SSLv23_method() is actually an equivalent of TLS_client_method() - * in OpenSSL v1.0.x. - * - * https://www.openssl.org/docs/man1.0.2/man3/SSLv23_method.html - */ - if (mode == FLB_TLS_SERVER_MODE) { - ssl_ctx = SSL_CTX_new(SSLv23_server_method()); - } - else { - ssl_ctx = SSL_CTX_new(SSLv23_client_method()); - } - -#else - if (mode == FLB_TLS_SERVER_MODE) { - ssl_ctx = SSL_CTX_new(TLS_server_method()); - } - else { - ssl_ctx = SSL_CTX_new(TLS_client_method()); - } -#endif - if (!ssl_ctx) { - flb_error("[openssl] could not create context"); - return NULL; - } - - ctx = flb_calloc(1, sizeof(struct tls_context)); - if (!ctx) { - flb_errno(); - return NULL; - } - ctx->ctx = ssl_ctx; - ctx->mode = mode; - ctx->debug_level = debug; - pthread_mutex_init(&ctx->mutex, NULL); - - /* Verify peer: by default OpenSSL always verify peer */ - if (verify == FLB_FALSE) { - SSL_CTX_set_verify(ssl_ctx, SSL_VERIFY_NONE, NULL); - } - else { - SSL_CTX_set_verify(ssl_ctx, SSL_VERIFY_PEER, NULL); - } - - /* ca_path | ca_file */ - if (ca_path) { - ret = SSL_CTX_load_verify_locations(ctx->ctx, NULL, ca_path); - if (ret != 1) { - ERR_error_string_n(ERR_get_error(), err_buf, sizeof(err_buf)-1); - flb_error("[tls] ca_path '%s' %lu: %s", - ca_path, ERR_get_error(), err_buf); - goto error; - } - } - else if (ca_file) { - ret = SSL_CTX_load_verify_locations(ctx->ctx, ca_file, NULL); - if (ret != 1) { - ERR_error_string_n(ERR_get_error(), err_buf, sizeof(err_buf)-1); - flb_error("[tls] ca_file '%s' %lu: %s", - ca_file, ERR_get_error(), err_buf); - goto error; - } - } - else { - load_system_certificates(ctx); - } - - /* crt_file */ - if (crt_file) { - ret = SSL_CTX_use_certificate_chain_file(ssl_ctx, crt_file); - if (ret != 1) { - ERR_error_string_n(ERR_get_error(), err_buf, sizeof(err_buf)-1); - flb_error("[tls] crt_file '%s' %lu: %s", - crt_file, ERR_get_error(), err_buf); - goto error; - } - } - - /* key_file */ - if (key_file) { - if (key_passwd) { - SSL_CTX_set_default_passwd_cb_userdata(ssl_ctx, - (void *) key_passwd); - } - ret = SSL_CTX_use_PrivateKey_file(ssl_ctx, key_file, - SSL_FILETYPE_PEM); - if (ret != 1) { - ERR_error_string_n(ERR_get_error(), err_buf, sizeof(err_buf)-1); - flb_error("[tls] key_file '%s' %lu: %s", - crt_file, ERR_get_error(), err_buf); - } - - /* Make sure the key and certificate file match */ - if (SSL_CTX_check_private_key(ssl_ctx) != 1) { - flb_error("[tls] private_key '%s' and password don't match", - key_file); - goto error; - } - } - - return ctx; - - error: - tls_context_destroy(ctx); - return NULL; -} - -static void *tls_session_create(struct flb_tls *tls, - int fd) -{ - struct tls_session *session; - struct tls_context *ctx = tls->ctx; - SSL *ssl; - - session = flb_calloc(1, sizeof(struct tls_session)); - if (!session) { - flb_errno(); - return NULL; - } - session->parent = ctx; - - pthread_mutex_lock(&ctx->mutex); - ssl = SSL_new(ctx->ctx); - - if (!ssl) { - flb_error("[openssl] could create new SSL context"); - flb_free(session); - pthread_mutex_unlock(&ctx->mutex); - return NULL; - } - - session->continuation_flag = FLB_FALSE; - session->ssl = ssl; - session->fd = fd; - SSL_set_fd(ssl, fd); - - /* - * TLS Debug Levels: - * - * 0: No debug, - * 1: Error - * 2: State change - * 3: Informational - * 4: Verbose - */ - if (tls->debug == 1) { - SSL_set_info_callback(session->ssl, tls_info_callback); - } - pthread_mutex_unlock(&ctx->mutex); - return session; -} - -static int tls_session_destroy(void *session) -{ - struct tls_session *ptr = session; - struct tls_context *ctx; - - if (!ptr) { - return 0; - } - ctx = ptr->parent; - - pthread_mutex_lock(&ctx->mutex); - - if (flb_socket_error(ptr->fd) == 0) { - SSL_shutdown(ptr->ssl); - } - SSL_free(ptr->ssl); - flb_free(ptr); - - pthread_mutex_unlock(&ctx->mutex); - - return 0; -} - -static int tls_net_read(struct flb_tls_session *session, - void *buf, size_t len) -{ - int ret; - char err_buf[256]; - struct tls_context *ctx; - struct tls_session *backend_session; - - if (session->ptr == NULL) { - flb_error("[tls] error: uninitialized backend session"); - - return -1; - } - - backend_session = (struct tls_session *) session->ptr; - - ctx = backend_session->parent; - - pthread_mutex_lock(&ctx->mutex); - - ERR_clear_error(); - - ret = SSL_read(backend_session->ssl, buf, len); - - if (ret <= 0) { - ret = SSL_get_error(backend_session->ssl, ret); - - if (ret == SSL_ERROR_WANT_READ) { - ret = FLB_TLS_WANT_READ; - } - else if (ret == SSL_ERROR_WANT_WRITE) { - ret = FLB_TLS_WANT_WRITE; - } - else if (ret == SSL_ERROR_SYSCALL) { - flb_errno(); - ERR_error_string_n(ret, err_buf, sizeof(err_buf)-1); - flb_error("[tls] syscall error: %s", err_buf); - - /* According to the documentation these are non-recoverable - * errors so we don't need to screen them before saving them - * to the net_error field. - */ - - session->connection->net_error = errno; - - ret = -1; - } - else if (ret < 0) { - ERR_error_string_n(ret, err_buf, sizeof(err_buf)-1); - flb_error("[tls] error: %s", err_buf); - } - else { - ret = -1; - } - } - - pthread_mutex_unlock(&ctx->mutex); - return ret; -} - -static int tls_net_write(struct flb_tls_session *session, - const void *data, size_t len) -{ - int ret; - char err_buf[256]; - size_t total = 0; - struct tls_context *ctx; - struct tls_session *backend_session; - - if (session->ptr == NULL) { - flb_error("[tls] error: uninitialized backend session"); - - return -1; - } - - backend_session = (struct tls_session *) session->ptr; - ctx = backend_session->parent; - - pthread_mutex_lock(&ctx->mutex); - - ERR_clear_error(); - - ret = SSL_write(backend_session->ssl, - (unsigned char *) data + total, - len - total); - - if (ret <= 0) { - ret = SSL_get_error(backend_session->ssl, ret); - - if (ret == SSL_ERROR_WANT_WRITE) { - ret = FLB_TLS_WANT_WRITE; - } - else if (ret == SSL_ERROR_WANT_READ) { - ret = FLB_TLS_WANT_READ; - } - else if (ret == SSL_ERROR_SYSCALL) { - flb_errno(); - ERR_error_string_n(ret, err_buf, sizeof(err_buf)-1); - flb_error("[tls] syscall error: %s", err_buf); - - /* According to the documentation these are non-recoverable - * errors so we don't need to screen them before saving them - * to the net_error field. - */ - - session->connection->net_error = errno; - - ret = -1; - } - else { - ERR_error_string_n(ret, err_buf, sizeof(err_buf)-1); - flb_error("[tls] error: %s", err_buf); - - ret = -1; - } - } - - pthread_mutex_unlock(&ctx->mutex); - - /* Update counter and check if we need to continue writing */ - return ret; -} - -static int tls_net_handshake(struct flb_tls *tls, - char *vhost, - void *ptr_session) -{ - int ret = 0; - char err_buf[256]; - struct tls_session *session = ptr_session; - struct tls_context *ctx; - - ctx = session->parent; - pthread_mutex_lock(&ctx->mutex); - - if (!session->continuation_flag) { - if (tls->mode == FLB_TLS_CLIENT_MODE) { - SSL_set_connect_state(session->ssl); - } - else if (tls->mode == FLB_TLS_SERVER_MODE) { - SSL_set_accept_state(session->ssl); - } - else { - flb_error("[tls] error: invalid tls mode : %d", tls->mode); - pthread_mutex_unlock(&ctx->mutex); - return -1; - } - - if (vhost != NULL) { - SSL_set_tlsext_host_name(session->ssl, vhost); - } - else if (tls->vhost) { - SSL_set_tlsext_host_name(session->ssl, tls->vhost); - } - } - - ERR_clear_error(); - - if (tls->mode == FLB_TLS_CLIENT_MODE) { - ret = SSL_connect(session->ssl); - } - else if (tls->mode == FLB_TLS_SERVER_MODE) { - ret = SSL_accept(session->ssl); - } - - if (ret != 1) { - ret = SSL_get_error(session->ssl, ret); - if (ret != SSL_ERROR_WANT_READ && - ret != SSL_ERROR_WANT_WRITE) { - ret = SSL_get_error(session->ssl, ret); - // The SSL_ERROR_SYSCALL with errno value of 0 indicates unexpected - // EOF from the peer. This is fixed in OpenSSL 3.0. - if (ret == 0) { - flb_error("[tls] error: unexpected EOF"); - } else { - ERR_error_string_n(ret, err_buf, sizeof(err_buf)-1); - flb_error("[tls] error: %s", err_buf); - } - - pthread_mutex_unlock(&ctx->mutex); - - return -1; - } - - if (ret == SSL_ERROR_WANT_WRITE) { - pthread_mutex_unlock(&ctx->mutex); - - session->continuation_flag = FLB_TRUE; - - return FLB_TLS_WANT_WRITE; - } - else if (ret == SSL_ERROR_WANT_READ) { - pthread_mutex_unlock(&ctx->mutex); - - session->continuation_flag = FLB_TRUE; - - return FLB_TLS_WANT_READ; - } - } - - session->continuation_flag = FLB_FALSE; - - pthread_mutex_unlock(&ctx->mutex); - flb_trace("[tls] connection and handshake OK"); - return 0; -} - -/* OpenSSL backend registration */ -static struct flb_tls_backend tls_openssl = { - .name = "openssl", - .context_create = tls_context_create, - .context_destroy = tls_context_destroy, - .session_create = tls_session_create, - .session_destroy = tls_session_destroy, - .net_read = tls_net_read, - .net_write = tls_net_write, - .net_handshake = tls_net_handshake, -}; diff --git a/fluent-bit/src/wamrc/CMakeLists.txt b/fluent-bit/src/wamrc/CMakeLists.txt deleted file mode 100644 index f8de6449f..000000000 --- a/fluent-bit/src/wamrc/CMakeLists.txt +++ /dev/null @@ -1,210 +0,0 @@ -string (TOLOWER ${CMAKE_HOST_SYSTEM_NAME} WAMR_BUILD_PLATFORM) -enable_language (CXX) - -if (FLB_SYSTEM_WINDOWS) - enable_language(ASM_MASM) -endif() -if (APPLE) - add_definitions(-DBH_PLATFORM_DARWIN) -endif () - -set (CMAKE_CXX_STANDARD 14) - -if (FLB_SYSTEM_WINDOWS) - add_definitions(-DCOMPILING_WASM_RUNTIME_API=1) -endif () - -# Reset default linker flags -set (CMAKE_SHARED_LIBRARY_LINK_C_FLAGS "") -set (CMAKE_SHARED_LIBRARY_LINK_CXX_FLAGS "") - -# WAMR features switch - -# Set WAMR_BUILD_TARGET, currently values supported: -# "X86_64", "AMD_64", "X86_32", "AARCH64[sub]", "ARM[sub]", "THUMB[sub]", -# "MIPS", "XTENSA", "RISCV64[sub]", "RISCV32[sub]" -if (NOT DEFINED WAMR_BUILD_TARGET) - if (CMAKE_SYSTEM_PROCESSOR MATCHES "^(arm64|aarch64)") - set (WAMR_BUILD_TARGET "AARCH64") - add_definitions(-DBUILD_TARGET="${WAMR_BUILD_TARGET}") - # For raspbian/buster: armv6l-unknown-linux-gnueabihf - elseif (CMAKE_SYSTEM_PROCESSOR MATCHES "^(armv6.*|armv7.*)") - set (WAMR_BUILD_TARGET "ARM") - elseif (CMAKE_SYSTEM_PROCESSOR STREQUAL "riscv64") - set (WAMR_BUILD_TARGET "RISCV64") - elseif (CMAKE_SIZEOF_VOID_P EQUAL 8) - # Build as X86_64 by default in 64-bit platform - set (WAMR_BUILD_TARGET "X86_64") - elseif (CMAKE_SIZEOF_VOID_P EQUAL 4) - # Build as X86_32 by default in 32-bit platform - set (WAMR_BUILD_TARGET "X86_32") - else () - message(SEND_ERROR "Unsupported build target platform!") - endif () -endif () - -if (NOT CMAKE_BUILD_TYPE) - set (CMAKE_BUILD_TYPE Release) -endif () - -add_definitions(-DWASM_ENABLE_INTERP=1) -add_definitions(-DWASM_ENABLE_WAMR_COMPILER=1) -add_definitions(-DWASM_ENABLE_BULK_MEMORY=1) -add_definitions(-DWASM_DISABLE_HW_BOUND_CHECK=1) -add_definitions(-DWASM_ENABLE_SHARED_MEMORY=1) -add_definitions(-DWASM_ENABLE_THREAD_MGR=1) -add_definitions(-DWASM_ENABLE_TAIL_CALL=1) -add_definitions(-DWASM_ENABLE_SIMD=1) -add_definitions(-DWASM_ENABLE_REF_TYPES=1) -add_definitions(-DWASM_ENABLE_CUSTOM_NAME_SECTION=1) -add_definitions(-DWASM_ENABLE_DUMP_CALL_STACK=1) -add_definitions(-DWASM_ENABLE_PERF_PROFILING=1) -add_definitions(-DWASM_ENABLE_LOAD_CUSTOM_SECTION=1) -if (WAMR_BUILD_LLVM_LEGACY_PM EQUAL 1) - add_definitions(-DWASM_ENABLE_LLVM_LEGACY_PM=1) -endif() - -if (DEFINED WAMR_BUILD_AOT_FUNC_PREFIX) - add_definitions(-DAOT_FUNC_PREFIX="${WAMR_BUILD_AOT_FUNC_PREFIX}") -endif () - -if (NOT MSVC) - # linker flags - set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -pie -fPIE") - if (NOT (CMAKE_C_COMPILER MATCHES ".*clang.*" OR CMAKE_C_COMPILER_ID MATCHES ".*Clang")) - set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--gc-sections") - endif () - set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall -Wformat -Wformat-security") - if (WAMR_BUILD_TARGET MATCHES "X86_.*" OR WAMR_BUILD_TARGET STREQUAL "AMD_64") - if (NOT (CMAKE_C_COMPILER MATCHES ".*clang.*" OR CMAKE_C_COMPILER_ID MATCHES ".*Clang")) - check_c_compiler_flag(-mindirect-branch-register FLB_WAMRC_INDIRECT_BRANCH_REGISTER_SUPPORTED) - if (FLB_WAMRC_INDIRECT_BRANCH_REGISTER_SUPPORTE) - set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mindirect-branch-register") - endif () - endif () - endif () -endif () - -# Searching homebrewed LLVM automatically on macOS. -if(FLB_SYSTEM_MACOS) - execute_process( - COMMAND brew --prefix llvm - RESULT_VARIABLE HOMEBREW_LLVM - OUTPUT_VARIABLE HOMEBREW_LLVM_PREFIX - OUTPUT_STRIP_TRAILING_WHITESPACE - ) - if (HOMEBREW_LLVM EQUAL 0 AND EXISTS "${HOMEBREW_LLVM_PREFIX}") - message(STATUS "Using llvm keg installed by Homebrew at ${HOMEBREW_LLVM_PREFIX}") - set(ENV{LLVM_DIR} "${HOMEBREW_LLVM_PREFIX}") - endif() -endif() - -# Enable LLVM -set (WAMR_BUILD_WITH_SYSTEM_LLVM 1) -if (NOT WAMR_BUILD_WITH_SYSTEM_LLVM) - set (LLVM_SRC_ROOT "${PROJECT_SOURCE_DIR}/../core/deps/llvm") - if (WAMR_BUILD_PLATFORM STREQUAL "windows") - if (NOT EXISTS "${LLVM_SRC_ROOT}/win32build") - message (FATAL_ERROR "Cannot find LLVM dir: ${LLVM_SRC_ROOT}/win32build") - endif () - set (CMAKE_PREFIX_PATH "${LLVM_SRC_ROOT}/win32build;${CMAKE_PREFIX_PATH}") - else() - if (NOT EXISTS "${LLVM_SRC_ROOT}/build") - message (FATAL_ERROR "Cannot find LLVM dir: ${LLVM_SRC_ROOT}/build") - endif () - set (CMAKE_PREFIX_PATH "${LLVM_SRC_ROOT}/build;${CMAKE_PREFIX_PATH}") - endif () -endif () - -if(LLVM_ENABLE_CURL STREQUAL FORCE_ON) - find_package(CURL REQUIRED) -else() - find_package(CURL) -endif() - -find_package(LLVM CONFIG) -if (LLVM_FOUND) - if (LLVM_PACKAGE_VERSION VERSION_LESS 13.0) - message(STATUS "Outdated LLVM ${LLVM_PACKAGE_VERSION} is found. WAMRC won't be built.") - set(LLVM_FOUND 0) - endif() -else() - message(STATUS "LLVM is not found. WAMRC won't be built.") -endif() -if (LLVM_FOUND) - include_directories(${LLVM_INCLUDE_DIRS}) - add_definitions(${LLVM_DEFINITIONS}) - - message(STATUS "Found LLVM ${LLVM_PACKAGE_VERSION}") - message(STATUS "Using LLVMConfig.cmake in: ${LLVM_DIR}") - - set (WAMR_ROOT_DIR ../../${FLB_PATH_LIB_WASM_MICRO_RUNTIME}) - set (SHARED_DIR ${WAMR_ROOT_DIR}/core/shared) - set (IWASM_DIR ${WAMR_ROOT_DIR}/core/iwasm) - set (APP_FRAMEWORK_DIR ${WAMER_ROOT_DIR}/core/app-framework) - - include_directories (${SHARED_DIR}/include - ${IWASM_DIR}/include) - - enable_language (ASM) - - include (${SHARED_DIR}/platform/${WAMR_BUILD_PLATFORM}/shared_platform.cmake) - include (${SHARED_DIR}/mem-alloc/mem_alloc.cmake) - include (${SHARED_DIR}/utils/shared_utils.cmake) - include (${SHARED_DIR}/utils/uncommon/shared_uncommon.cmake) - include (${IWASM_DIR}/libraries/thread-mgr/thread_mgr.cmake) - include (${IWASM_DIR}/libraries/libc-builtin/libc_builtin.cmake) - if (NOT MINGW) - if (NOT MSVC) - include (${IWASM_DIR}/libraries/libc-wasi/libc_wasi.cmake) - else() - include (${IWASM_DIR}/libraries/libc-uvwasi/libc_uvwasi.cmake) - endif() - endif() - include (${IWASM_DIR}/libraries/lib-pthread/lib_pthread.cmake) - include (${IWASM_DIR}/common/iwasm_common.cmake) - include (${IWASM_DIR}/interpreter/iwasm_interp.cmake) - include (${IWASM_DIR}/aot/iwasm_aot.cmake) - include (${IWASM_DIR}/compilation/iwasm_compl.cmake) - - add_library (vmlib-wamrc-static STATIC - ${PLATFORM_SHARED_SOURCE} - ${MEM_ALLOC_SHARED_SOURCE} - ${UTILS_SHARED_SOURCE} - ${UNCOMMON_SHARED_SOURCE} - ${THREAD_MGR_SOURCE} - ${LIBC_BUILTIN_SOURCE} - ${LIBC_WASI_SOURCE} - ${LIB_PTHREAD_SOURCE} - ${IWASM_COMMON_SOURCE} - ${IWASM_INTERP_SOURCE} - ${IWASM_AOT_SOURCE}) - add_library (aotclib-static STATIC ${IWASM_COMPL_SOURCE}) - add_executable (flb-wamrc-bin ${WAMR_ROOT_DIR}/wamr-compiler/main.c) - if (NOT MSVC) - target_link_libraries (flb-wamrc-bin aotclib-static vmlib-wamrc-static LLVMDemangle ${LLVM_AVAILABLE_LIBS} ${lib_ubsan} - -lm -lpthread ${lib_lldb} ${UV_A_LIBS}) - if (MINGW) - target_link_libraries (flb-wamrc-bin -lssp -lWs2_32) - else() - target_link_libraries (flb-wamrc-bin -ldl) - endif() - else() - target_link_libraries (flb-wamrc-bin aotclib-static vmlib-wamrc-static ${lib_lldb} ${LLVM_AVAILABLE_LIBS} ${lib_ubsan} - ${UV_A_LIBS}) - endif() - - set_target_properties(flb-wamrc-bin - PROPERTIES - OUTPUT_NAME flb-wamrc - ENABLE_EXPORTS ON) - install(TARGETS flb-wamrc-bin RUNTIME DESTINATION ${FLB_INSTALL_BINDIR} COMPONENT binary) - - # Include PDB file (if available) - if (MSVC) - target_link_options(flb-wamrc-bin - PUBLIC /pdb:$<TARGET_PDB_FILE:flb-wamrc-bin>) - install(FILES $<TARGET_PDB_FILE:flb-wamrc-bin> - DESTINATION "${FLB_INSTALL_BINDIR}") - endif() -endif ()
\ No newline at end of file diff --git a/fluent-bit/src/wasm/CMakeLists.txt b/fluent-bit/src/wasm/CMakeLists.txt deleted file mode 100644 index b345c4b45..000000000 --- a/fluent-bit/src/wasm/CMakeLists.txt +++ /dev/null @@ -1,111 +0,0 @@ -string (TOLOWER ${CMAKE_HOST_SYSTEM_NAME} WAMR_BUILD_PLATFORM) -if (FLB_SYSTEM_WINDOWS) - enable_language(ASM_MASM) -endif() -if (APPLE) - add_definitions(-DBH_PLATFORM_DARWIN) -endif () - -set (CMAKE_C_STANDARD 99) - -if (FLB_SYSTEM_WINDOWS) - add_definitions(-DCOMPILING_WASM_RUNTIME_API=1) -endif () - -# WAMR features switch - -# Set WAMR_BUILD_TARGET, currently values supported: -# "X86_64", "AMD_64", "X86_32", "AARCH64[sub]", "ARM[sub]", "THUMB[sub]", -# "MIPS", "XTENSA", "RISCV64[sub]", "RISCV32[sub]" -if (NOT DEFINED WAMR_BUILD_TARGET) - if (CMAKE_SYSTEM_PROCESSOR MATCHES "^(arm64|aarch64)") - set (WAMR_BUILD_TARGET "AARCH64") - if (FLB_SYSTEM_MACOS) - message(STATUS "macOS arm64 platform is poor support for AOT loading. Now disabling for it.") - set (WAMR_DISABLE_AOT_LOADING 1) - FLB_DEFINITION(FLB_WAMR_DISABLE_AOT_LOADING) - endif () - # For raspbian/buster: armv6l-unknown-linux-gnueabihf - elseif (CMAKE_SYSTEM_PROCESSOR MATCHES "^(armv6.*|armv7.*)") - set (WAMR_BUILD_TARGET "ARM") - elseif (CMAKE_SYSTEM_PROCESSOR STREQUAL "riscv64") - set (WAMR_BUILD_TARGET "RISCV64") - elseif (CMAKE_SIZEOF_VOID_P EQUAL 8) - # Build as X86_64 by default in 64-bit platform - set (WAMR_BUILD_TARGET "X86_64") - elseif (CMAKE_SIZEOF_VOID_P EQUAL 4) - # Build as X86_32 by default in 32-bit platform - set (WAMR_BUILD_TARGET "X86_32") - else () - message(SEND_ERROR "Unsupported build target platform!") - endif () -endif () - -if (NOT CMAKE_BUILD_TYPE) - set (CMAKE_BUILD_TYPE Release) -endif () - -set (WAMR_BUILD_MINI_LOADER 0) -set (WAMR_BUILD_INTERP 1) -set (WAMR_BUILD_FAST_INTERP 1) -if (NOT DEFINED WAMR_DISABLE_AOT_LOADING) - set (WAMR_BUILD_AOT 1) -endif () -set (WAMR_BUILD_JIT 0) -set (WAMR_BUILD_LIBC_BUILTIN 1) -if (MSVC) - # Currently, LIBC_UVWASI build is disabled. - # FIXME: Need to investigate how to build libuv and uvwasi without fetching repos. - set (WAMR_BUILD_LIBC_UVWASI 0) -else () - set (WAMR_BUILD_LIBC_WASI 1) -endif () -if (NOT MSVC) - set (WAMR_BUILD_LIB_PTHREAD 1) -endif () -set (WAMR_BUILD_REF_TYPES 0) - -if (NOT MSVC) - # linker flags - set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -pie -fPIE") - if (NOT (CMAKE_C_COMPILER MATCHES ".*clang.*" OR CMAKE_C_COMPILER_ID MATCHES ".*Clang")) - set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--gc-sections") - endif () - set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall -Wformat -Wformat-security") - if (WAMR_BUILD_TARGET MATCHES "X86_.*" OR WAMR_BUILD_TARGET STREQUAL "AMD_64") - if (NOT (CMAKE_C_COMPILER MATCHES ".*clang.*" OR CMAKE_C_COMPILER_ID MATCHES ".*Clang")) - check_c_compiler_flag(-mindirect-branch-register FLB_WASM_INDIRECT_BRANCH_REGISTER_SUPPORTED) - if (FLB_WASM_INDIRECT_BRANCH_REGISTER_SUPPORTED) - set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mindirect-branch-register") - endif () - endif () - endif () -endif () - -set (WAMR_BUILD_SIMD 0) -set (WAMR_ROOT_DIR ../../${FLB_PATH_LIB_WASM_MICRO_RUNTIME}) - -# build out vmlib-static -include (${WAMR_ROOT_DIR}/build-scripts/runtime_lib.cmake) -add_library(vmlib-static STATIC ${WAMR_RUNTIME_LIB_SOURCE}) - -# Application related -include (${SHARED_DIR}/utils/uncommon/shared_uncommon.cmake) - -set(src - flb_wasm.c - ${UNCOMMON_SHARED_SOURCE}) # link wasm-micro-runtime's uncommon object symbols (for bh_read_file_to_buffer) - -add_library(flb-wasm-static STATIC ${src}) - -if (FLB_JEMALLOC AND ${CMAKE_SYSTEM_NAME} MATCHES "Linux") - set(${JEMALLOC_LIBS} libjemalloc) - add_dependencies(flb-wasm-static libjemalloc) - include_directories("${CMAKE_BINARY_DIR}/include/") -endif () - -if (WAMR_BUILD_LIBC_UVWASI) - target_link_libraries(flb-wasm-static vmlib-static ${UV_A_LIBS}) -else () - target_link_libraries(flb-wasm-static vmlib-static ${JEMALLOC_LIBS}) -endif() diff --git a/fluent-bit/src/wasm/flb_wasm.c b/fluent-bit/src/wasm/flb_wasm.c deleted file mode 100644 index 2f75c9bba..000000000 --- a/fluent-bit/src/wasm/flb_wasm.c +++ /dev/null @@ -1,316 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* Don't use and expose bh_ prefixed headers in flb_wasm.h. - Their definitions are tightly coupled in wasm-micro-runtime library. */ -#include "bh_read_file.h" -#include "bh_getopt.h" - -#include <fluent-bit/flb_info.h> -#include <fluent-bit/flb_mem.h> -#include <fluent-bit/flb_log.h> -#include <fluent-bit/flb_slist.h> -#include <fluent-bit/wasm/flb_wasm.h> - -#ifdef FLB_SYSTEM_WINDOWS -#define STDIN_FILENO (_fileno( stdin )) -#define STDOUT_FILENO (_fileno( stdout )) -#define STDERR_FILENO (_fileno( stderr )) -#else -#include <unistd.h> -#endif - -void flb_wasm_init(struct flb_config *config) -{ - mk_list_init(&config->wasm_list); -} - -static int flb_wasm_load_wasm_binary(const char *wasm_path, int8_t **out_buf, uint32_t *out_size) -{ - char *buffer; - uint32_t buf_size; - buffer = bh_read_file_to_buffer(wasm_path, &buf_size); - if (!buffer) { - flb_error("Open wasm file [%s] failed.", wasm_path); - goto error; - } - -#if defined(FLB_WAMR_DISABLE_AOT_LOADING) - if ((get_package_type((const uint8_t *)buffer, buf_size) != Wasm_Module_Bytecode)) { - flb_error("WASM bytecode is expected but other file format"); - goto error; - } -#else - if ((get_package_type((const uint8_t *)buffer, buf_size) != Wasm_Module_Bytecode) && - (get_package_type((const uint8_t *)buffer, buf_size) != Wasm_Module_AoT)) { - flb_error("WASM bytecode or AOT object is expected but other file format"); - goto error; - } -#endif - - *out_buf = buffer; - *out_size = buf_size; - - return buffer != NULL; - -error: - if (buffer != NULL) { - BH_FREE(buffer); - } - - return FLB_FALSE; -} - -struct flb_wasm *flb_wasm_instantiate(struct flb_config *config, const char *wasm_path, - struct mk_list *accessible_dir_list, - int stdinfd, int stdoutfd, int stderrfd) -{ - struct flb_wasm *fw; - uint32_t buf_size, stack_size = 8 * 1024, heap_size = 8 * 1024; - int8_t *buffer = NULL; - char error_buf[128]; -#if WASM_ENABLE_LIBC_WASI != 0 - struct mk_list *head; - struct flb_slist_entry *wasi_dir; - const size_t accessible_dir_list_size = mk_list_size(accessible_dir_list); - const char **wasi_dir_list = NULL; - size_t dir_index = 0; -#endif - - wasm_module_t module = NULL; - wasm_module_inst_t module_inst = NULL; - wasm_exec_env_t exec_env = NULL; - - RuntimeInitArgs wasm_args; - - fw = flb_malloc(sizeof(struct flb_wasm)); - if (!fw) { - flb_errno(); - return NULL; - } - fw->tag_buffer = 0; - fw->record_buffer = 0; - -#if WASM_ENABLE_LIBC_WASI != 0 - wasi_dir_list = flb_malloc(sizeof(char *) * accessible_dir_list_size); - if (!wasi_dir_list) { - flb_errno(); - flb_free(fw); - return NULL; - } - mk_list_foreach(head, accessible_dir_list) { - wasi_dir = mk_list_entry(head, struct flb_slist_entry, _head); - wasi_dir_list[dir_index] = wasi_dir->str; - dir_index++; - } -#endif - - fw->config = config; - - memset(&wasm_args, 0, sizeof(RuntimeInitArgs)); - - wasm_args.mem_alloc_type = Alloc_With_Allocator; - wasm_args.mem_alloc_option.allocator.malloc_func = flb_malloc; - wasm_args.mem_alloc_option.allocator.realloc_func = flb_realloc; - wasm_args.mem_alloc_option.allocator.free_func = flb_free; - - if (!wasm_runtime_full_init(&wasm_args)) { - flb_error("Init runtime environment failed."); - return NULL; - } - - if(!flb_wasm_load_wasm_binary(wasm_path, &buffer, &buf_size)) { - goto error; - } - - module = wasm_runtime_load((uint8_t *)buffer, buf_size, error_buf, sizeof(error_buf)); - if (!module) { - flb_error("Load wasm module failed. error: %s", error_buf); - goto error; - } - -#if WASM_ENABLE_LIBC_WASI != 0 - wasm_runtime_set_wasi_args_ex(module, wasi_dir_list, accessible_dir_list_size, NULL, 0, - NULL, 0, NULL, 0, - (stdinfd != -1) ? stdinfd : STDIN_FILENO, - (stdoutfd != -1) ? stdoutfd : STDOUT_FILENO, - (stderrfd != -1) ? stderrfd : STDERR_FILENO); -#endif - - module_inst = wasm_runtime_instantiate(module, stack_size, heap_size, - error_buf, sizeof(error_buf)); - if (!module_inst) { - flb_error("Instantiate wasm module failed. error: %s", error_buf); - goto error; - } - - exec_env = wasm_runtime_create_exec_env(module_inst, stack_size); - if (!exec_env) { - flb_error("Create wasm execution environment failed."); - goto error; - } - - fw->buffer = buffer; - fw->module = module; - fw->module_inst = module_inst; - fw->exec_env = exec_env; - - mk_list_add(&fw->_head, &config->wasm_list); - -#if WASM_ENABLE_LIBC_WASI != 0 - flb_free(wasi_dir_list); -#endif - - return fw; - -error: -#if WASM_ENABLE_LIBC_WASI != 0 - if (wasi_dir_list != NULL) { - flb_free(wasi_dir_list); - } -#endif - if (exec_env) { - wasm_runtime_destroy_exec_env(exec_env); - } - if (module_inst) { - wasm_runtime_deinstantiate(module_inst); - } - if (module) { - wasm_runtime_unload(module); - } - if (buffer != NULL) { - BH_FREE(buffer); - } - if (fw != NULL) { - flb_free(fw); - } - - wasm_runtime_destroy(); - - return NULL; -} - -char *flb_wasm_call_function_format_json(struct flb_wasm *fw, const char *function_name, - const char* tag_data, size_t tag_len, - struct flb_time t, - const char* record_data, size_t record_len) -{ - const char *exception; - uint8_t *func_result; - wasm_function_inst_t func = NULL; - /* We should pass the length that is null terminator included into - * WASM runtime. This is why we add +1 for tag_len and record_len. - */ - fw->tag_buffer = wasm_runtime_module_dup_data(fw->module_inst, tag_data, tag_len+1); - fw->record_buffer = wasm_runtime_module_dup_data(fw->module_inst, record_data, record_len+1); - uint32_t func_args[6] = {fw->tag_buffer, tag_len, - t.tm.tv_sec, t.tm.tv_nsec, - fw->record_buffer, record_len}; - size_t args_size = sizeof(func_args) / sizeof(uint32_t); - - if (!(func = wasm_runtime_lookup_function(fw->module_inst, function_name, NULL))) { - flb_error("The %s wasm function is not found.", function_name); - return NULL; - } - - if (!wasm_runtime_call_wasm(fw->exec_env, func, args_size, func_args)) { - exception = wasm_runtime_get_exception(fw->module_inst); - flb_error("Got exception running wasm code: %s", exception); - wasm_runtime_clear_exception(fw->module_inst); - return NULL; - } - - // The return value is stored in the first element of the function argument array. - // It's a WASM pointer to null-terminated c char string. - // WAMR allows us to map WASM pointers to native pointers. - if (!wasm_runtime_validate_app_str_addr(fw->module_inst, func_args[0])) { - flb_warn("[wasm] returned value is invalid"); - return NULL; - } - func_result = wasm_runtime_addr_app_to_native(fw->module_inst, func_args[0]); - - if (func_result == NULL) { - return NULL; - } - - return (char *)flb_strdup(func_result); -} - -int flb_wasm_call_wasi_main(struct flb_wasm *fw) -{ -#if WASM_ENABLE_LIBC_WASI != 0 - wasm_function_inst_t func = NULL; - - if (!(func = wasm_runtime_lookup_wasi_start_function(fw->module_inst))) { - flb_error("The wasi mode main function is not found."); - return -1; - } - - return wasm_runtime_call_wasm(fw->exec_env, func, 0, NULL); -#else - return -1; -#endif -} - -void flb_wasm_buffer_free(struct flb_wasm *fw) -{ - if (fw->tag_buffer != 0) { - wasm_runtime_module_free(fw->module_inst, fw->tag_buffer); - } - if (fw->record_buffer != 0) { - wasm_runtime_module_free(fw->module_inst, fw->record_buffer); - } -} - -void flb_wasm_destroy(struct flb_wasm *fw) -{ - if (fw->exec_env) { - wasm_runtime_destroy_exec_env(fw->exec_env); - } - if (fw->module_inst) { - flb_wasm_buffer_free(fw); - wasm_runtime_deinstantiate(fw->module_inst); - } - if (fw->module) { - wasm_runtime_unload(fw->module); - } - if (fw->buffer) { - BH_FREE(fw->buffer); - } - wasm_runtime_destroy(); - - mk_list_del(&fw->_head); - flb_free(fw); -} - -int flb_wasm_destroy_all(struct flb_config *ctx) -{ - int c = 0; - struct mk_list *tmp; - struct mk_list *head; - struct flb_wasm *fw; - - mk_list_foreach_safe(head, tmp, &ctx->wasm_list) { - fw = mk_list_entry(head, struct flb_wasm, _head); - flb_wasm_destroy(fw); - c++; - } - - return c; -} diff --git a/fluent-bit/src/win32/winsvc.c b/fluent-bit/src/win32/winsvc.c deleted file mode 100644 index 9e0942ee5..000000000 --- a/fluent-bit/src/win32/winsvc.c +++ /dev/null @@ -1,148 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2019-2020 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <Windows.h> -#include <Shlwapi.h> - -struct flb_config; -extern struct flb_config *config; -extern int flb_engine_exit(struct flb_config*); -extern int flb_main(int, char**); - -/* Windows Service utils */ -#define svc_name "fluent-bit" -static SERVICE_STATUS_HANDLE hstatus; -static int win32_argc; -static char **win32_argv; - -/* - * A Windows Service uses 'C:\Windows\System32' as working directory - * by default. Here we use a more intuitive default path (where - * fluent-bit.exe exists). - */ -static int update_default_workdir(void) -{ - char path[MAX_PATH]; - - if (win32_argc < 1) { - return -1; - } - - if (strcpy_s(path, MAX_PATH, win32_argv[0])) { - return -1; - } - - if (!PathRemoveFileSpecA(path)) { - return -1; - } - - if (!SetCurrentDirectoryA(path)) { - return -1; - } - - return 0; -} - -static void svc_notify(DWORD status) -{ - SERVICE_STATUS ss; - - ss.dwServiceType = SERVICE_WIN32_OWN_PROCESS; - ss.dwCurrentState = status; - ss.dwWin32ExitCode = NO_ERROR; - ss.dwServiceSpecificExitCode = NO_ERROR; - ss.dwControlsAccepted = SERVICE_ACCEPT_STOP; - ss.dwWaitHint = 30000; - ss.dwCheckPoint = 0; - - /* - * According to MSDN (SetServiceStatus), accepting control on - * SERVICE_START_PENDING can crash the service. - */ - if (status == SERVICE_START_PENDING) { - ss.dwControlsAccepted = 0; - } - - SetServiceStatus(hstatus, &ss); -} - -static void WINAPI svc_handler(DWORD ctrl) -{ - switch (ctrl) - { - case SERVICE_CONTROL_STOP: - svc_notify(SERVICE_STOP_PENDING); - flb_engine_exit(config); - return; - default: - break; - } -} - -static void WINAPI svc_main(DWORD svc_argc, LPTSTR *svc_argv) -{ - hstatus = RegisterServiceCtrlHandler(svc_name, svc_handler); - if (!hstatus) { - return; - } - - update_default_workdir(); - - svc_notify(SERVICE_START_PENDING); - flb_main(win32_argc, win32_argv); - svc_notify(SERVICE_STOPPED); -} - -/* - * Notify SCM that Fluent Bit is running. - * - * Note: Call this function in the main execution flow (immediately - * before the engine is starting). - */ -void win32_started(void) -{ - if (hstatus) { - svc_notify(SERVICE_RUNNING); - } -} - -static const SERVICE_TABLE_ENTRY svc_table[] = { - {svc_name, svc_main}, - {NULL, NULL} -}; - -int win32_main(int argc, char **argv) -{ - win32_argc = argc; - win32_argv = argv; - - if (StartServiceCtrlDispatcher(svc_table)) { - return 0; - } - - if (GetLastError() != ERROR_FAILED_SERVICE_CONTROLLER_CONNECT) { - return -1; - } - - /* - * If we cannot connect to SCM, we assume that "fluent-bit.exe" - * was invoked from the command line. - */ - return flb_main(argc, argv); -} |